diff --git a/build.gradle b/build.gradle index e12d520e12de6..1b6b82d51c2d4 100644 --- a/build.gradle +++ b/build.gradle @@ -146,7 +146,7 @@ project.ext.externalDependency = [ 'log4jApi': "org.apache.logging.log4j:log4j-api:$log4jVersion", 'log4j12Api': "org.slf4j:log4j-over-slf4j:$slf4jVersion", 'log4j2Api': "org.apache.logging.log4j:log4j-to-slf4j:$log4jVersion", - 'lombok': 'org.projectlombok:lombok:1.18.12', + 'lombok': 'org.projectlombok:lombok:1.18.16', 'mariadbConnector': 'org.mariadb.jdbc:mariadb-java-client:2.6.0', 'mavenArtifact': "org.apache.maven:maven-artifact:$mavenVersion", 'mixpanel': 'com.mixpanel:mixpanel-java:1.4.4', @@ -187,7 +187,7 @@ project.ext.externalDependency = [ 'springBeans': "org.springframework:spring-beans:$springVersion", 'springContext': "org.springframework:spring-context:$springVersion", 'springCore': "org.springframework:spring-core:$springVersion", - 'springDocUI': 'org.springdoc:springdoc-openapi-ui:1.6.7', + 'springDocUI': 'org.springdoc:springdoc-openapi-ui:1.6.14', 'springJdbc': "org.springframework:spring-jdbc:$springVersion", 'springWeb': "org.springframework:spring-web:$springVersion", 'springWebMVC': "org.springframework:spring-webmvc:$springVersion", @@ -197,9 +197,11 @@ project.ext.externalDependency = [ 'springBootStarterWeb': "org.springframework.boot:spring-boot-starter-web:$springBootVersion", 'springBootStarterJetty': "org.springframework.boot:spring-boot-starter-jetty:$springBootVersion", 'springBootStarterCache': "org.springframework.boot:spring-boot-starter-cache:$springBootVersion", + 'springBootStarterValidation': "org.springframework.boot:spring-boot-starter-validation:$springBootVersion", 'springKafka': 'org.springframework.kafka:spring-kafka:2.8.11', 'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion", 'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.1.12', + 'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.41', 'testng': 'org.testng:testng:7.3.0', 'testContainers': 'org.testcontainers:testcontainers:' + testContainersVersion, 'testContainersJunit': 'org.testcontainers:junit-jupiter:' + testContainersVersion, diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index f88d2bdb966ce..65b3780431db9 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -13,4 +13,7 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.5' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.5' implementation 'commons-io:commons-io:2.11.0' + + compileOnly 'org.projectlombok:lombok:1.18.14' + annotationProcessor 'org.projectlombok:lombok:1.18.14' } \ No newline at end of file diff --git a/buildSrc/src/main/java/com/linkedin/metadata/models/registry/config b/buildSrc/src/main/java/com/linkedin/metadata/models/registry/config new file mode 120000 index 0000000000000..ea22cc67da2d4 --- /dev/null +++ b/buildSrc/src/main/java/com/linkedin/metadata/models/registry/config @@ -0,0 +1 @@ +../../../../../../../../../entity-registry/src/main/java/com/linkedin/metadata/models/registry/config \ No newline at end of file diff --git a/buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java b/buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java index 796d622860c15..25bf239ab835b 100644 --- a/buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java +++ b/buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java @@ -20,6 +20,7 @@ import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; + import org.gradle.api.DefaultTask; import org.gradle.api.tasks.*; @@ -31,13 +32,27 @@ public class GenerateJsonSchemaTask extends DefaultTask { private String inputDirectory; private String outputDirectory; + private ArrayNode aspectType; private Path combinedDirectory; + private Path jsonDirectory; public static final String sep = FileSystems.getDefault().getSeparator(); private static final JsonNodeFactory NODE_FACTORY = JacksonUtils.nodeFactory(); + private static final OpenApiEntities openApiEntities = new OpenApiEntities(NODE_FACTORY); + + @InputFile + @PathSensitive(PathSensitivity.NAME_ONLY) + public String getEntityRegistryYaml() { + return openApiEntities.getEntityRegistryYaml(); + } + + public void setEntityRegistryYaml(String entityRegistryYaml) { + openApiEntities.setEntityRegistryYaml(entityRegistryYaml); + } + public void setInputDirectory(String inputDirectory) { this.inputDirectory = inputDirectory; } @@ -78,6 +93,7 @@ public void generate() throws IOException { .filter(Files::isRegularFile) .map(Path::toFile) .forEach(this::generateSchema); + List nodesList = Files.walk(jsonDirectory) .filter(Files::isRegularFile) .filter(path -> { @@ -108,6 +124,18 @@ public void generate() throws IOException { } schemasNode.setAll(definitions); }); + + combinedDirectory = Paths.get(outputDirectory + sep + "combined"); + try { + Files.createDirectory(combinedDirectory); + } catch (FileAlreadyExistsException fae) { + // No-op + } + + // Add additional components and paths + openApiEntities.setCombinedDirectory(combinedDirectory); + ObjectNode extendedNode = openApiEntities.entityExtension(nodesList, schemasNode); + /* Minimal OpenAPI header openapi: 3.0.1 @@ -131,29 +159,23 @@ public void generate() throws IOException { .set("paths", NODE_FACTORY.objectNode() .set("/path", NODE_FACTORY.objectNode() .set("get", NODE_FACTORY.objectNode().set("tags", NODE_FACTORY.arrayNode().add("path"))))); - JsonNode combinedSchemaDefinitionsYaml = ((ObjectNode) NODE_FACTORY.objectNode().set("components", - NODE_FACTORY.objectNode().set("schemas", schemasNode))).setAll(yamlHeader); + + JsonNode combinedSchemaDefinitionsYaml = extendedNode.setAll(yamlHeader); final String yaml = new YAMLMapper().writeValueAsString(combinedSchemaDefinitionsYaml) - .replaceAll("definitions", "components/schemas") - .replaceAll("\n\\s+- type: \"null\"", ""); + .replaceAll("definitions", "components/schemas") + .replaceAll("\n\\s+description: null", "") + .replaceAll("\n\\s+- type: \"null\"", ""); - combinedDirectory = Paths.get(outputDirectory + sep + "combined"); - try { - Files.createDirectory(combinedDirectory); - } catch (FileAlreadyExistsException fae) { - // No-op - } Files.write(Paths.get(combinedDirectory + sep + "open-api.yaml"), yaml.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - JsonNode combinedSchemaDefinitionsJson = NODE_FACTORY.objectNode().set("definitions",schemasNode); + JsonNode combinedSchemaDefinitionsJson = NODE_FACTORY.objectNode().set("definitions", extendedNode); String prettySchema = JacksonUtils.prettyPrint(combinedSchemaDefinitionsJson); Files.write(Paths.get(Paths.get(outputDirectory) + sep + "combined" + sep + "schema.json"), prettySchema.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - } private final HashSet filenames = new HashSet<>(); @@ -183,5 +205,4 @@ private void generateSchema(final File file) { throw new RuntimeException(e); } } - } \ No newline at end of file diff --git a/buildSrc/src/main/java/io/datahubproject/OpenApiEntities.java b/buildSrc/src/main/java/io/datahubproject/OpenApiEntities.java new file mode 100644 index 0000000000000..7fbf013384b7d --- /dev/null +++ b/buildSrc/src/main/java/io/datahubproject/OpenApiEntities.java @@ -0,0 +1,603 @@ +package io.datahubproject; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; +import com.linkedin.metadata.models.registry.config.Entities; +import com.linkedin.metadata.models.registry.config.Entity; +import org.gradle.internal.Pair; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +public class OpenApiEntities { + private final static String MODEL_VERSION = "_v2"; + private final static String REQUEST_SUFFIX = "Request" + MODEL_VERSION; + private final static String RESPONSE_SUFFIX = "Response" + MODEL_VERSION; + + private final static String ASPECT_REQUEST_SUFFIX = "Aspect" + REQUEST_SUFFIX; + private final static String ASPECT_RESPONSE_SUFFIX = "Aspect" + RESPONSE_SUFFIX; + private final static String ENTITY_REQUEST_SUFFIX = "Entity" + REQUEST_SUFFIX; + private final static String ENTITY_RESPONSE_SUFFIX = "Entity" + RESPONSE_SUFFIX; + + private final JsonNodeFactory NODE_FACTORY; + private Map entityMap; + private String entityRegistryYaml; + private Path combinedDirectory; + + private final static Set SUPPORTED_ASPECT_PATHS = Set.of( + "domains", "ownership", "deprecation", "status", "globalTags", "glossaryTerms", "dataContractInfo", + "browsePathsV2" + ); + + public OpenApiEntities(JsonNodeFactory NODE_FACTORY) { + this.NODE_FACTORY = NODE_FACTORY; + } + + public String getEntityRegistryYaml() { + return entityRegistryYaml; + } + + public void setEntityRegistryYaml(String entityRegistryYaml) { + this.entityRegistryYaml = entityRegistryYaml; + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + mapper.findAndRegisterModules(); + try { + Entities entities = mapper.readValue(Paths.get(entityRegistryYaml).toFile(), Entities.class); + entityMap = entities.getEntities().stream() + .filter(e -> "core".equals(e.getCategory())) + .collect(Collectors.toMap(Entity::getName, Function.identity())); + } catch (IOException e) { + throw new IllegalArgumentException( + String.format("Error while reading entity yaml file in path %s: %s", entityRegistryYaml, e.getMessage())); + } + } + + public Path getCombinedDirectory() { + return combinedDirectory; + } + + public void setCombinedDirectory(Path combinedDirectory) { + this.combinedDirectory = combinedDirectory; + } + + public ObjectNode entityExtension(List nodesList, ObjectNode schemasNode) throws IOException { + // Generate entities schema + Set aspectDefinitions = nodesList.stream() + .map(nl -> nl.get("definitions").fieldNames()) + .flatMap(it -> StreamSupport.stream(Spliterators.spliteratorUnknownSize(it, Spliterator.ORDERED), false)) + .collect(Collectors.toSet()); + withWrappedAspects(schemasNode, aspectDefinitions); + + // Add entity schema + Set entitySchema = withEntitySchema(schemasNode, aspectDefinitions); + + // Write specific sections: components.* and paths + Set modelDefinitions = Stream.concat(aspectDefinitions.stream(), entitySchema.stream()) + .collect(Collectors.toSet()); + + // Just the component & parameters schema + Pair> parameters = buildParameters(schemasNode, modelDefinitions); + ObjectNode componentsNode = writeComponentsYaml(schemasNode, parameters.left()); + + // Just the entity paths + writePathsYaml(modelDefinitions, parameters.right()); + + return componentsNode; + } + + private static String toUpperFirst(String s) { + return s.substring(0, 1).toUpperCase() + s.substring(1); + } + + private Set withEntitySchema(ObjectNode schemasNode, Set definitions) { + return entityMap.values().stream() + // Make sure the primary key is defined + .filter(entity -> definitions.contains(toUpperFirst(entity.getKeyAspect()))) + .map(entity -> { + final String upperName = toUpperFirst(entity.getName()); + + ObjectNode entityDefinitions = NODE_FACTORY.objectNode(); + entityDefinitions.set(upperName + ENTITY_RESPONSE_SUFFIX, buildEntitySchema(entity, definitions, true)); + entityDefinitions.set(upperName + ENTITY_REQUEST_SUFFIX, buildEntitySchema(entity, definitions, false)); + entityDefinitions.set("Scroll" + upperName + ENTITY_RESPONSE_SUFFIX, buildEntityScrollSchema(entity)); + + schemasNode.setAll(entityDefinitions); + + return upperName; + }).collect(Collectors.toSet()); + } + + + private Set withWrappedAspects(ObjectNode schemasNode, Set aspects) { + return aspects.stream().peek(aspect -> { + ObjectNode aspectRef = NODE_FACTORY.objectNode() + .put("$ref", "#/definitions/" + aspect); + + ObjectNode responseProperties = NODE_FACTORY.objectNode(); + responseProperties.set("value", aspectRef); + responseProperties.set("systemMetadata", NODE_FACTORY.objectNode() + .put("description", "System metadata for the aspect.") + .put("$ref", "#/definitions/SystemMetadata")); + + ObjectNode responseWrapper = NODE_FACTORY.objectNode() + .put("type", "object") + .put("description", "Aspect wrapper object.") + .set("properties", responseProperties); + responseWrapper.set("required", NODE_FACTORY.arrayNode().add("value")); + schemasNode.set(aspect + ASPECT_RESPONSE_SUFFIX, responseWrapper); + + ObjectNode requestProperties = NODE_FACTORY.objectNode(); + requestProperties.set("value", aspectRef); + + ObjectNode requestWrapper = NODE_FACTORY.objectNode() + .put("type", "object") + .put("description", "Aspect wrapper object.") + .set("properties", requestProperties); + requestWrapper.set("required", NODE_FACTORY.arrayNode().add("value")); + schemasNode.set(aspect + ASPECT_REQUEST_SUFFIX, requestWrapper); + }).collect(Collectors.toSet()); + } + + private ObjectNode buildEntitySchema(Entity entity, Set aspectDefinitions, boolean isResponse) { + ObjectNode propertiesNode = NODE_FACTORY.objectNode(); + + propertiesNode.set("urn", NODE_FACTORY.objectNode() + .put("description", "Unique id for " + entity.getName()) + .put("type", "string")); + + propertiesNode.set(entity.getKeyAspect(), buildAspectRef(entity.getKeyAspect(), isResponse)); + + entity.getAspects().stream() + .filter(aspect -> aspectDefinitions.contains(toUpperFirst(aspect))) // Only if aspect is defined + .forEach(aspect -> propertiesNode.set(aspect, buildAspectRef(aspect, isResponse))); + + ObjectNode entityNode = NODE_FACTORY.objectNode() + .put("type", "object") + .put("description", Optional.ofNullable(entity.getDoc()) + .orElse(toUpperFirst(entity.getName()) + " object.")) + .set("properties", propertiesNode); + entityNode.set("required", NODE_FACTORY.arrayNode().add("urn")); + + return entityNode; + } + + private ObjectNode buildEntityScrollSchema(Entity entity) { + ObjectNode scrollResponsePropertiesNode = NODE_FACTORY.objectNode(); + + scrollResponsePropertiesNode.set("scrollId", NODE_FACTORY.objectNode() + .put("description", "Scroll id for pagination.") + .put("type", "string")); + + scrollResponsePropertiesNode.set("entities", NODE_FACTORY.objectNode() + .put("description", Optional.ofNullable(entity.getDoc()) + .orElse(toUpperFirst(entity.getName()) + " object.")) + .put("type", "array") + .set("items", NODE_FACTORY.objectNode().put("$ref", + String.format("#/components/schemas/%s%s", toUpperFirst(entity.getName()), ENTITY_RESPONSE_SUFFIX)))); + + ObjectNode scrollResponseNode = NODE_FACTORY.objectNode() + .put("type", "object") + .put("description", "Scroll across " + toUpperFirst(entity.getName()) + " objects.") + .set("properties", scrollResponsePropertiesNode); + scrollResponseNode.set("required", NODE_FACTORY.arrayNode().add("entities")); + + return scrollResponseNode; + } + + + private ObjectNode buildAspectRef(String aspect, boolean withSystemMetadata) { + if (withSystemMetadata) { + return NODE_FACTORY.objectNode() + .put("$ref", String.format("#/definitions/%s%s", toUpperFirst(aspect), ASPECT_RESPONSE_SUFFIX)); + } else { + return NODE_FACTORY.objectNode() + .put("$ref", String.format("#/definitions/%s%s", toUpperFirst(aspect), ASPECT_REQUEST_SUFFIX)); + } + } + + private Optional> generateEntityParameters(final Entity entity, Set definitions) { + /* + If not missing key + */ + if (definitions.contains(toUpperFirst(entity.getKeyAspect()))) { + final String parameterName = toUpperFirst(entity.getName()) + "Aspects"; + + ArrayNode aspects = NODE_FACTORY.arrayNode(); + entity.getAspects().stream() + .filter(aspect -> definitions.contains(toUpperFirst(aspect))) // Only if aspect is defined + .distinct() + .forEach(aspects::add); + + if (aspects.isEmpty()) { + aspects.add(entity.getKeyAspect()); + } + + ObjectNode itemsNode = NODE_FACTORY.objectNode() + .put("type", "string"); + itemsNode.set("enum", aspects); + itemsNode.set("default", aspects); + + ObjectNode schemaNode = NODE_FACTORY.objectNode() + .put("type", "array") + .set("items", itemsNode); + ObjectNode parameterSchemaNode = NODE_FACTORY.objectNode() + .put("in", "query") + .put("name", "aspects") + .put("explode", true) + .put("description", "Aspects to include in response.") + .set("schema", schemaNode); + + parameterSchemaNode.set("example", aspects); + + ObjectNode parameterNode = NODE_FACTORY.objectNode() + .set(parameterName + MODEL_VERSION, parameterSchemaNode); + + return Optional.of(Pair.of(parameterName, parameterNode)); + } + + return Optional.empty(); + } + + private Pair> buildParameters(ObjectNode schemasNode, Set definitions) { + ObjectNode parametersNode = NODE_FACTORY.objectNode(); + Set parameterDefinitions = entityMap.values().stream() + .flatMap(entity -> generateEntityParameters(entity, definitions).stream()) + .map(entityNode -> { + parametersNode.setAll(entityNode.right()); + return entityNode.left(); + }) + .collect(Collectors.toSet()); + + return Pair.of(extraParameters(parametersNode), parameterDefinitions); + } + + private ObjectNode writeComponentsYaml(ObjectNode schemasNode, ObjectNode parametersNode) throws IOException { + ObjectNode componentsNode = NODE_FACTORY.objectNode(); + componentsNode.set("schemas", schemasNode); + componentsNode.set("parameters", extraParameters(parametersNode)); + ObjectNode componentsDocNode = NODE_FACTORY.objectNode().set("components", componentsNode); + + final String componentsYaml = new YAMLMapper().writeValueAsString(componentsDocNode) + .replaceAll("definitions", "components/schemas") + .replaceAll("\n\\s+description: null", "") + .replaceAll("\n\\s+- type: \"null\"", ""); + Files.write(Paths.get(combinedDirectory + GenerateJsonSchemaTask.sep + "open-api-components.yaml"), + componentsYaml.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING); + + return componentsDocNode; + } + + private ObjectNode extraParameters(ObjectNode parametersNode) { + parametersNode.set("ScrollId" + MODEL_VERSION, NODE_FACTORY.objectNode() + .put("in", "query") + .put("name", "scrollId") + .put("description", "Scroll pagination token.") + .set("schema", NODE_FACTORY.objectNode() + .put("type", "string"))); + + ArrayNode sortFields = NODE_FACTORY.arrayNode(); + sortFields.add("urn"); + ObjectNode sortFieldsNode = NODE_FACTORY.objectNode() + .put("type", "string"); + sortFieldsNode.set("enum", sortFields); + sortFieldsNode.set("default", sortFields.get(0)); + + ObjectNode sortFieldsSchemaNode = NODE_FACTORY.objectNode() + .put("type", "array") + .put("default", "urn") + .set("items", sortFieldsNode); + parametersNode.set("SortBy" + MODEL_VERSION, NODE_FACTORY.objectNode() + .put("in", "query") + .put("name", "sort") + .put("explode", true) + .put("description", "Sort fields for pagination.") + .put("example", "urn") + .set("schema", sortFieldsSchemaNode)); + + parametersNode.set("SortOrder" + MODEL_VERSION, NODE_FACTORY.objectNode() + .put("in", "query") + .put("name", "sortOrder") + .put("explode", true) + .put("description", "Sort direction field for pagination.") + .put("example", "ASCENDING") + .set("schema", NODE_FACTORY.objectNode() + .put("default", "ASCENDING") + .put("$ref", "#/components/schemas/SortOrder"))); + + parametersNode.set("PaginationCount" + MODEL_VERSION, NODE_FACTORY.objectNode() + .put("in", "query") + .put("name", "count") + .put("description", "Number of items per page.") + .put("example", "10") + .set("schema", NODE_FACTORY.objectNode() + .put("type", "integer") + .put("default", 10) + .put("minimum", 1))); + parametersNode.set("ScrollQuery" + MODEL_VERSION, NODE_FACTORY.objectNode() + .put("in", "query") + .put("name", "query") + .put("description", "Structured search query.") + .put("example", "*") + .set("schema", NODE_FACTORY.objectNode() + .put("type", "string") + .put("default", "*"))); + + return parametersNode; + } + + private void writePathsYaml(Set modelDefinitions, Set parameterDefinitions) throws IOException { + ObjectNode pathsNode = NODE_FACTORY.objectNode(); + + entityMap.values().stream() + .filter(e -> modelDefinitions.contains(toUpperFirst(e.getName()))) + .forEach(entity -> { + + pathsNode.set(String.format("/%s", entity.getName().toLowerCase()), + buildListEntityPath(entity, parameterDefinitions)); + + pathsNode.set(String.format("/%s/{urn}", entity.getName().toLowerCase()), + buildSingleEntityPath(entity, parameterDefinitions)); + + }); + + buildEntityAspectPaths(pathsNode, modelDefinitions); + + ObjectNode pathsDocNode = NODE_FACTORY.objectNode().set("paths", pathsNode); + + final String componentsYaml = new YAMLMapper().writeValueAsString(pathsDocNode) + .replaceAll("\n\\s+- type: \"null\"", "") + .replaceAll("\n\\s+description: null", ""); + Files.write(Paths.get(combinedDirectory + GenerateJsonSchemaTask.sep + "open-api-paths.yaml"), + componentsYaml.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING); + } + + private void buildEntityAspectPaths(ObjectNode pathsNode, Set modelDefinitions) { + entityMap.values().stream() + .filter(e -> modelDefinitions.contains(toUpperFirst(e.getName()))) + .forEach(entity -> { + entity.getAspects().stream() + .filter(aspect -> SUPPORTED_ASPECT_PATHS.contains(aspect)) + .filter(aspect -> modelDefinitions.contains(toUpperFirst(aspect))) + .forEach(aspect -> pathsNode.set(String.format("/%s/{urn}/%s", + entity.getName().toLowerCase(), aspect.toLowerCase()), + buildSingleEntityAspectPath(entity, aspect))); + }); + } + + private ObjectNode buildListEntityPath(Entity entity, Set parameterDefinitions) { + final String upperFirst = toUpperFirst(entity.getName()); + final String aspectParameterName = upperFirst + "Aspects"; + ArrayNode tagsNode = NODE_FACTORY.arrayNode() + .add(entity.getName() + " Entity"); + + ObjectNode scrollMethod = NODE_FACTORY.objectNode() + .put("summary", String.format("Scroll %s.", upperFirst)) + .put("operationId", String.format("scroll", upperFirst)); + + ArrayNode scrollPathParametersNode = NODE_FACTORY.arrayNode(); + scrollMethod.set("parameters", scrollPathParametersNode); + scrollPathParametersNode.add(NODE_FACTORY.objectNode() + .put("in", "query") + .put("name", "systemMetadata") + .put("description", "Include systemMetadata with response.") + .set("schema", NODE_FACTORY.objectNode() + .put("type", "boolean") + .put("default", false))); + if (parameterDefinitions.contains(aspectParameterName)) { + scrollPathParametersNode.add(NODE_FACTORY.objectNode() + .put("$ref", String.format("#/components/parameters/%s", aspectParameterName + MODEL_VERSION))); + } + scrollPathParametersNode.add(NODE_FACTORY.objectNode() + .put("$ref", "#/components/parameters/PaginationCount" + MODEL_VERSION)); + scrollPathParametersNode.add(NODE_FACTORY.objectNode() + .put("$ref", "#/components/parameters/ScrollId" + MODEL_VERSION)); + scrollPathParametersNode.add(NODE_FACTORY.objectNode() + .put("$ref", "#/components/parameters/SortBy" + MODEL_VERSION)); + scrollPathParametersNode.add(NODE_FACTORY.objectNode() + .put("$ref", "#/components/parameters/SortOrder" + MODEL_VERSION)); + scrollPathParametersNode.add(NODE_FACTORY.objectNode() + .put("$ref", "#/components/parameters/ScrollQuery" + MODEL_VERSION)); + scrollMethod.set("parameters", scrollPathParametersNode); + scrollMethod.set("responses", NODE_FACTORY.objectNode() + .set("200", NODE_FACTORY.objectNode().put("description", "Success") + .set("content", NODE_FACTORY.objectNode().set("application/json", NODE_FACTORY.objectNode() + .set("schema", NODE_FACTORY.objectNode() + .put("$ref", String.format("#/components/schemas/Scroll%s%s", upperFirst, ENTITY_RESPONSE_SUFFIX))))))); + scrollMethod.set("tags", tagsNode); + + ObjectNode postMethod = NODE_FACTORY.objectNode() + .put("summary", "Create " + upperFirst) + .put("operationId", String.format("create", upperFirst)); + postMethod.set("requestBody", NODE_FACTORY.objectNode() + .put("description", "Create " + entity.getName() + " entities.") + .put("required", true) + .set("content", NODE_FACTORY.objectNode().set("application/json", NODE_FACTORY.objectNode() + .set("schema", NODE_FACTORY.objectNode().put("type", "array") + .set("items", NODE_FACTORY.objectNode().put("$ref", + String.format("#/components/schemas/%s%s", upperFirst, ENTITY_REQUEST_SUFFIX))))))); + postMethod.set("responses", NODE_FACTORY.objectNode() + .set("201", NODE_FACTORY.objectNode().put("description", "Create " + entity.getName() + " entities.") + .set("content", NODE_FACTORY.objectNode().set("application/json", NODE_FACTORY.objectNode() + .set("schema", NODE_FACTORY.objectNode().put("type", "array") + .set("items", NODE_FACTORY.objectNode().put("$ref", + String.format("#/components/schemas/%s%s", upperFirst, ENTITY_RESPONSE_SUFFIX)))))))); + postMethod.set("tags", tagsNode); + + ObjectNode listMethods = NODE_FACTORY.objectNode(); + listMethods.set("get", scrollMethod); + listMethods.set("post", postMethod); + + return listMethods; + } + + private ObjectNode buildSingleEntityPath(Entity entity, Set parameterDefinitions) { + final String upperFirst = toUpperFirst(entity.getName()); + final String aspectParameterName = upperFirst + "Aspects"; + ArrayNode tagsNode = NODE_FACTORY.arrayNode().add(entity.getName() + " Entity"); + + ObjectNode getMethod = NODE_FACTORY.objectNode() + .put("summary", String.format("Get %s by key.", entity.getName())) + .put("operationId", String.format("get", upperFirst)); + getMethod.set("tags", tagsNode); + ArrayNode singlePathParametersNode = NODE_FACTORY.arrayNode(); + getMethod.set("parameters", singlePathParametersNode); + singlePathParametersNode.add(NODE_FACTORY.objectNode() + .put("in", "query") + .put("name", "systemMetadata") + .put("description", "Include systemMetadata with response.") + .set("schema", NODE_FACTORY.objectNode() + .put("type", "boolean") + .put("default", false))); + if(parameterDefinitions.contains(aspectParameterName)) { + singlePathParametersNode.add(NODE_FACTORY.objectNode() + .put("$ref", String.format("#/components/parameters/%s", aspectParameterName + MODEL_VERSION))); + } + + ObjectNode responses = NODE_FACTORY.objectNode(); + getMethod.set("responses", responses); + responses.set("200", NODE_FACTORY.objectNode().put("description", "Success") + .set("content", NODE_FACTORY.objectNode().set("application/json", NODE_FACTORY.objectNode() + .set("schema", NODE_FACTORY.objectNode().put("$ref", + String.format("#/components/schemas/%s%s", upperFirst, ENTITY_RESPONSE_SUFFIX)))))); + responses.set("404", NODE_FACTORY.objectNode() + .put("description", "Not Found") + .set("content", NODE_FACTORY.objectNode() + .set("application/json", NODE_FACTORY.objectNode() + .set("schema", NODE_FACTORY.objectNode())))); + + ObjectNode headResponses = NODE_FACTORY.objectNode(); + headResponses.set("204", NODE_FACTORY.objectNode() + .put("description", entity.getName() + " exists.") + .set("content", NODE_FACTORY.objectNode() + .set("application/json", NODE_FACTORY.objectNode()))); + headResponses.set("404", NODE_FACTORY.objectNode() + .put("description", entity.getName() + " does not exist.") + .set("content", NODE_FACTORY.objectNode() + .set("application/json", NODE_FACTORY.objectNode()))); + ObjectNode headMethod = NODE_FACTORY.objectNode() + .put("summary", upperFirst + " existence.") + .put("operationId", String.format("head", upperFirst)) + .set("responses", headResponses); + headMethod.set("tags", tagsNode); + + ObjectNode deleteMethod = NODE_FACTORY.objectNode() + .put("summary", "Delete entity " + upperFirst) + .put("operationId", String.format("delete", upperFirst)) + .set("responses", NODE_FACTORY.objectNode() + .set("200", NODE_FACTORY.objectNode() + .put("description", "Delete " + entity.getName() + " entity.") + .set("content", NODE_FACTORY.objectNode() + .set("application/json", NODE_FACTORY.objectNode())))); + deleteMethod.set("tags", tagsNode); + + ObjectNode singlePathMethods = NODE_FACTORY.objectNode() + .set("parameters", NODE_FACTORY.arrayNode() + .add(NODE_FACTORY.objectNode() + .put("in", "path") + .put("name", "urn") + .put("required", true) + .set("schema", NODE_FACTORY.objectNode().put("type", "string")))); + singlePathMethods.set("get", getMethod); + singlePathMethods.set("head", headMethod); + singlePathMethods.set("delete", deleteMethod); + + return singlePathMethods; + } + + private ObjectNode buildSingleEntityAspectPath(Entity entity, String aspect) { + final String upperFirstEntity = toUpperFirst(entity.getName()); + final String upperFirstAspect = toUpperFirst(aspect); + + ArrayNode tagsNode = NODE_FACTORY.arrayNode() + .add(aspect + " Aspect"); + + ObjectNode getMethod = NODE_FACTORY.objectNode() + .put("summary", String.format("Get %s for %s.", aspect, entity.getName())) + .put("operationId", String.format("get%s", upperFirstAspect, upperFirstEntity)); + getMethod.set("tags", tagsNode); + ArrayNode singlePathParametersNode = NODE_FACTORY.arrayNode(); + getMethod.set("parameters", singlePathParametersNode); + singlePathParametersNode.add(NODE_FACTORY.objectNode() + .put("in", "query") + .put("name", "systemMetadata") + .put("description", "Include systemMetadata with response.") + .set("schema", NODE_FACTORY.objectNode() + .put("type", "boolean") + .put("default", false))); + getMethod.set("responses", NODE_FACTORY.objectNode().set("200", NODE_FACTORY.objectNode() + .put("description", "Success").set("content", NODE_FACTORY.objectNode() + .set("application/json", NODE_FACTORY.objectNode().set("schema", NODE_FACTORY.objectNode() + .put("$ref", + String.format("#/components/schemas/%s%s", upperFirstAspect, ASPECT_RESPONSE_SUFFIX))))))); + + ObjectNode headResponses = NODE_FACTORY.objectNode(); + headResponses.set("200", NODE_FACTORY.objectNode() + .put("description", String.format("%s on %s exists.", aspect, entity.getName())) + .set("content", NODE_FACTORY.objectNode() + .set("application/json", NODE_FACTORY.objectNode()))); + headResponses.set("404", NODE_FACTORY.objectNode() + .put("description", String.format("%s on %s does not exist.", aspect, entity.getName())) + .set("content", NODE_FACTORY.objectNode() + .set("application/json", NODE_FACTORY.objectNode()))); + ObjectNode headMethod = NODE_FACTORY.objectNode() + .put("summary", String.format("%s on %s existence.", aspect, upperFirstEntity)) + .put("operationId", String.format("head%s", upperFirstAspect, upperFirstEntity)) + .set("responses", headResponses); + headMethod.set("tags", tagsNode); + + ObjectNode deleteMethod = NODE_FACTORY.objectNode() + .put("summary", String.format("Delete %s on entity %s", aspect, upperFirstEntity)) + .put("operationId", String.format("delete%s", upperFirstAspect, upperFirstEntity)) + .set("responses", NODE_FACTORY.objectNode() + .set("200", NODE_FACTORY.objectNode() + .put("description", String.format("Delete %s on %s entity.", aspect, upperFirstEntity)) + .set("content", NODE_FACTORY.objectNode() + .set("application/json", NODE_FACTORY.objectNode())))); + deleteMethod.set("tags", tagsNode); + + ObjectNode postMethod = NODE_FACTORY.objectNode() + .put("summary", String.format("Create aspect %s on %s ", aspect, upperFirstEntity)) + .put("operationId", String.format("create%s", upperFirstAspect, upperFirstEntity)); + postMethod.set("requestBody", NODE_FACTORY.objectNode() + .put("description", String.format("Create aspect %s on %s entity.", aspect, upperFirstEntity)) + .put("required", true).set("content", NODE_FACTORY.objectNode() + .set("application/json", NODE_FACTORY.objectNode().set("schema", NODE_FACTORY.objectNode() + .put("$ref", + String.format("#/components/schemas/%s%s", upperFirstAspect, ASPECT_REQUEST_SUFFIX)))))); + postMethod.set("responses", NODE_FACTORY.objectNode().set("201", NODE_FACTORY.objectNode() + .put("description", String.format("Create aspect %s on %s entity.", aspect, upperFirstEntity)) + .set("content", NODE_FACTORY.objectNode().set("application/json", NODE_FACTORY.objectNode() + .set("schema", NODE_FACTORY.objectNode().put("$ref", + String.format("#/components/schemas/%s%s", upperFirstAspect, ASPECT_RESPONSE_SUFFIX))))))); + postMethod.set("tags", tagsNode); + + ObjectNode singlePathMethods = NODE_FACTORY.objectNode() + .set("parameters", NODE_FACTORY.arrayNode() + .add(NODE_FACTORY.objectNode() + .put("in", "path") + .put("name", "urn") + .put("required", true) + .set("schema", NODE_FACTORY.objectNode().put("type", "string")))); + singlePathMethods.set("get", getMethod); + singlePathMethods.set("head", headMethod); + singlePathMethods.set("delete", deleteMethod); + singlePathMethods.set("post", postMethod); + + return singlePathMethods; + } +} diff --git a/datahub-frontend/conf/routes b/datahub-frontend/conf/routes index 38e6f769027f0..3102c26497fed 100644 --- a/datahub-frontend/conf/routes +++ b/datahub-frontend/conf/routes @@ -33,6 +33,8 @@ GET /openapi/*path c POST /openapi/*path controllers.Application.proxy(path: String, request: Request) DELETE /openapi/*path controllers.Application.proxy(path: String, request: Request) PUT /openapi/*path controllers.Application.proxy(path: String, request: Request) +HEAD /openapi/*path controllers.Application.proxy(path: String, request: Request) +PATCH /openapi/*path controllers.Application.proxy(path: String, request: Request) # Map static resources from the /public folder to the /assets URL path GET /assets/*file controllers.Assets.at(path="/public", file) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java index f0899f8fbc0cb..0ec71e4ad58e4 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java @@ -145,6 +145,9 @@ private void mapDatasetProperties(@Nonnull Dataset dataset, @Nonnull DataMap dat properties.setQualifiedName(gmsProperties.getQualifiedName()); dataset.setProperties(properties); dataset.setDescription(properties.getDescription()); + if (gmsProperties.getUri() != null) { + dataset.setUri(gmsProperties.getUri().toString()); + } TimeStamp created = gmsProperties.getCreated(); if (created != null) { properties.setCreated(created.getTime()); diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/config/Entity.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/config/Entity.java index c446f63b65321..f32aa1aa8bd47 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/config/Entity.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/config/Entity.java @@ -1,12 +1,14 @@ package com.linkedin.metadata.models.registry.config; import java.util.List; + import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.NoArgsConstructor; import lombok.Value; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import javax.annotation.Nullable; @Value @@ -18,4 +20,7 @@ public class Entity { String doc; String keyAspect; List aspects; + + @Nullable + String category; } diff --git a/li-utils/build.gradle b/li-utils/build.gradle index e8b672a3a21fa..8f526cffba094 100644 --- a/li-utils/build.gradle +++ b/li-utils/build.gradle @@ -36,5 +36,5 @@ idea { } } -// Need to compile backing java definitions with the data template. +// Need to compile backing java parameterDefinitions with the data template. sourceSets.mainGeneratedDataTemplate.java.srcDirs('src/main/javaPegasus/') \ No newline at end of file diff --git a/metadata-auth/auth-api/build.gradle b/metadata-auth/auth-api/build.gradle index 2bf9e5243e152..7159aa5f15e61 100644 --- a/metadata-auth/auth-api/build.gradle +++ b/metadata-auth/auth-api/build.gradle @@ -31,7 +31,7 @@ dependencies() { api project(path: ':metadata-utils') implementation externalDependency.guava - implementation externalDependency.lombok + compileOnly externalDependency.lombok annotationProcessor externalDependency.lombok diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle index e304bb5329c62..fc72fc4257491 100644 --- a/metadata-integration/java/datahub-client/build.gradle +++ b/metadata-integration/java/datahub-client/build.gradle @@ -56,7 +56,7 @@ dependencies { testImplementation externalDependency.httpAsyncClient testRuntimeOnly externalDependency.logbackClassic - swaggerCodegen 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.33' + swaggerCodegen externalDependency.swaggerCli } task copyAvroSchemas { diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle index d2b584ceb6745..a2c643516dce6 100644 --- a/metadata-io/build.gradle +++ b/metadata-io/build.gradle @@ -43,7 +43,6 @@ dependencies { implementation externalDependency.resilience4j api externalDependency.springContext implementation externalDependency.swaggerAnnotations - swaggerCodegen 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.33' implementation(externalDependency.mixpanel) { exclude group: 'org.json', module: 'json' } @@ -121,33 +120,6 @@ project.compileJava { } } -tasks.register('generateOpenApiPojos', GenerateSwaggerCode) { - it.setInputFile( - file( - "${project(':metadata-models').projectDir}/src/generatedJsonSchema/combined/open-api.yaml" - ) - ) - it.setOutputDir(file("$projectDir/generated")) - it.setLanguage("spring") - it.setComponents(['models']) - it.setTemplateDir(file("$projectDir/src/main/resources/JavaSpring")) - it.setAdditionalProperties([ - "group-id" : "io.datahubproject", - "dateLibrary" : "java8", - "java11" : "true", - "modelPropertyNaming" : "original", - "modelPackage" : "io.datahubproject.openapi.generated"] as Map) - - dependsOn ':metadata-models:generateJsonSchema' -} - -compileJava.dependsOn generateOpenApiPojos -processResources.dependsOn generateOpenApiPojos -sourceSets.main.java.srcDir "${generateOpenApiPojos.outputDir}/src/main/java" -sourceSets.main.resources.srcDir "${generateOpenApiPojos.outputDir}/src/main/resources" - -checkstyleMain.exclude '**/generated/**' - clean { project.delete("$projectDir/generated") } \ No newline at end of file diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java index 03b81cfc703c4..02f3a7f835dcb 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java @@ -1147,12 +1147,13 @@ public void ingestEntities(@Nonnull final List entities, @Nonnull final } @Override - public void ingestEntity(Entity entity, AuditStamp auditStamp) { + public SystemMetadata ingestEntity(Entity entity, AuditStamp auditStamp) { SystemMetadata generatedSystemMetadata = new SystemMetadata(); generatedSystemMetadata.setRunId(DEFAULT_RUN_ID); generatedSystemMetadata.setLastObserved(System.currentTimeMillis()); ingestEntity(entity, auditStamp, generatedSystemMetadata); + return generatedSystemMetadata; } @Override @@ -1541,6 +1542,13 @@ public Boolean isSoftDeleted(@Nonnull final Urn urn) { return statusAspect != null && ((Status) statusAspect).isRemoved(); } + @Override + public Boolean exists(Urn urn, String aspectName) { + EntityAspectIdentifier dbKey = new EntityAspectIdentifier(urn.toString(), aspectName, ASPECT_LATEST_VERSION); + Map aspects = _aspectDao.batchGet(Set.of(dbKey)); + return aspects.values().stream().anyMatch(Objects::nonNull); + } + @Nullable @Override public RollbackResult deleteAspect(String urn, String aspectName, @Nonnull Map conditions, boolean hardDelete) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java index a045bb357d5f0..94b8d57efcc16 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java @@ -174,7 +174,7 @@ private List getEntitiesToSearch(@Nonnull List inputEntities) { */ @Nonnull public ScrollResult scrollAcrossEntities(@Nonnull List entities, @Nonnull String input, - @Nullable Filter postFilters, @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nonnull String keepAlive, + @Nullable Filter postFilters, @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nullable String keepAlive, int size, @Nullable SearchFlags searchFlags) { log.debug(String.format( "Searching Search documents entities: %s, input: %s, postFilters: %s, sortCriterion: %s, from: %s, size: %s", diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java index f698e28c0be6d..13a7d16b723a7 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java @@ -127,7 +127,7 @@ public ScrollResult scroll( @Nullable Filter filters, @Nullable SortCriterion sortCriterion, @Nullable String scrollId, - @Nonnull String keepAlive, + @Nullable String keepAlive, int size, @Nullable SearchFlags flags) { return getCachedScrollResults(entities, query, filters, sortCriterion, scrollId, keepAlive, size, flags); @@ -238,7 +238,7 @@ public ScrollResult getCachedScrollResults( @Nullable Filter filters, @Nullable SortCriterion sortCriterion, @Nullable String scrollId, - @Nonnull String keepAlive, + @Nullable String keepAlive, int size, @Nullable SearchFlags flags) { try (Timer.Context ignored = MetricUtils.timer(this.getClass(), "getCachedScrollResults").time()) { @@ -326,7 +326,7 @@ private ScrollResult getRawScrollResults( final Filter filters, final SortCriterion sortCriterion, @Nullable final String scrollId, - @Nonnull final String keepAlive, + @Nullable final String keepAlive, final int count, final boolean fulltext) { if (fulltext) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java index ce7b44c715d6b..32adce458770d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java @@ -27,6 +27,7 @@ import com.linkedin.metadata.shared.ElasticSearchIndexed; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.elasticsearch.action.search.SearchResponse; @Slf4j @@ -174,7 +175,7 @@ public List getBrowsePaths(@Nonnull String entityName, @Nonnull Urn urn) @Nonnull @Override public ScrollResult fullTextScroll(@Nonnull List entities, @Nonnull String input, @Nullable Filter postFilters, - @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nonnull String keepAlive, int size) { + @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nullable String keepAlive, int size) { log.debug(String.format( "Scrolling Structured Search documents entities: %s, input: %s, postFilters: %s, sortCriterion: %s, scrollId: %s, size: %s", entities, input, postFilters, sortCriterion, scrollId, size)); @@ -185,7 +186,7 @@ public ScrollResult fullTextScroll(@Nonnull List entities, @Nonnull Stri @Nonnull @Override public ScrollResult structuredScroll(@Nonnull List entities, @Nonnull String input, @Nullable Filter postFilters, - @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nonnull String keepAlive, int size) { + @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nullable String keepAlive, int size) { log.debug(String.format( "Scrolling FullText Search documents entities: %s, input: %s, postFilters: %s, sortCriterion: %s, scrollId: %s, size: %s", entities, input, postFilters, sortCriterion, scrollId, size)); @@ -193,6 +194,10 @@ public ScrollResult structuredScroll(@Nonnull List entities, @Nonnull St new SearchFlags().setFulltext(false)); } + public Optional raw(@Nonnull String indexName, @Nullable String jsonQuery) { + return esSearchDAO.raw(indexName, jsonQuery); + } + @Override public int maxResultSize() { return ESUtils.MAX_RESULT_SIZE; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java index e204cb6fd6fbe..f3864d99ba5e9 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java @@ -26,8 +26,10 @@ import io.opentelemetry.extension.annotations.WithSpan; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -40,6 +42,13 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.core.CountRequest; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.builder.SearchSourceBuilder; import static com.linkedin.metadata.Constants.*; import static com.linkedin.metadata.models.registry.template.util.TemplateUtil.*; @@ -52,6 +61,11 @@ @Slf4j @RequiredArgsConstructor public class ESSearchDAO { + private static final NamedXContentRegistry X_CONTENT_REGISTRY; + static { + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); + X_CONTENT_REGISTRY = new NamedXContentRegistry(searchModule.getNamedXContents()); + } private final EntityRegistry entityRegistry; private final RestHighLevelClient client; @@ -285,7 +299,7 @@ public Map aggregateByValue(@Nullable String entityName, @Nonnull */ @Nonnull public ScrollResult scroll(@Nonnull List entities, @Nonnull String input, @Nullable Filter postFilters, - @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nonnull String keepAlive, int size, SearchFlags searchFlags) { + @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nullable String keepAlive, int size, SearchFlags searchFlags) { final String finalInput = input.isEmpty() ? "*" : input; String[] indexArray = entities.stream() .map(indexConvention::getEntityIndexName) @@ -302,11 +316,11 @@ public ScrollResult scroll(@Nonnull List entities, @Nonnull String input if (supportsPointInTime()) { if (System.currentTimeMillis() + 10000 <= searchAfterWrapper.getExpirationTime()) { pitId = searchAfterWrapper.getPitId(); - } else { + } else if (keepAlive != null) { pitId = createPointInTime(indexArray, keepAlive); } } - } else if (supportsPointInTime()) { + } else if (supportsPointInTime() && keepAlive != null) { pitId = createPointInTime(indexArray, keepAlive); } @@ -326,6 +340,23 @@ public ScrollResult scroll(@Nonnull List entities, @Nonnull String input return executeAndExtract(entitySpecs, searchRequest, transformedFilters, scrollId, keepAlive, size); } + public Optional raw(@Nonnull String indexName, @Nullable String jsonQuery) { + return Optional.ofNullable(jsonQuery).map(json -> { + try { + XContentParser parser = XContentType.JSON.xContent().createParser(X_CONTENT_REGISTRY, + LoggingDeprecationHandler.INSTANCE, json); + SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(parser); + + SearchRequest searchRequest = new SearchRequest(indexConvention.getIndexName(indexName)); + searchRequest.source(searchSourceBuilder); + + return client.search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + private boolean supportsPointInTime() { return pointInTimeCreationEnabled && ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH.equalsIgnoreCase(elasticSearchImplementation); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java index 5973f77da28aa..dbd933d59d7f3 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java @@ -228,7 +228,7 @@ public SearchRequest getSearchRequest(@Nonnull String input, @Nullable Filter fi @Nonnull @WithSpan public SearchRequest getSearchRequest(@Nonnull String input, @Nullable Filter filter, - @Nullable SortCriterion sortCriterion, @Nullable Object[] sort, @Nullable String pitId, @Nonnull String keepAlive, + @Nullable SortCriterion sortCriterion, @Nullable Object[] sort, @Nullable String pitId, @Nullable String keepAlive, int size, SearchFlags searchFlags) { SearchRequest searchRequest = new PITAwareSearchRequest(); SearchFlags finalSearchFlags = applyDefaultSearchFlags(searchFlags, input, DEFAULT_SERVICE_SEARCH_FLAGS); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java index 741eb5568d2ea..12c081a5c25a6 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java @@ -261,11 +261,11 @@ public static String extractTargetIndex(String id) { } public static void setSearchAfter(SearchSourceBuilder searchSourceBuilder, @Nullable Object[] sort, - @Nullable String pitId, String keepAlive) { + @Nullable String pitId, @Nullable String keepAlive) { if (sort != null && sort.length > 0) { searchSourceBuilder.searchAfter(sort); } - if (StringUtils.isNotBlank(pitId)) { + if (StringUtils.isNotBlank(pitId) && keepAlive != null) { PointInTimeBuilder pointInTimeBuilder = new PointInTimeBuilder(pitId); pointInTimeBuilder.setKeepAlive(TimeValue.parseTimeValue(keepAlive, "keepAlive")); searchSourceBuilder.pointInTimeBuilder(pointInTimeBuilder); diff --git a/metadata-models/build.gradle b/metadata-models/build.gradle index 2e8efae9b7bce..db01be3ccebdf 100644 --- a/metadata-models/build.gradle +++ b/metadata-models/build.gradle @@ -2,6 +2,7 @@ import io.datahubproject.GenerateJsonSchemaTask apply plugin: 'java-library' apply plugin: 'pegasus' +apply plugin: 'org.hidetake.swagger.generator' tasks.withType(JavaCompile).configureEach { javaCompiler = javaToolchains.compilerFor { @@ -24,9 +25,25 @@ dependencies { api project(':li-utils') dataModel project(':li-utils') + compileOnly externalDependency.lombok + annotationProcessor externalDependency.lombok + compileOnly externalDependency.swaggerAnnotations + compileOnly externalDependency.springBootStarterValidation + compileOnly externalDependency.jacksonCore + compileOnly externalDependency.jacksonDataBind + + swaggerCodegen externalDependency.swaggerCli testImplementation externalDependency.guava } +sourceSets { + main { + java { + srcDirs = ["$buildDir/openapi/generated/src/main/java"] + } + } +} + mainAvroSchemaJar.dependsOn generateAvroSchema pegasus.main.generationModes = [PegasusGenerationMode.PEGASUS, PegasusGenerationMode.AVRO] @@ -35,9 +52,32 @@ pegasus.main.generationModes = [PegasusGenerationMode.PEGASUS, PegasusGeneration tasks.register('generateJsonSchema', GenerateJsonSchemaTask) { it.setInputDirectory("$projectDir/src/mainGeneratedAvroSchema") it.setOutputDirectory("$projectDir/src/generatedJsonSchema") + it.setEntityRegistryYaml("${project(':metadata-models').projectDir}/src/main/resources/entity-registry.yml") dependsOn generateAvroSchema } -clean { - project.delete("$projectDir/src/generatedJsonSchema") -} \ No newline at end of file +// https://github.com/int128/gradle-swagger-generator-plugin#task-type-generateswaggercode +task openApiGenerate(type: GenerateSwaggerCode, dependsOn: 'generateJsonSchema') { + inputFile = file("$projectDir/src/generatedJsonSchema/combined/open-api.yaml") + outputDir = file("$buildDir/openapi/generated") + language = "spring" + components = ["models"] + templateDir = file("$projectDir/src/main/resources/JavaSpring") + additionalProperties = [ + 'group-id' : "io.datahubproject", + 'dateLibrary' : "java8", + 'java11' : "true", + 'modelPropertyNaming': "original", + 'modelPackage' : "io.datahubproject.openapi.generated" + ] +} +tasks.getByName("compileJava").dependsOn(openApiGenerate) + +checkstyleMain.exclude '**/generated/**' + +task cleanExtraDirs { + delete "$projectDir/src/generatedJsonSchema" +} +clean.finalizedBy(cleanExtraDirs) + +checkstyleMain.exclude '**/generated/**' diff --git a/metadata-models/src/main/resources/JavaSpring/interface.mustache b/metadata-models/src/main/resources/JavaSpring/interface.mustache new file mode 100644 index 0000000000000..b09d4f3a66cc0 --- /dev/null +++ b/metadata-models/src/main/resources/JavaSpring/interface.mustache @@ -0,0 +1,25 @@ +{{#jackson}} +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +{{/jackson}} +/** +* {{#description}}{{.}}{{/description}}{{^description}}{{classname}}{{/description}} +*/ +{{#jackson}} +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "__type") +@JsonSubTypes({ + {{#subTypes}} + @JsonSubTypes.Type(value = {{classname}}.class, name = "{{classname}}"){{^@last}},{{/@last}} + {{/subTypes}} +}) +{{/jackson}} +public interface {{{classname}}} { +{{#vendorExtensions}} +{{#x-discriminator-type-getter}} + {{x-discriminator-type}} {{x-discriminator-type-getter}}(); +{{/x-discriminator-type-getter}} +{{/vendorExtensions}} +} \ No newline at end of file diff --git a/metadata-models/src/main/resources/JavaSpring/model.mustache b/metadata-models/src/main/resources/JavaSpring/model.mustache new file mode 100644 index 0000000000000..a048f249a6b3d --- /dev/null +++ b/metadata-models/src/main/resources/JavaSpring/model.mustache @@ -0,0 +1,41 @@ +package {{package}}; + +{{^x-is-composed-model}} +import java.util.Objects; +{{#imports}}import {{import}}; +{{/imports}} +{{#serializableModel}} +import java.io.Serializable; +{{/serializableModel}} +{{#useBeanValidation}} +import org.springframework.validation.annotation.Validated; +import javax.validation.Valid; +import com.fasterxml.jackson.annotation.JsonInclude; +import javax.validation.constraints.*; +{{/useBeanValidation}} +{{#jackson}} +{{#withXml}} +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +{{/withXml}} +{{/jackson}} +{{#withXml}} +import javax.xml.bind.annotation.*; +{{/withXml}} +{{/x-is-composed-model}} + +{{#models}} +{{#model}} +{{#isComposedModel}} +{{>interface}} +{{/isComposedModel}} +{{^isComposedModel}} +{{#isEnum}} +{{>enumOuterClass}} +{{/isEnum}} +{{^isEnum}} +{{>pojo}} +{{/isEnum}} +{{/isComposedModel}} +{{/model}} +{{/models}} \ No newline at end of file diff --git a/metadata-models/src/main/resources/JavaSpring/pojo.mustache b/metadata-models/src/main/resources/JavaSpring/pojo.mustache new file mode 100644 index 0000000000000..e0b6f3e7a1a6f --- /dev/null +++ b/metadata-models/src/main/resources/JavaSpring/pojo.mustache @@ -0,0 +1,177 @@ +{{#if hasVars}} +{{else}} +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +{{/if}} +import lombok.Builder; +import lombok.Getter; +import lombok.extern.jackson.Jacksonized; +{{#if interfaceModels}} +import com.fasterxml.jackson.annotation.JsonTypeInfo; +{{/if}} +/** + * {{#description}}{{.}}{{/description}}{{^description}}{{classname}}{{/description}} + */{{#description}} +{{#useOas2}}@ApiModel{{/useOas2}}{{^useOas2}}@Schema{{/useOas2}}(description = "{{{description}}}"){{/description}} +{{#useBeanValidation}}@Validated{{/useBeanValidation}} +{{>generatedAnnotation}}{{#discriminator}}{{>typeInfoAnnotation}}{{/discriminator}}{{>xmlAnnotation}} +@Jacksonized @Builder(toBuilder = true) +{{#if interfaceModels}} +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.EXISTING_PROPERTY, property = "__type") +{{/if}} +@JsonInclude(JsonInclude.Include.NON_NULL) +public class {{classname}} {{#parent}}extends {{{parent}}}{{/parent}} {{#serializableModel}}implements Serializable {{#interfaceModels}}, {{classname}}{{^@last}}, {{/@last}}{{#@last}} {{/@last}}{{/interfaceModels}}{{/serializableModel}}{{^serializableModel}}{{#interfaceModels}}{{#@first}}implements {{/@first}}{{classname}}{{^@last}}, {{/@last}}{{#@last}}{{/@last}}{{/interfaceModels}}{{/serializableModel}} { +{{#serializableModel}} + private static final long serialVersionUID = 1L; + +{{/serializableModel}} +{{#if interfaceModels}} + + @Getter(value = lombok.AccessLevel.NONE) + @JsonProperty(value = "__type", defaultValue = "{{classname}}") @Builder.Default + private String __type = "{{classname}}"; + + /** + * Name of this subclass in SimpleClassName format + * @return __type + **/ + @Schema(required = true, description = "Name of this subclass in SimpleClassName format", allowableValues = {"{{classname}}"}, + defaultValue = "{{classname}}") + @NotNull + public String get__type() { + return __type; + } +{{/if}} + + {{#vars}} + {{#baseItems this}} + {{#isEnum}} +{{>enumClass}} + {{/isEnum}} + {{/baseItems}} + {{#jackson}} + {{#vendorExtensions.x-is-discriminator-property}} + @JsonTypeId + {{/vendorExtensions.x-is-discriminator-property}} + {{^vendorExtensions.x-is-discriminator-property}} + @JsonProperty("{{baseName}}") @Builder.Default{{#withXml}} + @JacksonXmlProperty({{#isXmlAttribute}}isAttribute = true, {{/isXmlAttribute}}{{#xmlNamespace}}namespace="{{xmlNamespace}}", {{/xmlNamespace}}localName = "{{#xmlName}}{{xmlName}}{{/xmlName}}{{^xmlName}}{{baseName}}{{/xmlName}}"){{/withXml}} + {{/vendorExtensions.x-is-discriminator-property}} + {{/jackson}} + {{#gson}} + @SerializedName("{{baseName}}") + {{/gson}} + {{#isContainer}} + {{#useBeanValidation}}@Valid{{/useBeanValidation}} + private {{{datatypeWithEnum}}} {{name}}{{#required}} = {{{defaultValue}}}{{/required}}{{^required}} = null{{/required}}; + {{/isContainer}} + {{^isContainer}} + private {{{datatypeWithEnum}}} {{name}} = {{{defaultValue}}}; + {{/isContainer}} + + {{/vars}} + {{#vars}} + public {{classname}} {{name}}({{{datatypeWithEnum}}} {{name}}) { + this.{{name}} = {{name}}; + return this; + } + {{#isListContainer}} + + public {{classname}} add{{nameInCamelCase}}Item({{{items.datatypeWithEnum}}} {{name}}Item) { + {{^required}} + if (this.{{name}} == null) { + this.{{name}} = {{{defaultValue}}}; + } + {{/required}} + this.{{name}}.add({{name}}Item); + return this; + } + {{/isListContainer}} + {{#isMapContainer}} + + public {{classname}} put{{nameInCamelCase}}Item(String key, {{{items.datatypeWithEnum}}} {{name}}Item) { + {{^required}} + if (this.{{name}} == null) { + this.{{name}} = {{{defaultValue}}}; + } + {{/required}} + this.{{name}}.put(key, {{name}}Item); + return this; + } + {{/isMapContainer}} + + /** + {{#description}} + * {{{description}}} + {{/description}} + {{^description}} + * Get {{name}} + {{/description}} + {{#minimum}} + * minimum: {{minimum}} + {{/minimum}} + {{#maximum}} + * maximum: {{maximum}} + {{/maximum}} + * @return {{name}} + **/ + {{#vendorExtensions.extraAnnotation}} + {{{vendorExtensions.extraAnnotation}}} + {{/vendorExtensions.extraAnnotation}} + {{#useOas2}} + @ApiModelProperty({{#example}}example = "{{{example}}}", {{/example}}{{#required}}required = {{required}}, {{/required}}{{#isReadOnly}}readOnly = {{{isReadOnly}}}, {{/isReadOnly}}value = "{{{description}}}") + {{/useOas2}} + {{^useOas2}} + @Schema({{#example}}example = "{{{example}}}", {{/example}}{{#required}}required = {{required}}, {{/required}}{{#isReadOnly}}accessMode = Schema.AccessMode.READ_ONLY, {{/isReadOnly}}description = "{{{description}}}") + {{/useOas2}} + {{#useBeanValidation}}{{>beanValidation}}{{/useBeanValidation}} public {{{datatypeWithEnum}}} {{getter}}() { + return {{name}}; + } + + public void {{setter}}({{{datatypeWithEnum}}} {{name}}) { + this.{{name}} = {{name}}; + } + + {{/vars}} + + @Override + public boolean equals(java.lang.Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + }{{#hasVars}} + {{classname}} {{classVarName}} = ({{classname}}) o; + return {{#vars}}Objects.equals(this.{{name}}, {{classVarName}}.{{name}}){{#hasMore}} && + {{/hasMore}}{{/vars}}{{#parent}} && + super.equals(o){{/parent}};{{/hasVars}}{{^hasVars}} + return true;{{/hasVars}} + } + + @Override + public int hashCode() { + return Objects.hash({{#vars}}{{name}}{{#hasMore}}, {{/hasMore}}{{/vars}}{{#parent}}{{#hasVars}}, {{/hasVars}}super.hashCode(){{/parent}}); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class {{classname}} {\n"); + {{#parent}}sb.append(" ").append(toIndentedString(super.toString())).append("\n");{{/parent}} + {{#vars}}sb.append(" {{name}}: ").append(toIndentedString({{name}})).append("\n"); + {{/vars}}sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} \ No newline at end of file diff --git a/metadata-models/src/main/resources/JavaSpring/readme.txt b/metadata-models/src/main/resources/JavaSpring/readme.txt new file mode 100644 index 0000000000000..f127cd3e68553 --- /dev/null +++ b/metadata-models/src/main/resources/JavaSpring/readme.txt @@ -0,0 +1,2 @@ +Original: +https://github.com/swagger-api/swagger-codegen-generators/tree/master/src/main/resources/handlebars/JavaSpring \ No newline at end of file diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml index 5f54525e1e862..56fc5f6568eb7 100644 --- a/metadata-models/src/main/resources/entity-registry.yml +++ b/metadata-models/src/main/resources/entity-registry.yml @@ -100,6 +100,7 @@ entities: - dataProcessInstanceRelationships - dataProcessInstanceRunEvent - name: chart + category: core keyAspect: chartKey aspects: - chartInfo @@ -183,6 +184,7 @@ entities: - origin - name: domain doc: A data domain within an organization. + category: core keyAspect: domainKey aspects: - domainProperties @@ -190,6 +192,7 @@ entities: - ownership - name: container doc: A container of related data assets. + category: core keyAspect: containerKey aspects: - containerProperties @@ -206,6 +209,7 @@ entities: - domains - browsePathsV2 - name: tag + category: core keyAspect: tagKey aspects: - tagProperties @@ -213,6 +217,7 @@ entities: - deprecation - status - name: glossaryTerm + category: core keyAspect: glossaryTermKey aspects: - glossaryTermInfo @@ -225,6 +230,7 @@ entities: - status - browsePaths - name: glossaryNode + category: core keyAspect: glossaryNodeKey aspects: - glossaryNodeInfo diff --git a/metadata-service/health-servlet/src/main/java/com/datahub/health/config/SpringWebConfig.java b/metadata-service/health-servlet/src/main/java/com/datahub/health/config/SpringWebConfig.java index 4354ef04b9d8f..76d9a6744c4cf 100644 --- a/metadata-service/health-servlet/src/main/java/com/datahub/health/config/SpringWebConfig.java +++ b/metadata-service/health-servlet/src/main/java/com/datahub/health/config/SpringWebConfig.java @@ -1,6 +1,7 @@ package com.datahub.health.config; import io.swagger.v3.oas.annotations.OpenAPIDefinition; +import io.swagger.v3.oas.annotations.info.Info; import io.swagger.v3.oas.annotations.servers.Server; import java.util.List; import org.springframework.context.annotation.Configuration; @@ -15,7 +16,8 @@ @EnableWebMvc -@OpenAPIDefinition(servers = {@Server(url = "/health/", description = "Default Server URL")}) +@OpenAPIDefinition(info = @Info(title = "DataHub OpenAPI", version = "1.0.0"), + servers = {@Server(url = "/health/", description = "Default Server URL")}) @Configuration public class SpringWebConfig implements WebMvcConfigurer { diff --git a/metadata-service/openapi-analytics-servlet/build.gradle b/metadata-service/openapi-analytics-servlet/build.gradle new file mode 100644 index 0000000000000..6475d215db5f5 --- /dev/null +++ b/metadata-service/openapi-analytics-servlet/build.gradle @@ -0,0 +1,67 @@ +plugins { + id 'java' + id 'org.hidetake.swagger.generator' +} + +dependencies { + + implementation project(':metadata-auth:auth-api') + implementation project(':metadata-service:auth-impl') + implementation project(':metadata-service:factories') + implementation project(':metadata-service:openapi-servlet') + implementation project(':metadata-models') + + implementation externalDependency.springBoot + implementation externalDependency.springCore + implementation externalDependency.springDocUI + implementation externalDependency.springWeb + implementation externalDependency.springWebMVC + implementation externalDependency.springBeans + implementation externalDependency.springContext + + implementation externalDependency.reflections + implementation externalDependency.slf4jApi + compileOnly externalDependency.lombok + + implementation externalDependency.antlr4Runtime + implementation externalDependency.antlr4 + + annotationProcessor externalDependency.lombok + + testImplementation externalDependency.testng + testImplementation externalDependency.mockito + testImplementation externalDependency.springBootTest + + swaggerCodegen externalDependency.swaggerCli +} + +sourceSets { + main { + java { + srcDirs = ["$buildDir/openapi/generated/src/main/java", 'src/main/java'] + } + } +} + +// https://github.com/int128/gradle-swagger-generator-plugin#task-type-generateswaggercode +task openApiGenerate(type: GenerateSwaggerCode) { + inputFile = file("$projectDir/src/main/resources/open-api.yaml") + outputDir = file("$buildDir/openapi/generated") + + language = 'spring' + + components = ["apis"] + templateDir = file("$projectDir/src/main/resources/JavaSpring") + additionalProperties = [ + 'group-id' : "io.datahubproject", + 'dateLibrary' : "java8", + 'java11' : "true", + 'modelPropertyNaming': "original", + 'modelPackage' : "io.datahubproject.openapi.generated", + 'apiPackage' : "io.datahubproject.openapi.generated.controller", + 'delegatePattern' : "true" + ] +} +tasks.getByName("compileJava").dependsOn(openApiGenerate) + +checkstyleMain.exclude '**/generated/**' \ No newline at end of file diff --git a/metadata-service/openapi-analytics-servlet/src/main/java/io/datahubproject/openapi/config/OpenapiAnalyticsConfig.java b/metadata-service/openapi-analytics-servlet/src/main/java/io/datahubproject/openapi/config/OpenapiAnalyticsConfig.java new file mode 100644 index 0000000000000..7816e81fe4a6d --- /dev/null +++ b/metadata-service/openapi-analytics-servlet/src/main/java/io/datahubproject/openapi/config/OpenapiAnalyticsConfig.java @@ -0,0 +1,14 @@ +package io.datahubproject.openapi.config; + +import io.datahubproject.openapi.delegates.DatahubUsageEventsImpl; +import io.datahubproject.openapi.generated.controller.DatahubUsageEventsApiDelegate; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class OpenapiAnalyticsConfig { + @Bean + public DatahubUsageEventsApiDelegate datahubUsageEventsApiDelegate() { + return new DatahubUsageEventsImpl(); + } +} diff --git a/metadata-service/openapi-analytics-servlet/src/main/java/io/datahubproject/openapi/delegates/DatahubUsageEventsImpl.java b/metadata-service/openapi-analytics-servlet/src/main/java/io/datahubproject/openapi/delegates/DatahubUsageEventsImpl.java new file mode 100644 index 0000000000000..99e47f32555df --- /dev/null +++ b/metadata-service/openapi-analytics-servlet/src/main/java/io/datahubproject/openapi/delegates/DatahubUsageEventsImpl.java @@ -0,0 +1,48 @@ +package io.datahubproject.openapi.delegates; + +import com.linkedin.metadata.search.elasticsearch.ElasticSearchService; +import io.datahubproject.openapi.generated.controller.DatahubUsageEventsApiDelegate; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import com.datahub.authentication.Authentication; +import com.datahub.authentication.AuthenticationContext; +import com.datahub.authorization.ConjunctivePrivilegeGroup; +import com.datahub.authorization.DisjunctivePrivilegeGroup; +import com.datahub.authorization.AuthorizerChain; +import org.springframework.beans.factory.annotation.Value; +import com.google.common.collect.ImmutableList; +import io.datahubproject.openapi.exception.UnauthorizedException; +import com.datahub.authorization.AuthUtil; +import com.linkedin.metadata.authorization.PoliciesConfig; + +import java.util.Optional; +import java.util.Objects; + +public class DatahubUsageEventsImpl implements DatahubUsageEventsApiDelegate { + + @Autowired + private ElasticSearchService _searchService; + @Autowired + private AuthorizerChain _authorizationChain; + @Value("${authorization.restApiAuthorization:false}") + private boolean _restApiAuthorizationEnabled; + + final public static String DATAHUB_USAGE_INDEX = "datahub_usage_event"; + + @Override + public ResponseEntity raw(String body) { + Authentication authentication = AuthenticationContext.getAuthentication(); + checkAnalyticsAuthorized(authentication); + return ResponseEntity.of(_searchService.raw(DATAHUB_USAGE_INDEX, body).map(Objects::toString)); + } + + private void checkAnalyticsAuthorized(Authentication authentication) { + String actorUrnStr = authentication.getActor().toUrnStr(); + DisjunctivePrivilegeGroup orGroup = new DisjunctivePrivilegeGroup(ImmutableList.of(new ConjunctivePrivilegeGroup( + ImmutableList.of(PoliciesConfig.GET_ANALYTICS_PRIVILEGE.getType())))); + + if (_restApiAuthorizationEnabled && !AuthUtil.isAuthorized(_authorizationChain, actorUrnStr, Optional.empty(), orGroup)) { + throw new UnauthorizedException(actorUrnStr + " is unauthorized to get analytics."); + } + } +} diff --git a/metadata-service/openapi-analytics-servlet/src/main/resources/JavaSpring/apiController.mustache b/metadata-service/openapi-analytics-servlet/src/main/resources/JavaSpring/apiController.mustache new file mode 100644 index 0000000000000..6b22940f0e7ed --- /dev/null +++ b/metadata-service/openapi-analytics-servlet/src/main/resources/JavaSpring/apiController.mustache @@ -0,0 +1,160 @@ +package {{package}}; + +{{^isJava8or11}} +{{#imports}}import {{import}}; +{{/imports}} +{{/isJava8or11}} +{{^isDelegate}} +import com.fasterxml.jackson.databind.ObjectMapper; +{{/isDelegate}} +{{^isJava8or11}} +import io.swagger.annotations.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +{{/isJava8or11}} +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; +{{^isJava8or11}} +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RequestPart; +import org.springframework.web.multipart.MultipartFile; + +{{#useBeanValidation}} +{{#jakarta}} +import jakarta.validation.Valid; +import jakarta.validation.constraints.*; +{{/jakarta}} +{{^jakarta}} +import javax.validation.Valid; +import javax.validation.constraints.*; +{{/jakarta}} +{{/useBeanValidation}} +{{/isJava8or11}} +{{^isDelegate}} +{{#jakarta}} +import jakarta.servlet.http.HttpServletRequest; +{{/jakarta}} +{{^jakarta}} +import javax.servlet.http.HttpServletRequest; +{{/jakarta}} + {{#isJava8or11}} +import java.util.Optional; + {{/isJava8or11}} +{{/isDelegate}} +{{^jdk8-no-delegate}} + {{#useOptional}} +import java.util.Optional; + {{/useOptional}} +{{/jdk8-no-delegate}} +{{^isJava8or11}} + {{^isDelegate}} +import java.io.IOException; + {{/isDelegate}} +import java.util.List; + {{#async}} +import java.util.concurrent.Callable; + {{/async}} +{{/isJava8or11}} +{{>generatedAnnotation}} +@Controller +@RequestMapping("/v1/analytics") +{{#operations}} +public class {{classname}}Controller implements {{classname}} { + +{{#isDelegate}} + private final {{classname}}Delegate delegate; + + @org.springframework.beans.factory.annotation.Autowired + public {{classname}}Controller({{classname}}Delegate delegate) { + this.delegate = delegate; + } + {{#isJava8or11}} + + @Override + public {{classname}}Delegate getDelegate() { + return delegate; + } + {{/isJava8or11}} +{{/isDelegate}} +{{^isDelegate}} + {{^isJava8or11}} + private static final Logger log = LoggerFactory.getLogger({{classname}}Controller.class); + + {{/isJava8or11}} + private final ObjectMapper objectMapper; + + private final HttpServletRequest request; + + @org.springframework.beans.factory.annotation.Autowired + public {{classname}}Controller(ObjectMapper objectMapper, HttpServletRequest request) { + this.objectMapper = objectMapper; + this.request = request; + } + {{#isJava8or11}} + + @Override + public Optional getObjectMapper() { + return Optional.ofNullable(objectMapper); + } + + @Override + public Optional getRequest() { + return Optional.ofNullable(request); + } + {{/isJava8or11}} + +{{/isDelegate}} +{{^isJava8or11}} +{{#operation}} + public {{#async}}Callable<{{/async}}ResponseEntity<{{>returnTypes}}>{{#async}}>{{/async}} {{operationId}}({{#allParams}}{{>queryParams}}{{>pathParams}}{{>headerParams}}{{>bodyParams}}{{>formParams}}{{#hasMore}},{{/hasMore}}{{/allParams}}) { + {{^isDelegate}} + {{^async}} + String accept = request.getHeader("Accept"); + {{#examples}} + if (accept != null && accept.contains("{{{contentType}}}")) { + try { + return new ResponseEntity<{{>returnTypes}}>(objectMapper.readValue("{{#lambdaRemoveLineBreak}}{{#lambdaEscapeDoubleQuote}}{{{example}}}{{/lambdaEscapeDoubleQuote}}{{/lambdaRemoveLineBreak}}", {{>exampleReturnTypes}}.class), HttpStatus.NOT_IMPLEMENTED); + } catch (IOException e) { + log.error("Couldn't serialize response for content type {{{contentType}}}", e); + return new ResponseEntity<{{>returnTypes}}>(HttpStatus.INTERNAL_SERVER_ERROR); + } + } + + {{/examples}} + return new ResponseEntity<{{>returnTypes}}>(HttpStatus.NOT_IMPLEMENTED); + {{/async}} + {{#async}} + return new CallablereturnTypes}}>>() { + @Override + public ResponseEntity<{{>returnTypes}}> call() { + String accept = request.getHeader("Accept"); + {{#examples}} + if (accept != null && accept.contains("{{{contentType}}}")) { + try { + return new ResponseEntity<{{>returnTypes}}>(objectMapper.readValue("{{#lambdaRemoveLineBreak}}{{#lambdaEscapeDoubleQuote}}{{{example}}}{{/lambdaEscapeDoubleQuote}}{{/lambdaRemoveLineBreak}}", {{>exampleReturnTypes}}.class), HttpStatus.NOT_IMPLEMENTED); + } catch (IOException e) { + log.error("Couldn't serialize response for content type {{{contentType}}}", e); + return new ResponseEntity<{{>returnTypes}}>(HttpStatus.INTERNAL_SERVER_ERROR); + } + } + + {{/examples}} + return new ResponseEntity<{{>returnTypes}}>(HttpStatus.NOT_IMPLEMENTED); + } + }; + {{/async}} + {{/isDelegate}} + {{#isDelegate}} + return delegate.{{operationId}}({{#allParams}}{{paramName}}{{#hasMore}}, {{/hasMore}}{{/allParams}}); + {{/isDelegate}} + } + +{{/operation}} +{{/isJava8or11}} +} +{{/operations}} \ No newline at end of file diff --git a/metadata-service/openapi-analytics-servlet/src/main/resources/JavaSpring/readme.txt b/metadata-service/openapi-analytics-servlet/src/main/resources/JavaSpring/readme.txt new file mode 100644 index 0000000000000..f127cd3e68553 --- /dev/null +++ b/metadata-service/openapi-analytics-servlet/src/main/resources/JavaSpring/readme.txt @@ -0,0 +1,2 @@ +Original: +https://github.com/swagger-api/swagger-codegen-generators/tree/master/src/main/resources/handlebars/JavaSpring \ No newline at end of file diff --git a/metadata-service/openapi-analytics-servlet/src/main/resources/open-api.yaml b/metadata-service/openapi-analytics-servlet/src/main/resources/open-api.yaml new file mode 100644 index 0000000000000..3c756b100699f --- /dev/null +++ b/metadata-service/openapi-analytics-servlet/src/main/resources/open-api.yaml @@ -0,0 +1,31 @@ +openapi: "3.0.0" +info: + title: Analytics API + description: This is a service for DataHub Analytics. + version: v1 + +paths: + /datahub_usage_events/_search: + post: + summary: Raw datahub_usage_event data. (Experimental) + operationId: "raw" + tags: + - "DataHub Usage" + requestBody: + content: + application/json: + schema: # Request body contents + type: string + example: >- + { + "query": { + "match_all": {} + } + } + responses: + '200': + description: "Success" + content: + application/json: + schema: + type: string diff --git a/metadata-service/openapi-analytics-servlet/src/test/java/io/datahubproject/openapi/config/OpenAPIAnalyticsTestConfiguration.java b/metadata-service/openapi-analytics-servlet/src/test/java/io/datahubproject/openapi/config/OpenAPIAnalyticsTestConfiguration.java new file mode 100644 index 0000000000000..98f0db8fd10ef --- /dev/null +++ b/metadata-service/openapi-analytics-servlet/src/test/java/io/datahubproject/openapi/config/OpenAPIAnalyticsTestConfiguration.java @@ -0,0 +1,48 @@ +package io.datahubproject.openapi.config; + +import com.datahub.authentication.Actor; +import com.datahub.authentication.ActorType; +import com.datahub.authentication.Authentication; +import com.datahub.authentication.AuthenticationContext; +import com.datahub.authorization.AuthorizationResult; +import com.datahub.authorization.AuthorizerChain; +import com.linkedin.metadata.search.elasticsearch.ElasticSearchService; +import org.elasticsearch.action.search.SearchResponse; +import org.mockito.Mockito; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Primary; + +import java.io.IOException; +import java.util.Optional; + +import static io.datahubproject.openapi.delegates.DatahubUsageEventsImpl.DATAHUB_USAGE_INDEX; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + + +@TestConfiguration +public class OpenAPIAnalyticsTestConfiguration { + @Bean + @Primary + public ElasticSearchService datahubUsageEventsApiDelegate() throws IOException { + ElasticSearchService elasticSearchService = mock(ElasticSearchService.class); + SearchResponse mockResp = mock(SearchResponse.class); + when(elasticSearchService.raw(eq(DATAHUB_USAGE_INDEX), anyString())) + .thenReturn(Optional.of(mockResp)); + return elasticSearchService; + } + + @Bean + public AuthorizerChain authorizerChain() { + AuthorizerChain authorizerChain = Mockito.mock(AuthorizerChain.class); + + Authentication authentication = Mockito.mock(Authentication.class); + when(authentication.getActor()).thenReturn(new Actor(ActorType.USER, "datahub")); + when(authorizerChain.authorize(any())).thenReturn(new AuthorizationResult(null, AuthorizationResult.Type.ALLOW, "")); + AuthenticationContext.setAuthentication(authentication); + + return authorizerChain; + } +} diff --git a/metadata-service/openapi-analytics-servlet/src/test/java/io/datahubproject/openapi/delegates/DatahubUsageEventsImplTest.java b/metadata-service/openapi-analytics-servlet/src/test/java/io/datahubproject/openapi/delegates/DatahubUsageEventsImplTest.java new file mode 100644 index 0000000000000..af2a24391fea8 --- /dev/null +++ b/metadata-service/openapi-analytics-servlet/src/test/java/io/datahubproject/openapi/delegates/DatahubUsageEventsImplTest.java @@ -0,0 +1,44 @@ +package io.datahubproject.openapi.delegates; + +import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor; +import io.datahubproject.openapi.config.OpenAPIAnalyticsTestConfiguration; +import io.datahubproject.openapi.config.SpringWebConfig; +import io.datahubproject.openapi.generated.controller.DatahubUsageEventsApiController; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Import; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; + + +@SpringBootTest(classes = {SpringWebConfig.class}) +@ComponentScan(basePackages = {"io.datahubproject.openapi.generated.controller"}) +@Import({DatahubUsageEventsImpl.class, OpenAPIAnalyticsTestConfiguration.class}) +public class DatahubUsageEventsImplTest extends AbstractTestNGSpringContextTests { + @BeforeTest + public void disableAssert() { + PathSpecBasedSchemaAnnotationVisitor.class.getClassLoader() + .setClassAssertionStatus(PathSpecBasedSchemaAnnotationVisitor.class.getName(), false); + } + + @Autowired + private DatahubUsageEventsApiController analyticsController; + + @Test + public void initTest() { + assertNotNull(analyticsController); + } + + @Test + public void analyticsControllerTest() { + ResponseEntity resp = analyticsController.raw(""); + assertEquals(resp.getStatusCode(), HttpStatus.OK); + } +} diff --git a/metadata-service/openapi-entity-servlet/build.gradle b/metadata-service/openapi-entity-servlet/build.gradle new file mode 100644 index 0000000000000..7f9c472b91fac --- /dev/null +++ b/metadata-service/openapi-entity-servlet/build.gradle @@ -0,0 +1,82 @@ +plugins { + id 'java' + id 'org.hidetake.swagger.generator' +} + +dependencies { + + implementation project(':metadata-auth:auth-api') + implementation project(':metadata-service:auth-impl') + implementation project(':metadata-service:factories') + implementation project(':metadata-service:openapi-servlet') + implementation project(':metadata-models') + + implementation externalDependency.springBoot + implementation externalDependency.springCore + implementation externalDependency.springDocUI + implementation externalDependency.springWeb + implementation externalDependency.springWebMVC + implementation externalDependency.springBeans + implementation externalDependency.springContext + + implementation externalDependency.reflections + implementation externalDependency.slf4jApi + compileOnly externalDependency.lombok + + implementation externalDependency.antlr4Runtime + implementation externalDependency.antlr4 + + annotationProcessor externalDependency.lombok + + testImplementation externalDependency.testng + testImplementation externalDependency.mockito + testImplementation externalDependency.springBootTest + + swaggerCodegen externalDependency.swaggerCli + swaggerCodegen project(':metadata-service:openapi-entity-servlet:generators') +} + +sourceSets { + main { + java { + srcDirs = ["$buildDir/openapi/generated/src/main/java", 'src/main/java'] + } + } +} + + +task mergeApiComponents(dependsOn: ':metadata-models:generateJsonSchema') { + doLast { + mkdir("$buildDir/openapi") + File combined = file("$buildDir/openapi/open-api.yaml") + def components = file("${project(':metadata-models').projectDir}/src/generatedJsonSchema/combined/open-api-components.yaml").getText('UTF-8') + def api = file("$projectDir/src/main/resources/entity-v2.0.yml").getText('UTF-8') + def paths = file("${project(':metadata-models').projectDir}/src/generatedJsonSchema/combined/open-api-paths.yaml").getText('UTF-8') + combined.text = (components + paths + api).replaceAll("---\n", "\n") + } + outputs.file(file("$buildDir/openapi/open-api.yaml")) +} + +// https://github.com/int128/gradle-swagger-generator-plugin#task-type-generateswaggercode +task openApiGenerate(type: GenerateSwaggerCode, dependsOn: [mergeApiComponents, ':metadata-service:openapi-entity-servlet:generators:jar']) { + inputFile = file("$buildDir/openapi/open-api.yaml") + outputDir = file("$buildDir/openapi/generated") + + // custom generator class + language = 'io.datahubproject.CustomSpringCodegen' + + components = ["apis"] + templateDir = file("$projectDir/src/main/resources/JavaSpring") + additionalProperties = [ + 'group-id' : "io.datahubproject", + 'dateLibrary' : "java8", + 'java11' : "true", + 'modelPropertyNaming': "original", + 'modelPackage' : "io.datahubproject.openapi.generated", + 'apiPackage' : "io.datahubproject.openapi.generated.controller", + 'delegatePattern' : "false" + ] +} +tasks.getByName("compileJava").dependsOn(openApiGenerate) + +checkstyleMain.exclude '**/generated/**' \ No newline at end of file diff --git a/metadata-service/openapi-entity-servlet/generators/build.gradle b/metadata-service/openapi-entity-servlet/generators/build.gradle new file mode 100644 index 0000000000000..cb54ae0d9fc05 --- /dev/null +++ b/metadata-service/openapi-entity-servlet/generators/build.gradle @@ -0,0 +1,10 @@ +plugins { + id 'java' +} + +dependencies { + implementation externalDependency.swaggerCli + + compileOnly externalDependency.lombok + annotationProcessor externalDependency.lombok +} \ No newline at end of file diff --git a/metadata-service/openapi-entity-servlet/generators/src/main/java/io/datahubproject/CustomSpringCodegen.java b/metadata-service/openapi-entity-servlet/generators/src/main/java/io/datahubproject/CustomSpringCodegen.java new file mode 100644 index 0000000000000..ef36d8aa38785 --- /dev/null +++ b/metadata-service/openapi-entity-servlet/generators/src/main/java/io/datahubproject/CustomSpringCodegen.java @@ -0,0 +1,43 @@ +package io.datahubproject; + +import io.swagger.codegen.v3.generators.java.SpringCodegen; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; +import java.util.Map; + + +@Slf4j +public class CustomSpringCodegen extends SpringCodegen { + + public CustomSpringCodegen() { + super(); + } + + @Override + public String getName() { + return "custom-spring"; + } + + @Override + public Map postProcessOperations(Map objs) { + Map result = super.postProcessOperations(objs); + List> imports = (List) objs.get("imports"); + + for (Map importMap : imports) { + for (String type : importMap.values()) { + if (type.contains("EntityRequest") && !type.contains(".Scroll")) { + additionalProperties.put("requestClass", type); + } + if (type.contains("EntityResponse") && !type.contains(".Scroll")) { + additionalProperties.put("responseClass", type); + } + if (type.contains("EntityResponse") && type.contains(".Scroll")) { + additionalProperties.put("scrollResponseClass", type); + } + } + } + + return result; + } +} diff --git a/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/delegates/EntityApiDelegateImpl.java b/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/delegates/EntityApiDelegateImpl.java new file mode 100644 index 0000000000000..5d1065e80d419 --- /dev/null +++ b/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/delegates/EntityApiDelegateImpl.java @@ -0,0 +1,411 @@ +package io.datahubproject.openapi.delegates; + +import com.linkedin.common.urn.Urn; +import com.linkedin.metadata.entity.EntityService; +import com.datahub.authentication.Authentication; +import com.datahub.authentication.AuthenticationContext; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.query.SearchFlags; +import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.search.ScrollResult; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchService; +import io.datahubproject.openapi.dto.UpsertAspectRequest; +import io.datahubproject.openapi.dto.UrnResponseMap; +import io.datahubproject.openapi.entities.EntitiesController; +import com.datahub.authorization.AuthorizerChain; +import io.datahubproject.openapi.generated.BrowsePathsV2AspectRequestV2; +import io.datahubproject.openapi.generated.BrowsePathsV2AspectResponseV2; +import io.datahubproject.openapi.generated.DeprecationAspectRequestV2; +import io.datahubproject.openapi.generated.DeprecationAspectResponseV2; +import io.datahubproject.openapi.generated.DomainsAspectRequestV2; +import io.datahubproject.openapi.generated.DomainsAspectResponseV2; +import io.datahubproject.openapi.generated.GlobalTagsAspectRequestV2; +import io.datahubproject.openapi.generated.GlobalTagsAspectResponseV2; +import io.datahubproject.openapi.generated.GlossaryTermsAspectRequestV2; +import io.datahubproject.openapi.generated.GlossaryTermsAspectResponseV2; +import io.datahubproject.openapi.generated.OwnershipAspectRequestV2; +import io.datahubproject.openapi.generated.OwnershipAspectResponseV2; +import io.datahubproject.openapi.generated.SortOrder; +import io.datahubproject.openapi.generated.StatusAspectRequestV2; +import io.datahubproject.openapi.generated.StatusAspectResponseV2; +import io.datahubproject.openapi.exception.UnauthorizedException; +import io.datahubproject.openapi.util.OpenApiEntitiesUtil; +import com.datahub.authorization.ConjunctivePrivilegeGroup; +import com.datahub.authorization.DisjunctivePrivilegeGroup; +import com.linkedin.metadata.models.EntitySpec; +import com.datahub.authorization.ResourceSpec; +import com.linkedin.metadata.authorization.PoliciesConfig; +import com.google.common.collect.ImmutableList; +import com.datahub.authorization.AuthUtil; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; + +import javax.validation.Valid; +import javax.validation.constraints.Min; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static io.datahubproject.openapi.util.ReflectionCache.toLowerFirst; + +public class EntityApiDelegateImpl { + final private EntityRegistry _entityRegistry; + final private EntityService _entityService; + final private SearchService _searchService; + final private EntitiesController _v1Controller; + final private AuthorizerChain _authorizationChain; + + final private boolean _restApiAuthorizationEnabled; + final private Class _reqClazz; + final private Class _respClazz; + final private Class _scrollRespClazz; + + final private StackWalker walker = StackWalker.getInstance(); + + public EntityApiDelegateImpl(EntityService entityService, SearchService searchService, EntitiesController entitiesController, + boolean restApiAuthorizationEnabled, AuthorizerChain authorizationChain, + Class reqClazz, Class respClazz, Class scrollRespClazz) { + this._entityService = entityService; + this._searchService = searchService; + this._entityRegistry = entityService.getEntityRegistry(); + this._v1Controller = entitiesController; + this._authorizationChain = authorizationChain; + this._restApiAuthorizationEnabled = restApiAuthorizationEnabled; + this._reqClazz = reqClazz; + this._respClazz = respClazz; + this._scrollRespClazz = scrollRespClazz; + } + + public ResponseEntity get(String urn, Boolean systemMetadata, List aspects) { + String[] requestedAspects = Optional.ofNullable(aspects).map(asp -> asp.stream().distinct().toArray(String[]::new)).orElse(null); + ResponseEntity result = _v1Controller.getEntities(new String[]{urn}, requestedAspects); + return ResponseEntity.of(OpenApiEntitiesUtil.convertEntity(Optional.ofNullable(result) + .map(HttpEntity::getBody).orElse(null), _respClazz, systemMetadata)); + } + + public ResponseEntity> create(List body) { + List aspects = body.stream() + .flatMap(b -> OpenApiEntitiesUtil.convertEntityToUpsert(b, _reqClazz, _entityRegistry).stream()) + .collect(Collectors.toList()); + _v1Controller.postEntities(aspects); + List responses = body.stream() + .map(req -> OpenApiEntitiesUtil.convertToResponse(req, _respClazz, _entityRegistry)) + .collect(Collectors.toList()); + return ResponseEntity.ok(responses); + } + + public ResponseEntity delete(String urn) { + _v1Controller.deleteEntities(new String[]{urn}, false); + return new ResponseEntity<>(HttpStatus.OK); + } + + public ResponseEntity head(String urn) { + try { + Urn entityUrn = Urn.createFromString(urn); + if (_entityService.exists(entityUrn)) { + return new ResponseEntity<>(HttpStatus.NO_CONTENT); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + public ResponseEntity getAspect(String urn, Boolean systemMetadata, String aspect, Class entityRespClass, + Class aspectRespClazz) { + String[] requestedAspects = new String[]{aspect}; + ResponseEntity result = _v1Controller.getEntities(new String[]{urn}, requestedAspects); + return ResponseEntity.of(OpenApiEntitiesUtil.convertAspect(result.getBody(), aspect, entityRespClass, aspectRespClazz, + systemMetadata)); + } + + public ResponseEntity createAspect(String urn, String aspectName, AQ body, Class reqClazz, Class respClazz) { + UpsertAspectRequest aspectUpsert = OpenApiEntitiesUtil.convertAspectToUpsert(urn, body, reqClazz); + _v1Controller.postEntities(Stream.of(aspectUpsert).filter(Objects::nonNull).collect(Collectors.toList())); + AR response = OpenApiEntitiesUtil.convertToResponseAspect(body, respClazz); + return ResponseEntity.ok(response); + } + + public ResponseEntity headAspect(String urn, String aspect) { + try { + Urn entityUrn = Urn.createFromString(urn); + if (_entityService.exists(entityUrn, aspect)) { + return new ResponseEntity<>(HttpStatus.NO_CONTENT); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + public ResponseEntity deleteAspect(String urn, String aspect) { + _entityService.deleteAspect(urn, aspect, Map.of(), false); + _v1Controller.deleteEntities(new String[]{urn}, false); + return new ResponseEntity<>(HttpStatus.OK); + } + + public ResponseEntity createDomains(DomainsAspectRequestV2 body, String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return createAspect(urn, methodNameToAspectName(methodName), body, DomainsAspectRequestV2.class, DomainsAspectResponseV2.class); + } + + public ResponseEntity createGlobalTags(GlobalTagsAspectRequestV2 body, String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return createAspect(urn, methodNameToAspectName(methodName), body, GlobalTagsAspectRequestV2.class, GlobalTagsAspectResponseV2.class); + } + + public ResponseEntity createGlossaryTerms(GlossaryTermsAspectRequestV2 body, String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return createAspect(urn, methodNameToAspectName(methodName), body, GlossaryTermsAspectRequestV2.class, GlossaryTermsAspectResponseV2.class); + } + + public ResponseEntity createOwnership(OwnershipAspectRequestV2 body, String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return createAspect(urn, methodNameToAspectName(methodName), body, OwnershipAspectRequestV2.class, OwnershipAspectResponseV2.class); + } + + public ResponseEntity createStatus(StatusAspectRequestV2 body, String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return createAspect(urn, methodNameToAspectName(methodName), body, StatusAspectRequestV2.class, StatusAspectResponseV2.class); + } + + public ResponseEntity deleteDomains(String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return deleteAspect(urn, methodNameToAspectName(methodName)); + } + + public ResponseEntity deleteGlobalTags(String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return deleteAspect(urn, methodNameToAspectName(methodName)); + } + + public ResponseEntity deleteGlossaryTerms(String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return deleteAspect(urn, methodNameToAspectName(methodName)); + } + + public ResponseEntity deleteOwnership(String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return deleteAspect(urn, methodNameToAspectName(methodName)); + } + + public ResponseEntity deleteStatus(String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return deleteAspect(urn, methodNameToAspectName(methodName)); + } + + public ResponseEntity getDomains(String urn, Boolean systemMetadata) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return getAspect(urn, systemMetadata, methodNameToAspectName(methodName), _respClazz, + DomainsAspectResponseV2.class); + } + + public ResponseEntity getGlobalTags(String urn, Boolean systemMetadata) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return getAspect(urn, systemMetadata, methodNameToAspectName(methodName), _respClazz, + GlobalTagsAspectResponseV2.class); + } + + public ResponseEntity getGlossaryTerms(String urn, Boolean systemMetadata) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return getAspect(urn, systemMetadata, methodNameToAspectName(methodName), _respClazz, + GlossaryTermsAspectResponseV2.class); + } + + public ResponseEntity getOwnership(String urn, Boolean systemMetadata) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return getAspect(urn, systemMetadata, methodNameToAspectName(methodName), _respClazz, + OwnershipAspectResponseV2.class); + } + + public ResponseEntity getStatus(String urn, Boolean systemMetadata) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return getAspect(urn, systemMetadata, methodNameToAspectName(methodName), _respClazz, + StatusAspectResponseV2.class); + } + + public ResponseEntity headDomains(String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return headAspect(urn, methodNameToAspectName(methodName)); + } + + public ResponseEntity headGlobalTags(String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return headAspect(urn, methodNameToAspectName(methodName)); + } + + public ResponseEntity headGlossaryTerms(String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return headAspect(urn, methodNameToAspectName(methodName)); + } + + public ResponseEntity headOwnership(String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return headAspect(urn, methodNameToAspectName(methodName)); + } + + public ResponseEntity headStatus(String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return headAspect(urn, methodNameToAspectName(methodName)); + } + + protected static String methodNameToAspectName(String methodName) { + return toLowerFirst(methodName.replaceFirst("^(get|head|delete|create)", "")); + } + + public ResponseEntity deleteDeprecation(String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return deleteAspect(urn, methodNameToAspectName(methodName)); + } + + public ResponseEntity deleteBrowsePathsV2(String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return deleteAspect(urn, methodNameToAspectName(methodName)); + } + + public ResponseEntity getDeprecation(String urn, @Valid Boolean systemMetadata) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return getAspect(urn, systemMetadata, methodNameToAspectName(methodName), _respClazz, + DeprecationAspectResponseV2.class); + } + + public ResponseEntity headDeprecation(String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return headAspect(urn, methodNameToAspectName(methodName)); + } + + public ResponseEntity createDeprecation(@Valid DeprecationAspectRequestV2 body, String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return createAspect(urn, methodNameToAspectName(methodName), body, DeprecationAspectRequestV2.class, + DeprecationAspectResponseV2.class); + } + + public ResponseEntity headBrowsePathsV2(String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return headAspect(urn, methodNameToAspectName(methodName)); + } + + public ResponseEntity getBrowsePathsV2(String urn, @Valid Boolean systemMetadata) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return getAspect(urn, systemMetadata, methodNameToAspectName(methodName), _respClazz, + BrowsePathsV2AspectResponseV2.class); + } + + public ResponseEntity createBrowsePathsV2(@Valid BrowsePathsV2AspectRequestV2 body, String urn) { + String methodName = walker.walk(frames -> frames + .findFirst() + .map(StackWalker.StackFrame::getMethodName)).get(); + return createAspect(urn, methodNameToAspectName(methodName), body, BrowsePathsV2AspectRequestV2.class, + BrowsePathsV2AspectResponseV2.class); + } + + public ResponseEntity scroll(@Valid Boolean systemMetadata, @Valid List aspects, @Min(1) @Valid Integer count, + @Valid String scrollId, @Valid List sort, @Valid SortOrder sortOrder, @Valid String query) { + + Authentication authentication = AuthenticationContext.getAuthentication(); + EntitySpec entitySpec = OpenApiEntitiesUtil.responseClassToEntitySpec(_entityRegistry, _respClazz); + checkScrollAuthorized(authentication, entitySpec); + + // TODO multi-field sort + SortCriterion sortCriterion = new SortCriterion(); + sortCriterion.setField(Optional.ofNullable(sort).map(s -> s.get(0)).orElse("urn")); + sortCriterion.setOrder(com.linkedin.metadata.query.filter.SortOrder.valueOf(Optional.ofNullable(sortOrder) + .map(Enum::name).orElse("ASCENDING"))); + + SearchFlags searchFlags = new SearchFlags() + .setFulltext(false) + .setSkipAggregates(true) + .setSkipHighlighting(true); + + ScrollResult result = _searchService.scrollAcrossEntities( + List.of(entitySpec.getName()), + query, null, sortCriterion, scrollId, null, count, searchFlags); + + String[] urns = result.getEntities().stream() + .map(SearchEntity::getEntity) + .map(Urn::toString) + .toArray(String[]::new); + String[] requestedAspects = Optional.ofNullable(aspects) + .map(asp -> asp.stream().distinct().toArray(String[]::new)) + .orElse(null); + List entities = Optional.ofNullable(_v1Controller.getEntities(urns, requestedAspects).getBody()) + .map(body -> body.getResponses().entrySet()) + .map(entries -> OpenApiEntitiesUtil.convertEntities(entries, _respClazz, systemMetadata)) + .orElse(List.of()); + + return ResponseEntity.of(OpenApiEntitiesUtil.convertToScrollResponse(_scrollRespClazz, result.getScrollId(), entities)); + } + + private void checkScrollAuthorized(Authentication authentication, EntitySpec entitySpec) { + String actorUrnStr = authentication.getActor().toUrnStr(); + DisjunctivePrivilegeGroup orGroup = new DisjunctivePrivilegeGroup(ImmutableList.of(new ConjunctivePrivilegeGroup( + ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE.getType())))); + + List> resourceSpecs = List.of(Optional.of(new ResourceSpec(entitySpec.getName(), ""))); + if (_restApiAuthorizationEnabled && !AuthUtil.isAuthorizedForResources(_authorizationChain, actorUrnStr, resourceSpecs, orGroup)) { + throw new UnauthorizedException(actorUrnStr + " is unauthorized to get entities."); + } + } +} diff --git a/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/util/OpenApiEntitiesUtil.java b/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/util/OpenApiEntitiesUtil.java new file mode 100644 index 0000000000000..13c2d83343aa0 --- /dev/null +++ b/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/util/OpenApiEntitiesUtil.java @@ -0,0 +1,279 @@ +package io.datahubproject.openapi.util; + +import com.linkedin.common.urn.Urn; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.util.Pair; +import io.datahubproject.openapi.dto.UpsertAspectRequest; +import io.datahubproject.openapi.dto.UrnResponseMap; +import io.datahubproject.openapi.generated.EntityResponse; +import io.datahubproject.openapi.generated.OneOfGenericAspectValue; +import io.datahubproject.openapi.generated.SystemMetadata; +import lombok.extern.slf4j.Slf4j; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static io.datahubproject.openapi.util.ReflectionCache.toLowerFirst; +import static io.datahubproject.openapi.util.ReflectionCache.toUpperFirst; + + +@Slf4j +public class OpenApiEntitiesUtil { + private final static String MODEL_VERSION = "V2"; + private final static String REQUEST_SUFFIX = "Request" + MODEL_VERSION; + private final static String RESPONSE_SUFFIX = "Response" + MODEL_VERSION; + + private final static String ASPECT_REQUEST_SUFFIX = "Aspect" + REQUEST_SUFFIX; + private final static String ASPECT_RESPONSE_SUFFIX = "Aspect" + RESPONSE_SUFFIX; + private final static String ENTITY_REQUEST_SUFFIX = "Entity" + REQUEST_SUFFIX; + private final static String ENTITY_RESPONSE_SUFFIX = "Entity" + RESPONSE_SUFFIX; + + private OpenApiEntitiesUtil() { + } + + private final static ReflectionCache REFLECT = ReflectionCache.builder() + .basePackage("io.datahubproject.openapi.generated") + .build(); + + + public static UpsertAspectRequest convertAspectToUpsert(String entityUrn, Object aspectRequest, Class aspectRequestClazz) { + try { + UpsertAspectRequest.UpsertAspectRequestBuilder builder = UpsertAspectRequest.builder(); + builder.entityType(Urn.createFromString(entityUrn).getEntityType()); + builder.entityUrn(entityUrn); + + // i.e. GlobalTagsAspectRequestV2 + if (aspectRequest != null) { + // i.e. GlobalTags + Method valueMethod = REFLECT.lookupMethod(aspectRequestClazz, "getValue"); + Object aspect = valueMethod.invoke(aspectRequest); + + if (aspect != null) { + builder.aspect((OneOfGenericAspectValue) aspect); + return builder.build(); + } + } + + return null; + } catch (Exception e) { + log.error("Error reflecting urn: {} aspect: {}", entityUrn, aspectRequestClazz.getName()); + throw new RuntimeException(e); + } + } + public static List convertEntityToUpsert(Object openapiEntity, Class fromClazz, EntityRegistry entityRegistry) { + final EntitySpec entitySpec = requestClassToEntitySpec(entityRegistry, fromClazz); + + return entitySpec.getAspectSpecs().stream() + .map(aspectSpec -> { + try { + UpsertAspectRequest.UpsertAspectRequestBuilder builder = UpsertAspectRequest.builder(); + builder.entityType(entitySpec.getName()); + builder.entityUrn((String) REFLECT.lookupMethod(fromClazz, "getUrn").invoke(openapiEntity)); + + String upperAspectName = toUpperFirst(aspectSpec.getName()); + Method aspectMethod = REFLECT.lookupMethod(fromClazz, "get" + upperAspectName); + + // i.e. GlobalTagsAspectRequestV2 + Object aspectRequest = aspectMethod.invoke(openapiEntity); + if (aspectRequest != null) { + Class aspectRequestClazz = REFLECT.lookupClass(upperAspectName + ASPECT_REQUEST_SUFFIX); + + // i.e. GlobalTags + Method valueMethod = REFLECT.lookupMethod(aspectRequestClazz, "getValue"); + Object aspect = valueMethod.invoke(aspectRequest); + + if (aspect != null) { + builder.aspect((OneOfGenericAspectValue) aspect); + return builder.build(); + } + } + + return null; + } catch (Exception e) { + log.error("Error reflecting entity: {} aspect: {}", entitySpec.getName(), aspectSpec.getName()); + throw new RuntimeException(e); + } + }).filter(Objects::nonNull).collect(Collectors.toList()); + } + + public static Optional convertAspect(UrnResponseMap urnResponseMap, String aspectName, Class entityClazz, + Class aspectClazz, boolean withSystemMetadata) { + return convertEntity(urnResponseMap, entityClazz, withSystemMetadata).map(entity -> { + try { + Method aspectMethod = REFLECT.lookupMethod(entityClazz, "get" + toUpperFirst(aspectName)); + return aspectClazz.cast(aspectMethod.invoke(entity)); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + }); + + } + + public static Optional convertEntity(UrnResponseMap urnResponseMap, Class toClazz, boolean withSystemMetadata) { + return Optional.ofNullable(urnResponseMap) + .flatMap(respMap -> respMap.getResponses().entrySet().stream().findFirst()) + .flatMap(entry -> convertEntities(Set.of(entry), toClazz, withSystemMetadata).stream().findFirst()); + } + + public static List convertEntities(Set> entityResponseSet, Class toClazz, boolean withSystemMetadata) { + if (entityResponseSet != null) { + return entityResponseSet.stream().map(entry -> { + try { + // i.e. DataContractEntityResponseV2.Builder + Pair, Object> builderPair = REFLECT.getBuilder(toClazz); + Set builderMethods = Arrays.stream(builderPair.getFirst().getMethods()) + .map(Method::getName).collect(Collectors.toSet()); + + REFLECT.lookupMethod(builderPair, "urn", String.class).invoke(builderPair.getSecond(), entry.getKey()); + + entry.getValue().getAspects().entrySet().forEach(aspectEntry -> { + try { + if (builderMethods.contains(aspectEntry.getKey())) { + String upperFirstAspect = toUpperFirst(aspectEntry.getKey()); + Class aspectClazz = REFLECT.lookupClass(upperFirstAspect); + Class aspectRespClazz = REFLECT.lookupClass(upperFirstAspect + ASPECT_RESPONSE_SUFFIX); + Class aspectRespClazzBuilder = REFLECT.lookupClass(String.join("", + upperFirstAspect, ASPECT_RESPONSE_SUFFIX, + "$", upperFirstAspect, ASPECT_RESPONSE_SUFFIX, "Builder")); + Object aspectBuilder = REFLECT.lookupMethod(aspectRespClazz, "builder").invoke(null); + + REFLECT.lookupMethod(aspectRespClazzBuilder, "value", aspectClazz).invoke(aspectBuilder, aspectEntry.getValue().getValue()); + + if (withSystemMetadata) { + REFLECT.lookupMethod(aspectRespClazzBuilder, "systemMetadata", SystemMetadata.class) + .invoke(aspectBuilder, aspectEntry.getValue().getSystemMetadata()); + } + + REFLECT.lookupMethod(builderPair, aspectEntry.getKey(), aspectRespClazz).invoke(builderPair.getSecond(), + REFLECT.lookupMethod(aspectRespClazzBuilder, "build").invoke(aspectBuilder)); + } + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + }); + + return toClazz.cast(REFLECT.lookupMethod(builderPair, "build").invoke(builderPair.getSecond())); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + }).collect(Collectors.toList()); + } + return List.of(); + } + + public static T convertToResponseAspect(I source, Class targetClazz) { + if (source != null) { + try { + Class sourceClazz = REFLECT.lookupClass(source.getClass().getSimpleName()); + Method valueMethod = REFLECT.lookupMethod(sourceClazz, "getValue"); + Object aspect = valueMethod.invoke(source); + + Pair, Object> builderPair = REFLECT.getBuilder(targetClazz); + REFLECT.lookupMethod(builderPair, "value", valueMethod.getReturnType()).invoke(builderPair.getSecond(), aspect); + + return targetClazz.cast(REFLECT.lookupMethod(builderPair, "build").invoke(builderPair.getSecond())); + } catch (InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + return null; + } + + public static T convertToResponse(I source, Class targetClazz, EntityRegistry entityRegistry) { + if (source != null) { + try { + Class sourceClazz = REFLECT.lookupClass(source.getClass().getSimpleName()); + Pair, Object> builderPair = REFLECT.getBuilder(targetClazz); + copy(Pair.of(sourceClazz, source), builderPair, "urn"); + + final EntitySpec entitySpec = requestClassToEntitySpec(entityRegistry, sourceClazz); + entitySpec.getAspectSpecs().stream() + .forEach(aspectSpec -> { + try { + copy(Pair.of(sourceClazz, source), builderPair, aspectSpec.getName()); + } catch (InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException(e); + } + }); + + return targetClazz.cast(REFLECT.lookupMethod(builderPair, "build").invoke(builderPair.getSecond())); + } catch (InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + return null; + } + + public static Optional convertToScrollResponse(Class scrollRespClazz, String scrollId, List entityResults) { + if (entityResults != null) { + try { + Pair, Object> builderPair = REFLECT.getBuilder(scrollRespClazz); + REFLECT.lookupMethod(builderPair.getFirst(), "scrollId", String.class).invoke(builderPair.getSecond(), scrollId); + REFLECT.lookupMethod(builderPair.getFirst(), "entities", List.class).invoke(builderPair.getSecond(), entityResults); + + return Optional.of(scrollRespClazz.cast(REFLECT.lookupMethod(builderPair, "build").invoke(builderPair.getSecond()))); + + } catch (InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + return Optional.empty(); + } + + + + private static void copy(Pair, Object> sourcePair, Pair, Object> builderPair, String method) + throws InvocationTargetException, IllegalAccessException { + Method sourceMethod = REFLECT.lookupMethod(sourcePair, String.format("get%s", toUpperFirst(method))); + if (sourceMethod != null) { + Class paramClazz = null; + Object param = null; + if (sourceMethod.getReturnType().getSimpleName().contains("Request")) { + Object sourceParam = sourceMethod.invoke(sourcePair.getSecond()); + if (sourceParam != null) { + paramClazz = REFLECT.lookupClass(sourceMethod.getReturnType().getSimpleName().replace("Request", "Response")); + Pair, Object> aspectBuilder = REFLECT.getBuilder(paramClazz); + + for (Method m : sourceMethod.getReturnType().getMethods()) { + if (m.getName().startsWith("get") && !Objects.equals("getClass", m.getName())) { + String getterMethod = m.getName().replaceFirst("^get", ""); + copy(Pair.of(sourceMethod.getReturnType(), sourceMethod.invoke(sourcePair.getSecond())), + aspectBuilder, getterMethod); + } + } + + param = REFLECT.lookupMethod(aspectBuilder, "build").invoke(aspectBuilder.getSecond()); + } + } else { + paramClazz = sourceMethod.getReturnType(); + param = sourceMethod.invoke(sourcePair.getSecond()); + } + + if (param != null) { + Method targetMethod = REFLECT.lookupMethod(builderPair, toLowerFirst(method), paramClazz); + targetMethod.invoke(builderPair.getSecond(), param); + } + } else { + log.info("Class {} doesn't container method {}", sourcePair.getFirst(), + String.format("get%s", toUpperFirst(method))); + } + } + + public static EntitySpec requestClassToEntitySpec(EntityRegistry entityRegistry, Class reqClazz) { + final String entityType = toLowerFirst(reqClazz.getSimpleName().replace(ENTITY_REQUEST_SUFFIX, "")); + return entityRegistry.getEntitySpec(entityType); + } + + public static EntitySpec responseClassToEntitySpec(EntityRegistry entityRegistry, Class respClazz) { + String entityType = toLowerFirst(respClazz.getSimpleName().replace(ENTITY_RESPONSE_SUFFIX, "")); + return entityRegistry.getEntitySpec(entityType); + } +} diff --git a/metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/api.mustache b/metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/api.mustache new file mode 100644 index 0000000000000..cbc5d9e1996ea --- /dev/null +++ b/metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/api.mustache @@ -0,0 +1,172 @@ +/** + * NOTE: This class is auto generated by the swagger code generator program ({{{generatorVersion}}}). + * https://github.com/swagger-api/swagger-codegen + * Do not edit the class manually. + */ +package {{package}}; + +{{#imports}}import {{import}}; +{{/imports}} +{{#jdk8-no-delegate}} +import com.fasterxml.jackson.databind.ObjectMapper; +{{/jdk8-no-delegate}} +{{#useOas2}} +import io.swagger.annotations.*; +{{/useOas2}} +{{^useOas2}} +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +{{/useOas2}} +{{#jdk8-no-delegate}} +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +{{/jdk8-no-delegate}} +import org.springframework.http.ResponseEntity; +{{#useBeanValidation}} +import org.springframework.validation.annotation.Validated; +{{/useBeanValidation}} +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RequestPart; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.bind.annotation.CookieValue; + +{{#jdk8-no-delegate}} +import javax.servlet.http.HttpServletRequest; +{{/jdk8-no-delegate}} +{{#useBeanValidation}} +import javax.validation.Valid; +import javax.validation.constraints.*; +{{/useBeanValidation}} +{{#jdk8-no-delegate}} +import java.io.IOException; +{{/jdk8-no-delegate}} +import java.util.List; +import java.util.Map; +{{#jdk8-no-delegate}} +import java.util.Optional; +{{/jdk8-no-delegate}} +{{^jdk8-no-delegate}} + {{#useOptional}} +import java.util.Optional; + {{/useOptional}} +{{/jdk8-no-delegate}} +{{#async}} +import java.util.concurrent.{{^isJava8or11}}Callable{{/isJava8or11}}{{#isJava8or11}}CompletableFuture{{/isJava8or11}}; +{{/async}} + +{{>generatedAnnotation}} +{{#useBeanValidation}} +@Validated +{{/useBeanValidation}} +{{#useOas2}} +@Api(value = "{{{baseName}}}", description = "the {{{baseName}}} API") +{{/useOas2}} +{{#operations}} +public interface {{classname}} { +{{#isJava8or11}} + + {{^isDelegate}} + Logger log = LoggerFactory.getLogger({{classname}}.class); + + {{#defaultInterfaces}}default {{/defaultInterfaces}}Optional getObjectMapper(){{^defaultInterfaces}};{{/defaultInterfaces}}{{#defaultInterfaces}}{ + return Optional.empty(); + }{{/defaultInterfaces}} + + {{#defaultInterfaces}}default {{/defaultInterfaces}}Optional getRequest(){{^defaultInterfaces}};{{/defaultInterfaces}}{{#defaultInterfaces}}{ + return Optional.empty(); + }{{/defaultInterfaces}} + + {{#defaultInterfaces}}default Optional getAcceptHeader() { + return getRequest().map(r -> r.getHeader("Accept")); + }{{/defaultInterfaces}} + {{/isDelegate}} + {{#isDelegate}} + {{classname}}Delegate getDelegate(); + {{/isDelegate}} +{{/isJava8or11}} +{{#operation}} +{{#contents}} +{{#@first}} + + {{#useOas2}} + @ApiOperation(value = "{{{summary}}}", nickname = "{{{operationId}}}", notes = "{{{notes}}}"{{#returnBaseType}}, response = {{{returnBaseType}}}.class{{/returnBaseType}}{{#returnContainer}}, responseContainer = "{{{returnContainer}}}"{{/returnContainer}}{{#hasAuthMethods}}, authorizations = { + {{#authMethods}}@Authorization(value = "{{name}}"{{#isOAuth}}, scopes = { {{#each scopes}} + @AuthorizationScope(scope = "{{@key}}", description = "{{this}}"){{^@last}},{{/@last}}{{/each}} + }{{/isOAuth}}){{#hasMore}}, + {{/hasMore}}{{/authMethods}} + }{{/hasAuthMethods}}, tags={ {{#vendorExtensions.x-tags}}"{{tag}}",{{/vendorExtensions.x-tags}} }) + @ApiResponses(value = { {{#responses}} + @ApiResponse(code = {{{code}}}, message = "{{{message}}}"{{#baseType}}, response = {{{baseType}}}.class{{/baseType}}{{#containerType}}, responseContainer = "{{{containerType}}}"{{/containerType}}){{#hasMore}},{{/hasMore}}{{/responses}} }) + {{#implicitHeaders}} + @ApiImplicitParams({ + {{#headerParams}} + {{>implicitHeader}} + {{/headerParams}} + }) + {{/implicitHeaders}} + {{/useOas2}} + {{^useOas2}} + @io.swagger.v3.oas.annotations.Operation(summary = "{{{summary}}}", description = "{{{notes}}}"{{#hasAuthMethods}}, security = { + {{#authMethods}}@SecurityRequirement(name = "{{name}}"{{#isOAuth}}, scopes = { + {{#each scopes}}"{{@key}}"{{^@last}}, + {{/@last}}{{/each}} + }{{/isOAuth}}){{#hasMore}}, + {{/hasMore}}{{/authMethods}} + }{{/hasAuthMethods}}, tags={ {{#vendorExtensions.x-tags}}"{{tag}}"{{#hasMore}}, {{/hasMore}}{{/vendorExtensions.x-tags}} }) + @ApiResponses(value = { {{#responses}} + @ApiResponse(responseCode = "{{{code}}}", description = "{{{message}}}"{{^vendorExtensions.x-java-is-response-void}}{{#baseType}}, content = @Content({{#schema.extensions.x-content-type}}mediaType = "{{schema.extensions.x-content-type}}", {{/schema.extensions.x-content-type}}{{^containerType}}schema = @Schema(implementation = {{{baseType}}}.class)){{/containerType}}{{#containerType}}array = @ArraySchema(schema = @Schema(implementation = {{{baseType}}}.class))){{/containerType}}{{/baseType}}{{/vendorExtensions.x-java-is-response-void}}){{#hasMore}}, + {{/hasMore}}{{/responses}} }) + {{/useOas2}} + @RequestMapping(value = "{{{path}}}",{{#singleContentTypes}}{{#hasProduces}} + produces = "{{{vendorExtensions.x-accepts}}}", {{/hasProduces}}{{#hasConsumes}} + consumes = "{{{vendorExtensions.x-contentType}}}",{{/hasConsumes}}{{/singleContentTypes}}{{^singleContentTypes}}{{#hasProduces}} + produces = { {{#produces}}"{{{mediaType}}}"{{#hasMore}}, {{/hasMore}}{{/produces}} }, {{/hasProduces}}{{#hasConsumes}} + consumes = { {{#consumes}}"{{{mediaType}}}"{{#hasMore}}, {{/hasMore}}{{/consumes}} }, {{/hasConsumes}}{{/singleContentTypes}} + method = RequestMethod.{{httpMethod}}) + {{#defaultInterfaces}}default {{/defaultInterfaces}}{{#responseWrapper}}{{.}}<{{/responseWrapper}}ResponseEntity<{{>returnTypes}}>{{#responseWrapper}}>{{/responseWrapper}} {{#delegate-method}}_{{/delegate-method}}{{operationId}}({{#parameters}}{{>queryParams}}{{>pathParams}}{{>headerParams}}{{>cookieParams}}{{>bodyParams}}{{>formParams}}{{#hasMore}}, {{/hasMore}}{{/parameters}}){{^defaultInterfaces}}{{#throwsException}} throws Exception{{/throwsException}};{{/defaultInterfaces}}{{#defaultInterfaces}}{{#throwsException}} throws Exception{{/throwsException}} { + {{#delegate-method}} + return {{operationId}}({{#parameters}}{{paramName}}{{#hasMore}}, {{/hasMore}}{{/parameters}}); + } + + // Override this method + default {{#responseWrapper}}{{.}}<{{/responseWrapper}}ResponseEntity<{{>returnTypes}}>{{#responseWrapper}}>{{/responseWrapper}} {{operationId}}({{#parameters}}{{^isBinary}}{{{dataType}}}{{/isBinary}}{{#isBinary}}MultipartFile{{/isBinary}} {{paramName}}{{#hasMore}},{{/hasMore}}{{/parameters}}) { + {{/delegate-method}} + {{^isDelegate}} + if(getObjectMapper().isPresent() && getAcceptHeader().isPresent()) { + {{#examples}} + if (getAcceptHeader().get().contains("{{{contentType}}}")) { + try { + return {{#async}}CompletableFuture.completedFuture({{/async}}new ResponseEntity<>(getObjectMapper().get().readValue("{{#lambdaRemoveLineBreak}}{{#lambdaEscapeDoubleQuote}}{{{example}}}{{/lambdaEscapeDoubleQuote}}{{/lambdaRemoveLineBreak}}", {{>exampleReturnTypes}}.class), HttpStatus.NOT_IMPLEMENTED){{#async}}){{/async}}; + } catch (IOException e) { + log.error("Couldn't serialize response for content type {{{contentType}}}", e); + return {{#async}}CompletableFuture.completedFuture({{/async}}new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR){{#async}}){{/async}}; + } + } + {{/examples}} + } else { + log.warn("ObjectMapper or HttpServletRequest not configured in default {{classname}} interface so no example is generated"); + } + return {{#async}}CompletableFuture.completedFuture({{/async}}new ResponseEntity<>(HttpStatus.NOT_IMPLEMENTED){{#async}}){{/async}}; + {{/isDelegate}} + {{#isDelegate}} + return getDelegate().{{operationId}}({{#parameters}}{{paramName}}{{#hasMore}}, {{/hasMore}}{{/parameters}}); + {{/isDelegate}} + }{{/defaultInterfaces}} + +{{/@first}} +{{/contents}} +{{/operation}} +} +{{/operations}} diff --git a/metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/apiController.mustache b/metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/apiController.mustache new file mode 100644 index 0000000000000..9499eba3f4b22 --- /dev/null +++ b/metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/apiController.mustache @@ -0,0 +1,127 @@ +package {{package}}; + +import io.datahubproject.openapi.delegates.EntityApiDelegateImpl; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.search.SearchService; +import io.datahubproject.openapi.entities.EntitiesController; +import com.datahub.authorization.AuthorizerChain; + +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.beans.factory.annotation.Value; + +{{#imports}}import {{import}}; +{{/imports}} + +{{^isDelegate}} +import com.fasterxml.jackson.databind.ObjectMapper; +{{/isDelegate}} + +{{#useOas2}} +import io.swagger.annotations.*; +{{/useOas2}} +{{^useOas2}} +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +{{/useOas2}} +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; + +import org.springframework.web.bind.annotation.CookieValue; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RequestPart; +{{^useOas2}} +{{/useOas2}} +import org.springframework.web.multipart.MultipartFile; + + {{#useBeanValidation}} +import javax.validation.constraints.*; +import javax.validation.Valid; + {{/useBeanValidation}} + +{{^isDelegate}} +import javax.servlet.http.HttpServletRequest; + {{#isJava8or11}} +import java.util.Optional; + {{/isJava8or11}} +{{/isDelegate}} +{{^jdk8-no-delegate}} + {{#useOptional}} +import java.util.Optional; + {{/useOptional}} +{{/jdk8-no-delegate}} + + {{^isDelegate}} +import java.io.IOException; + {{/isDelegate}} +import java.util.List; +import java.util.Map; + {{#async}} +import java.util.concurrent.Callable; + {{/async}} + +{{>generatedAnnotation}} +{{#useOas2}} +{{/useOas2}} +{{^useOas2}} +{{/useOas2}} +{{#operations}} +@RestController +@RequestMapping("/v2/entity") +public class {{classname}}Controller implements {{classname}} { + + private static final Logger log = LoggerFactory.getLogger({{classname}}Controller.class); + + private final ObjectMapper objectMapper; + + private final HttpServletRequest request; + + private final EntityApiDelegateImpl<{{requestClass}}, {{responseClass}}, {{scrollResponseClass}}> delegate; + + @org.springframework.beans.factory.annotation.Autowired + public {{classname}}Controller(ObjectMapper objectMapper, HttpServletRequest request, EntityService entityService, + SearchService searchService, EntitiesController v1Controller, AuthorizerChain authorizationChain, + @Value("${authorization.restApiAuthorization:false}") boolean restApiAuthorizationEnabled) { + this.objectMapper = objectMapper; + this.request = request; + this.delegate = new EntityApiDelegateImpl<{{requestClass}}, {{responseClass}}, {{scrollResponseClass}}>(entityService, searchService, v1Controller, + restApiAuthorizationEnabled, authorizationChain, {{requestClass}}.class, {{responseClass}}.class, {{scrollResponseClass}}.class); + } + {{#isJava8or11}} + + @Override + public Optional getObjectMapper() { + return Optional.ofNullable(objectMapper); + } + + @Override + public Optional getRequest() { + return Optional.ofNullable(request); + } + {{/isJava8or11}} + +{{#operation}} +{{#contents}} +{{#@first}} + @Override + public ResponseEntity<{{>returnTypes}}>{{#async}}>{{/async}} {{operationId}}({{#parameters}}{{>queryParams}}{{>pathParams}}{{>headerParams}}{{>bodyParams}}{{>formParams}}{{>cookieParams}}{{#hasMore}},{{/hasMore}}{{/parameters}}) { + return delegate.{{operationId}}({{#allParams}}{{paramName}}{{#hasMore}}, {{/hasMore}}{{/allParams}}); + } + +{{/@first}} +{{/contents}} +{{/operation}} +} +{{/operations}} \ No newline at end of file diff --git a/metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/readme.txt b/metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/readme.txt new file mode 100644 index 0000000000000..f127cd3e68553 --- /dev/null +++ b/metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/readme.txt @@ -0,0 +1,2 @@ +Original: +https://github.com/swagger-api/swagger-codegen-generators/tree/master/src/main/resources/handlebars/JavaSpring \ No newline at end of file diff --git a/metadata-service/openapi-entity-servlet/src/main/resources/entity-v2.0.yml b/metadata-service/openapi-entity-servlet/src/main/resources/entity-v2.0.yml new file mode 100644 index 0000000000000..bebbf05e9334b --- /dev/null +++ b/metadata-service/openapi-entity-servlet/src/main/resources/entity-v2.0.yml @@ -0,0 +1,6 @@ + +openapi: "3.0.0" +info: + title: Entity API + description: This is a service for DataHub Entities. + version: v2 diff --git a/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/config/OpenAPIEntityTestConfiguration.java b/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/config/OpenAPIEntityTestConfiguration.java new file mode 100644 index 0000000000000..b7e255b8c270e --- /dev/null +++ b/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/config/OpenAPIEntityTestConfiguration.java @@ -0,0 +1,119 @@ +package io.datahubproject.openapi.config; + +import com.datahub.authentication.Actor; +import com.datahub.authentication.ActorType; +import com.datahub.authentication.Authentication; +import com.datahub.authentication.AuthenticationContext; +import com.datahub.authorization.AuthorizationResult; +import com.datahub.authorization.AuthorizerChain; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityServiceImpl; +import com.linkedin.metadata.models.registry.ConfigEntityRegistry; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.models.registry.EntityRegistryException; +import com.linkedin.metadata.search.ScrollResult; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.metadata.search.SearchService; +import com.linkedin.metadata.systemmetadata.SystemMetadataService; +import com.linkedin.metadata.timeline.TimelineService; +import io.datahubproject.openapi.dto.UrnResponseMap; +import io.datahubproject.openapi.entities.EntitiesController; +import io.datahubproject.openapi.generated.EntityResponse; +import io.datahubproject.openapi.relationships.RelationshipsController; +import io.datahubproject.openapi.timeline.TimelineController; +import org.mockito.Mockito; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Primary; +import org.springframework.http.ResponseEntity; + +import java.util.Arrays; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + + +@TestConfiguration +public class OpenAPIEntityTestConfiguration { + @Bean + public ObjectMapper objectMapper() { + return new ObjectMapper(new YAMLFactory()); + } + + @Bean + @Primary + public EntityService entityService(final EntityRegistry mockRegistry) { + EntityService entityService = mock(EntityServiceImpl.class); + when(entityService.getEntityRegistry()).thenReturn(mockRegistry); + return entityService; + } + + @Bean + @Primary + public SearchService searchService() { + SearchService searchService = mock(SearchService.class); + when(searchService.scrollAcrossEntities(anyList(), any(), any(), any(), + any(), any(), anyInt(), any())) + .thenReturn(new ScrollResult().setEntities(new SearchEntityArray())); + + return searchService; + } + + @Bean + public AuthorizerChain authorizerChain() { + AuthorizerChain authorizerChain = Mockito.mock(AuthorizerChain.class); + + Authentication authentication = Mockito.mock(Authentication.class); + when(authentication.getActor()).thenReturn(new Actor(ActorType.USER, "datahub")); + when(authorizerChain.authorize(any())).thenReturn(new AuthorizationResult(null, AuthorizationResult.Type.ALLOW, "")); + AuthenticationContext.setAuthentication(authentication); + + return authorizerChain; + } + + @MockBean(name = "elasticSearchSystemMetadataService") + public SystemMetadataService systemMetadataService; + + @MockBean + public TimelineService timelineService; + + @Bean("entityRegistry") + @Primary + public ConfigEntityRegistry configEntityRegistry() throws EntityRegistryException { + return new ConfigEntityRegistry( + OpenAPIEntityTestConfiguration.class.getClassLoader().getResourceAsStream("entity-registry.yml")); + } + + /* Controllers not under this module */ + @Bean + @Primary + public EntitiesController entitiesController() { + EntitiesController entitiesController = mock(EntitiesController.class); + when(entitiesController.getEntities(any(), any())) + .thenAnswer(params -> { + String[] urns = params.getArgument(0); + String[] aspects = params.getArgument(1); + return ResponseEntity.ok(UrnResponseMap.builder() + .responses(Arrays.stream(urns) + .map(urn -> Map.entry(urn, EntityResponse.builder().urn(urn).build())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) + .build()); + }); + + return entitiesController; + } + + @MockBean + public TimelineController timelineController; + + @MockBean + public RelationshipsController relationshipsController; +} diff --git a/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/delegates/EntityApiDelegateImplTest.java b/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/delegates/EntityApiDelegateImplTest.java new file mode 100644 index 0000000000000..fc2aae1a75ab8 --- /dev/null +++ b/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/delegates/EntityApiDelegateImplTest.java @@ -0,0 +1,203 @@ +package io.datahubproject.openapi.delegates; + +import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor; +import io.datahubproject.openapi.config.OpenAPIEntityTestConfiguration; +import io.datahubproject.openapi.config.SpringWebConfig; +import io.datahubproject.openapi.generated.BrowsePathEntry; +import io.datahubproject.openapi.generated.BrowsePathsV2; +import io.datahubproject.openapi.generated.BrowsePathsV2AspectRequestV2; +import io.datahubproject.openapi.generated.ChartEntityRequestV2; +import io.datahubproject.openapi.generated.ChartEntityResponseV2; +import io.datahubproject.openapi.generated.DatasetEntityRequestV2; +import io.datahubproject.openapi.generated.DatasetEntityResponseV2; +import io.datahubproject.openapi.generated.Deprecation; +import io.datahubproject.openapi.generated.DeprecationAspectRequestV2; +import io.datahubproject.openapi.generated.Domains; +import io.datahubproject.openapi.generated.DomainsAspectRequestV2; +import io.datahubproject.openapi.generated.GlobalTags; +import io.datahubproject.openapi.generated.GlobalTagsAspectRequestV2; +import io.datahubproject.openapi.generated.GlossaryTermAssociation; +import io.datahubproject.openapi.generated.GlossaryTerms; +import io.datahubproject.openapi.generated.GlossaryTermsAspectRequestV2; +import io.datahubproject.openapi.generated.Owner; +import io.datahubproject.openapi.generated.Ownership; +import io.datahubproject.openapi.generated.OwnershipAspectRequestV2; +import io.datahubproject.openapi.generated.OwnershipType; +import io.datahubproject.openapi.generated.ScrollChartEntityResponseV2; +import io.datahubproject.openapi.generated.ScrollDatasetEntityResponseV2; +import io.datahubproject.openapi.generated.Status; +import io.datahubproject.openapi.generated.StatusAspectRequestV2; +import io.datahubproject.openapi.generated.TagAssociation; +import io.datahubproject.openapi.generated.controller.ChartApiController; +import io.datahubproject.openapi.generated.controller.DatasetApiController; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Import; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import java.util.List; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; + + +@SpringBootTest(classes = {SpringWebConfig.class}) +@ComponentScan(basePackages = {"io.datahubproject.openapi.generated.controller"}) +@Import({OpenAPIEntityTestConfiguration.class}) +public class EntityApiDelegateImplTest extends AbstractTestNGSpringContextTests { + @BeforeTest + public void disableAssert() { + PathSpecBasedSchemaAnnotationVisitor.class.getClassLoader() + .setClassAssertionStatus(PathSpecBasedSchemaAnnotationVisitor.class.getName(), false); + } + + @Autowired + private ChartApiController chartApiController; + @Autowired + private DatasetApiController datasetApiController; + + @Test + public void initTest() { + assertNotNull(chartApiController); + assertNotNull(datasetApiController); + } + + @Test + public void chartApiControllerTest() { + final String testUrn = "urn:li:chart:(looker,baz1)"; + + ChartEntityRequestV2 req = ChartEntityRequestV2.builder() + .urn(testUrn) + .build(); + ChartEntityResponseV2 resp = chartApiController.create(List.of(req)).getBody().get(0); + assertEquals(resp.getUrn(), testUrn); + + resp = chartApiController.get(testUrn, false, List.of()).getBody(); + assertEquals(resp.getUrn(), testUrn); + + ResponseEntity deleteResp = chartApiController.delete(testUrn); + assertEquals(deleteResp.getStatusCode(), HttpStatus.OK); + + ResponseEntity headResp = chartApiController.head(testUrn); + assertEquals(headResp.getStatusCode(), HttpStatus.NOT_FOUND); + + ResponseEntity scrollResp = chartApiController.scroll( + false, List.of(), 10, null, null, null, null); + assertEquals(scrollResp.getStatusCode(), HttpStatus.OK); + assertNotNull(scrollResp.getBody().getEntities()); + } + + @Test + public void datasetApiControllerTest() { + final String testUrn = "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"; + + DatasetEntityRequestV2 req = DatasetEntityRequestV2.builder() + .urn(testUrn) + .build(); + DatasetEntityResponseV2 resp = datasetApiController.create(List.of(req)).getBody().get(0); + assertEquals(resp.getUrn(), testUrn); + + resp = datasetApiController.get(testUrn, false, List.of()).getBody(); + assertEquals(resp.getUrn(), testUrn); + + ResponseEntity deleteResp = datasetApiController.delete(testUrn); + assertEquals(deleteResp.getStatusCode(), HttpStatus.OK); + + ResponseEntity headResp = datasetApiController.head(testUrn); + assertEquals(headResp.getStatusCode(), HttpStatus.NOT_FOUND); + + ResponseEntity scrollResp = datasetApiController.scroll( + false, List.of(), 10, null, null, null, null); + assertEquals(scrollResp.getStatusCode(), HttpStatus.OK); + assertNotNull(scrollResp.getBody().getEntities()); + } + + @Test + public void browsePathsTest() { + final String testUrn = "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"; + + BrowsePathsV2AspectRequestV2 req = BrowsePathsV2AspectRequestV2.builder() + .value(BrowsePathsV2.builder().path(List.of(BrowsePathEntry.builder().urn(testUrn) + .id("path").build())).build()).build(); + assertEquals(datasetApiController.createBrowsePathsV2(testUrn, req).getStatusCode(), HttpStatus.OK); + assertEquals(datasetApiController.deleteBrowsePathsV2(testUrn).getStatusCode(), HttpStatus.OK); + assertEquals(datasetApiController.getBrowsePathsV2(testUrn, false).getStatusCode(), HttpStatus.NOT_FOUND); + assertEquals(datasetApiController.headBrowsePathsV2(testUrn).getStatusCode(), HttpStatus.NOT_FOUND); + } + + @Test + public void deprecationTest() { + final String testUrn = "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"; + + DeprecationAspectRequestV2 req = DeprecationAspectRequestV2.builder() + .value(Deprecation.builder().deprecated(true).build()).build(); + assertEquals(datasetApiController.createDeprecation(testUrn, req).getStatusCode(), HttpStatus.OK); + assertEquals(datasetApiController.deleteDeprecation(testUrn).getStatusCode(), HttpStatus.OK); + assertEquals(datasetApiController.getDeprecation(testUrn, false).getStatusCode(), HttpStatus.NOT_FOUND); + assertEquals(datasetApiController.headDeprecation(testUrn).getStatusCode(), HttpStatus.NOT_FOUND); + } + + @Test + public void domainsTest() { + final String testUrn = "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"; + + DomainsAspectRequestV2 req = DomainsAspectRequestV2.builder() + .value(Domains.builder().domains(List.of("my_domain")).build()).build(); + assertEquals(datasetApiController.createDomains(testUrn, req).getStatusCode(), HttpStatus.OK); + assertEquals(datasetApiController.deleteDomains(testUrn).getStatusCode(), HttpStatus.OK); + assertEquals(datasetApiController.getDomains(testUrn, false).getStatusCode(), HttpStatus.NOT_FOUND); + assertEquals(datasetApiController.headDomains(testUrn).getStatusCode(), HttpStatus.NOT_FOUND); + } + + @Test + public void ownershipTest() { + final String testUrn = "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"; + + OwnershipAspectRequestV2 req = OwnershipAspectRequestV2.builder() + .value(Ownership.builder().owners(List.of(Owner.builder().owner("me").type(OwnershipType.BUSINESS_OWNER).build())).build()).build(); + assertEquals(datasetApiController.createOwnership(testUrn, req).getStatusCode(), HttpStatus.OK); + assertEquals(datasetApiController.deleteOwnership(testUrn).getStatusCode(), HttpStatus.OK); + assertEquals(datasetApiController.getOwnership(testUrn, false).getStatusCode(), HttpStatus.NOT_FOUND); + assertEquals(datasetApiController.headOwnership(testUrn).getStatusCode(), HttpStatus.NOT_FOUND); + } + + @Test + public void statusTest() { + final String testUrn = "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"; + + StatusAspectRequestV2 req = StatusAspectRequestV2.builder().value(Status.builder().removed(true).build()).build(); + assertEquals(datasetApiController.createStatus(testUrn, req).getStatusCode(), HttpStatus.OK); + assertEquals(datasetApiController.deleteStatus(testUrn).getStatusCode(), HttpStatus.OK); + assertEquals(datasetApiController.getStatus(testUrn, false).getStatusCode(), HttpStatus.NOT_FOUND); + assertEquals(datasetApiController.headStatus(testUrn).getStatusCode(), HttpStatus.NOT_FOUND); + } + + @Test + public void globalTagsTest() { + final String testUrn = "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"; + + GlobalTagsAspectRequestV2 req = GlobalTagsAspectRequestV2.builder() + .value(GlobalTags.builder().tags(List.of(TagAssociation.builder().tag("tag").build())).build()).build(); + assertEquals(datasetApiController.createGlobalTags(testUrn, req).getStatusCode(), HttpStatus.OK); + assertEquals(datasetApiController.deleteGlobalTags(testUrn).getStatusCode(), HttpStatus.OK); + assertEquals(datasetApiController.getGlobalTags(testUrn, false).getStatusCode(), HttpStatus.NOT_FOUND); + assertEquals(datasetApiController.headGlobalTags(testUrn).getStatusCode(), HttpStatus.NOT_FOUND); + } + + @Test + public void glossaryTermsTest() { + final String testUrn = "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"; + + GlossaryTermsAspectRequestV2 req = GlossaryTermsAspectRequestV2.builder() + .value(GlossaryTerms.builder().terms(List.of(GlossaryTermAssociation.builder().urn("term urn").build())).build()).build(); + assertEquals(datasetApiController.createGlossaryTerms(testUrn, req).getStatusCode(), HttpStatus.OK); + assertEquals(datasetApiController.deleteGlossaryTerms(testUrn).getStatusCode(), HttpStatus.OK); + assertEquals(datasetApiController.getGlossaryTerms(testUrn, false).getStatusCode(), HttpStatus.NOT_FOUND); + assertEquals(datasetApiController.headGlossaryTerms(testUrn).getStatusCode(), HttpStatus.NOT_FOUND); + } +} diff --git a/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/util/OpenApiEntitiesUtilTest.java b/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/util/OpenApiEntitiesUtilTest.java new file mode 100644 index 0000000000000..8f87b041a7e03 --- /dev/null +++ b/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/util/OpenApiEntitiesUtilTest.java @@ -0,0 +1,55 @@ +package io.datahubproject.openapi.util; + +import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor; +import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; +import com.linkedin.metadata.models.registry.EntityRegistry; +import io.datahubproject.openapi.config.OpenAPIEntityTestConfiguration; +import io.datahubproject.openapi.dto.UpsertAspectRequest; +import io.datahubproject.openapi.generated.ContainerEntityRequestV2; +import io.datahubproject.openapi.generated.ContainerKey; +import io.datahubproject.openapi.generated.ContainerKeyAspectRequestV2; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.PropertySource; +import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import java.util.List; + +import static org.junit.Assert.assertNotNull; +import static org.testng.AssertJUnit.assertEquals; + + +@Import({OpenAPIEntityTestConfiguration.class}) +@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) +public class OpenApiEntitiesUtilTest extends AbstractTestNGSpringContextTests { + @Autowired + private EntityRegistry entityRegistry; + + @BeforeTest + public void disableAssert() { + PathSpecBasedSchemaAnnotationVisitor.class.getClassLoader() + .setClassAssertionStatus(PathSpecBasedSchemaAnnotationVisitor.class.getName(), false); + } + + @Test + public void testInitialization() { + assertNotNull(entityRegistry); + } + + @Test + public void containerConversionTest() { + ContainerEntityRequestV2 test = ContainerEntityRequestV2.builder() + .urn("urn:li:container:123") + .containerKey(ContainerKeyAspectRequestV2.builder().value(ContainerKey.builder().guid("123").build()).build()) + .build(); + List expected = List.of(UpsertAspectRequest.builder() + .entityType("container") + .entityUrn("urn:li:container:123") + .aspect(ContainerKey.builder().guid("123").build()) + .build()); + + assertEquals(expected, OpenApiEntitiesUtil.convertEntityToUpsert(test, ContainerEntityRequestV2.class, entityRegistry)); + } +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java index 6148149ca6da4..9feb9c8e5640f 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java @@ -2,6 +2,7 @@ import io.datahubproject.openapi.converter.StringToChangeCategoryConverter; import io.swagger.v3.oas.annotations.OpenAPIDefinition; +import io.swagger.v3.oas.annotations.info.Info; import io.swagger.v3.oas.annotations.servers.Server; import java.util.List; import org.springframework.context.annotation.Configuration; @@ -16,7 +17,8 @@ @EnableWebMvc -@OpenAPIDefinition(servers = {@Server(url = "/openapi/", description = "Default Server URL")}) +@OpenAPIDefinition(info = @Info(title = "DataHub OpenAPI", version = "2.0.0"), + servers = {@Server(url = "/openapi/", description = "Default Server URL")}) @Configuration public class SpringWebConfig implements WebMvcConfigurer { diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/entities/EntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/entities/EntitiesController.java index 64d815af91aef..6439e2f31f7b0 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/entities/EntitiesController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/entities/EntitiesController.java @@ -52,7 +52,7 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import static com.linkedin.metadata.utils.PegasusUtils.*; +import static com.linkedin.metadata.utils.PegasusUtils.urnToEntityName; @RestController @@ -201,7 +201,7 @@ public ResponseEntity> deleteEntities( .map(proposal -> MappingUtil.ingestProposal(proposal, actorUrnStr, _entityService)) .filter(Pair::getSecond) .map(Pair::getFirst) - .map(urnString -> new AspectRowSummary().urn(urnString)) + .map(urnString -> AspectRowSummary.builder().urn(urnString).build()) .collect(Collectors.toList())) .rowsDeletedFromEntityDeletion(deleteRequests.size()) .build())); diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java index 4d0e5e7df29d5..68a8c8ca49235 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java @@ -7,13 +7,16 @@ import com.datahub.authorization.DisjunctivePrivilegeGroup; import com.datahub.authorization.ResourceSpec; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.avro2pegasus.events.KafkaAuditHeader; import com.linkedin.avro2pegasus.events.UUID; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.ByteString; +import com.linkedin.data.DataList; import com.linkedin.data.DataMap; import com.linkedin.data.template.RecordTemplate; import com.linkedin.entity.Aspect; @@ -33,27 +36,33 @@ import com.linkedin.util.Pair; import io.datahubproject.openapi.dto.RollbackRunResultDto; import io.datahubproject.openapi.dto.UpsertAspectRequest; -import io.datahubproject.openapi.generated.AspectRowSummary; -import io.datahubproject.openapi.generated.AspectType; -import io.datahubproject.openapi.generated.AuditStamp; -import io.datahubproject.openapi.generated.EntityResponse; -import io.datahubproject.openapi.generated.EnvelopedAspect; -import io.datahubproject.openapi.generated.MetadataChangeProposal; -import io.datahubproject.openapi.generated.OneOfEnvelopedAspectValue; -import io.datahubproject.openapi.generated.OneOfGenericAspectValue; -import io.datahubproject.openapi.generated.Status; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import io.datahubproject.openapi.generated.AspectRowSummary; +import io.datahubproject.openapi.generated.AspectType; +import io.datahubproject.openapi.generated.AuditStamp; +import io.datahubproject.openapi.generated.EntityResponse; +import io.datahubproject.openapi.generated.EnvelopedAspect; +import io.datahubproject.openapi.generated.MetadataChangeProposal; +import io.datahubproject.openapi.generated.OneOfEnvelopedAspectValue; +import io.datahubproject.openapi.generated.OneOfGenericAspectValue; +import io.datahubproject.openapi.generated.Status; import lombok.extern.slf4j.Slf4j; +import org.apache.avro.Schema; import org.reflections.Reflections; import org.reflections.scanners.SubTypesScanner; import org.springframework.beans.factory.config.BeanDefinition; @@ -63,8 +72,9 @@ import org.springframework.http.MediaType; import org.springframework.web.client.HttpClientErrorException; -import static com.linkedin.metadata.Constants.*; -import static java.nio.charset.StandardCharsets.*; +import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME; +import static io.datahubproject.openapi.util.ReflectionCache.toUpperFirst; +import static java.nio.charset.StandardCharsets.UTF_8; @Slf4j public class MappingUtil { @@ -72,24 +82,19 @@ private MappingUtil() { } + private static final JsonNodeFactory NODE_FACTORY = JsonNodeFactory.instance; private static final Map> ENVELOPED_ASPECT_TYPE_MAP = new HashMap<>(); private static final Map, String> ASPECT_NAME_MAP = new HashMap<>(); private static final Map> PEGASUS_TYPE_MAP = new HashMap<>(); - private static final Pattern CLASS_NAME_PATTERN = - Pattern.compile("(\"com\\.linkedin\\.)([a-z]+?\\.)+?(?[A-Z]\\w+?)(\":\\{)(?.*?)(}})"); - private static final Pattern GLOBAL_TAGS_PATTERN = - Pattern.compile("\"globalTags\":\\{"); - private static final Pattern GLOSSARY_TERMS_PATTERN = - Pattern.compile("\"glossaryTerms\":\\{"); private static final String DISCRIMINATOR = "__type"; - private static final Pattern CLASS_TYPE_NAME_PATTERN = - Pattern.compile("(\\s+?\"__type\"\\s+?:\\s+?\")(?\\w*?)(\"[,]?\\s+?)(?[\\S\\s]*?)(\\s+})"); private static final String PEGASUS_PACKAGE = "com.linkedin"; - private static final String GLOBAL_TAGS = "GlobalTags"; - private static final String GLOSSARY_TERMS = "GlossaryTerms"; + private static final ReflectionCache REFLECT_AVRO = ReflectionCache.builder() + .basePackage("com.linkedin.pegasus2avro").build(); + private static final ReflectionCache REFLECT_OPENAPI = ReflectionCache.builder() + .basePackage("io.datahubproject.openapi.generated").build(); static { // Build a map from __type name to generated class @@ -117,59 +122,75 @@ public static Map mapServiceResponse(Map mapEnvelopedAspect(entry.getValue(), objectMapper)))); + return EntityResponse.builder() + .entityName(entityResponse.getEntityName()) + .urn(entityResponse.getUrn().toString()) + .aspects(entityResponse.getAspects() + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> mapEnvelopedAspect(entry.getValue(), objectMapper)))).build(); } public static EnvelopedAspect mapEnvelopedAspect(com.linkedin.entity.EnvelopedAspect envelopedAspect, - ObjectMapper objectMapper) { - return new EnvelopedAspect() - .name(envelopedAspect.getName()) - .timestamp(envelopedAspect.getTimestamp()) - .version(envelopedAspect.getVersion()) - .type(AspectType.fromValue(envelopedAspect.getType().name().toUpperCase(Locale.ROOT))) - .created(objectMapper.convertValue(envelopedAspect.getCreated().data(), AuditStamp.class)) - .value(mapAspectValue(envelopedAspect.getName(), envelopedAspect.getValue(), objectMapper)); + ObjectMapper objectMapper) { + return EnvelopedAspect.builder() + .name(envelopedAspect.getName()) + .timestamp(envelopedAspect.getTimestamp()) + .version(envelopedAspect.getVersion()) + .type(AspectType.fromValue(envelopedAspect.getType().name().toUpperCase(Locale.ROOT))) + .created(objectMapper.convertValue(envelopedAspect.getCreated().data(), AuditStamp.class)) + .value(mapAspectValue(envelopedAspect.getName(), envelopedAspect.getValue(), objectMapper)).build(); + } + + private static DataMap insertDiscriminator(@Nullable Class parentClazz, DataMap dataMap) { + if (REFLECT_OPENAPI.lookupMethod(parentClazz, "get__type") != null) { + dataMap.put(DISCRIMINATOR, parentClazz.getSimpleName()); + } + + Set> requiresDiscriminator = dataMap.entrySet().stream() + .filter(e -> e.getValue() instanceof DataMap) + .filter(e -> e.getKey().startsWith(PEGASUS_PACKAGE + ".")) + .map(e -> Map.entry(e.getKey(), (DataMap) e.getValue())) + .collect(Collectors.toSet()); + requiresDiscriminator.forEach(e -> { + dataMap.remove(e.getKey()); + dataMap.put(DISCRIMINATOR, e.getKey().substring(e.getKey().lastIndexOf('.') + 1)); + dataMap.putAll(e.getValue()); + }); + + Set> recurse = dataMap.entrySet().stream() + .filter(e -> e.getValue() instanceof DataMap || e.getValue() instanceof DataList) + .flatMap(e -> { + if (e.getValue() instanceof DataList) { + return ((DataList) e.getValue()).stream() + .filter(item -> item instanceof DataMap) + .map(item -> Pair.of((String) null, (DataMap) item)); + } else { + return Stream.of(Pair.of(e.getKey(), (DataMap) e.getValue())); + } + }).collect(Collectors.toSet()); + + recurse.forEach(e -> { + if (e.getKey() != null) { + Class getterClazz = null; + if (parentClazz != null) { + Method getMethod = REFLECT_OPENAPI.lookupMethod(parentClazz, "get" + toUpperFirst(e.getKey())); + getterClazz = getMethod.getReturnType(); + } + insertDiscriminator(getterClazz, e.getValue()); + } else { + insertDiscriminator(null, e.getValue()); + } + }); + + return dataMap; } public static OneOfEnvelopedAspectValue mapAspectValue(String aspectName, Aspect aspect, ObjectMapper objectMapper) { Class aspectClass = ENVELOPED_ASPECT_TYPE_MAP.get(aspectName); - DataMap wrapper = aspect.data(); - wrapper.put(DISCRIMINATOR, aspectClass.getSimpleName()); - String dataMapAsJson; + DataMap wrapper = insertDiscriminator(aspectClass, aspect.data()); try { - dataMapAsJson = objectMapper.writeValueAsString(wrapper); - Matcher classNameMatcher = CLASS_NAME_PATTERN.matcher(dataMapAsJson); - while (classNameMatcher.find()) { - String className = classNameMatcher.group("className"); - String content = classNameMatcher.group("content"); - StringBuilder replacement = new StringBuilder("\"" + DISCRIMINATOR + "\" : \"" + className + "\""); - - if (content.length() > 0) { - replacement.append(",") - .append(content); - } - replacement.append("}"); - dataMapAsJson = classNameMatcher.replaceFirst(Matcher.quoteReplacement(replacement.toString())); - classNameMatcher = CLASS_NAME_PATTERN.matcher(dataMapAsJson); - } - // Global Tags & Glossary Terms will not have the explicit class name in the DataMap, so we handle them differently - Matcher globalTagsMatcher = GLOBAL_TAGS_PATTERN.matcher(dataMapAsJson); - while (globalTagsMatcher.find()) { - String replacement = "\"globalTags\" : {\"" + DISCRIMINATOR + "\" : \"GlobalTags\","; - dataMapAsJson = globalTagsMatcher.replaceFirst(Matcher.quoteReplacement(replacement)); - globalTagsMatcher = GLOBAL_TAGS_PATTERN.matcher(dataMapAsJson); - } - Matcher glossaryTermsMatcher = GLOSSARY_TERMS_PATTERN.matcher(dataMapAsJson); - while (glossaryTermsMatcher.find()) { - String replacement = "\"glossaryTerms\" : {\"" + DISCRIMINATOR + "\" : \"GlossaryTerms\","; - dataMapAsJson = glossaryTermsMatcher.replaceFirst(Matcher.quoteReplacement(replacement)); - glossaryTermsMatcher = GLOSSARY_TERMS_PATTERN.matcher(dataMapAsJson); - } + String dataMapAsJson = objectMapper.writeValueAsString(wrapper); return objectMapper.readValue(dataMapAsJson, aspectClass); } catch (JsonProcessingException e) { throw new RuntimeException(e); @@ -206,36 +227,79 @@ private static String getAspectName(Class cls) { return new String(c); } + private static Optional shouldDiscriminate(String parentShortClass, String fieldName, ObjectNode node) { + try { + if (parentShortClass != null) { + Class pegasus2AvroClazz = REFLECT_AVRO.lookupClass(parentShortClass, true); + Method getClassSchema = REFLECT_AVRO.lookupMethod(pegasus2AvroClazz, "getClassSchema"); + Schema avroSchema = (Schema) getClassSchema.invoke(null); + Schema.Field avroField = avroSchema.getField(fieldName); + + if (avroField.schema().isUnion()) { + Class discriminatedClazz = REFLECT_AVRO.lookupClass(node.get(DISCRIMINATOR).asText(), true); + return Optional.of(discriminatedClazz.getName().replace(".pegasus2avro", "")); + } + } + + // check leaf + Iterator itr = node.fieldNames(); + itr.next(); + if (!itr.hasNext()) { // only contains discriminator + Class discriminatedClazz = REFLECT_AVRO.lookupClass(node.get(DISCRIMINATOR).asText(), true); + return Optional.of(discriminatedClazz.getName().replace(".pegasus2avro", "")); + } + + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + return Optional.empty(); + } + + private static void replaceDiscriminator(ObjectNode node) { + replaceDiscriminator(null, null, null, node); + } + private static void replaceDiscriminator(@Nullable ObjectNode parentNode, @Nullable String parentDiscriminator, + @Nullable String propertyName, @Nonnull ObjectNode node) { + + final String discriminator; + if (node.isObject() && node.has(DISCRIMINATOR)) { + Optional discriminatorClassName = shouldDiscriminate(parentDiscriminator, propertyName, node); + if (parentNode != null && discriminatorClassName.isPresent()) { + discriminator = node.remove(DISCRIMINATOR).asText(); + parentNode.remove(propertyName); + parentNode.set(propertyName, NODE_FACTORY.objectNode().set(discriminatorClassName.get(), node)); + } else { + discriminator = node.remove(DISCRIMINATOR).asText(); + } + } else { + discriminator = null; + } + + List> objectChildren = new LinkedList<>(); + node.fields().forEachRemaining(entry -> { + if (entry.getValue().isObject()) { + objectChildren.add(entry); + } else if (entry.getValue().isArray()) { + entry.getValue().forEach(i -> { + if (i.isObject()) { + objectChildren.add(Map.entry(entry.getKey(), i)); + } + }); + } + }); + + objectChildren.forEach(entry -> + replaceDiscriminator(node, discriminator, entry.getKey(), (ObjectNode) entry.getValue()) + ); + } @Nonnull public static GenericAspect convertGenericAspect(@Nonnull io.datahubproject.openapi.generated.GenericAspect genericAspect, ObjectMapper objectMapper) { try { ObjectNode jsonTree = (ObjectNode) objectMapper.valueToTree(genericAspect).get("value"); - jsonTree.remove(DISCRIMINATOR); + replaceDiscriminator(jsonTree); String pretty = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonTree); - Matcher classTypeNameMatcher = CLASS_TYPE_NAME_PATTERN.matcher(pretty); - while (classTypeNameMatcher.find()) { - String classTypeName = classTypeNameMatcher.group("classTypeName"); - String content = classTypeNameMatcher.group("content"); - StringBuilder replacement = new StringBuilder(); - // Global Tags & Glossary Terms get used as both a union type and a non-union type, in the DataMap this means - // that it does not want the explicit class name if it is being used explicitly as a non-union type field on an aspect - if (!GLOBAL_TAGS.equals(classTypeName) && !GLOSSARY_TERMS.equals(classTypeName)) { - String pegasusClassName = PEGASUS_TYPE_MAP.get(classTypeName).getName(); - replacement.append("\"").append(pegasusClassName).append("\" : {"); - - if (content.length() > 0) { - replacement.append(content); - } - replacement.append("}}"); - } else { - replacement.append(content) - .append("}"); - } - pretty = classTypeNameMatcher.replaceFirst(Matcher.quoteReplacement(replacement.toString())); - classTypeNameMatcher = CLASS_TYPE_NAME_PATTERN.matcher(pretty); - } return new GenericAspect().setContentType(genericAspect.getContentType()) .setValue(ByteString.copyString(pretty, UTF_8)); } catch (JsonProcessingException e) { @@ -297,16 +361,16 @@ public static Pair ingestProposal(com.linkedin.mxe.MetadataChan } public static MetadataChangeProposal mapToProposal(UpsertAspectRequest aspectRequest) { - MetadataChangeProposal metadataChangeProposal = new MetadataChangeProposal(); + MetadataChangeProposal.MetadataChangeProposalBuilder metadataChangeProposal = MetadataChangeProposal.builder(); io.datahubproject.openapi.generated.GenericAspect - genericAspect = new io.datahubproject.openapi.generated.GenericAspect() + genericAspect = io.datahubproject.openapi.generated.GenericAspect.builder() .value(aspectRequest.getAspect()) - .contentType(MediaType.APPLICATION_JSON_VALUE); + .contentType(MediaType.APPLICATION_JSON_VALUE).build(); io.datahubproject.openapi.generated.GenericAspect keyAspect = null; if (aspectRequest.getEntityKeyAspect() != null) { - keyAspect = new io.datahubproject.openapi.generated.GenericAspect() + keyAspect = io.datahubproject.openapi.generated.GenericAspect.builder() .contentType(MediaType.APPLICATION_JSON_VALUE) - .value(aspectRequest.getEntityKeyAspect()); + .value(aspectRequest.getEntityKeyAspect()).build(); } metadataChangeProposal.aspect(genericAspect) .changeType(io.datahubproject.openapi.generated.ChangeType.UPSERT) @@ -315,7 +379,7 @@ public static MetadataChangeProposal mapToProposal(UpsertAspectRequest aspectReq .entityUrn(aspectRequest.getEntityUrn()) .entityType(aspectRequest.getEntityType()); - return metadataChangeProposal; + return metadataChangeProposal.build(); } public static com.linkedin.mxe.MetadataChangeProposal mapToServiceProposal(MetadataChangeProposal metadataChangeProposal, @@ -385,7 +449,7 @@ public static UpsertAspectRequest createStatusRemoval(Urn urn, EntityService ent throw new IllegalArgumentException("Entity type is not valid for soft deletes: " + urn.getEntityType()); } return UpsertAspectRequest.builder() - .aspect(new Status().removed(true)) + .aspect(Status.builder().removed(true).build()) .entityUrn(urn.toString()) .entityType(urn.getEntityType()) .build(); diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/ReflectionCache.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/ReflectionCache.java new file mode 100644 index 0000000000000..12f7652aff587 --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/ReflectionCache.java @@ -0,0 +1,138 @@ +package io.datahubproject.openapi.util; + +import com.google.common.reflect.ClassPath; +import com.linkedin.util.Pair; +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Slf4j +@Builder +public class ReflectionCache { + private static final ConcurrentHashMap METHOD_CACHE = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap> CLASS_CACHE = new ConcurrentHashMap<>(); + + private final String basePackage; + private final Set subPackages; + @Builder.Default // appropriate for lombok + private final Function, String> getBuilderName = clazz -> + String.join("", clazz.getSimpleName(), "$", clazz.getSimpleName(), "Builder"); + + public static class ReflectionCacheBuilder { + public ReflectionCacheBuilder basePackage(String basePackage) { + return basePackage(basePackage, Set.of()); + } + + public ReflectionCacheBuilder basePackage(String basePackage, Set packageExclusions) { + this.basePackage = basePackage; + return subPackages(findSubPackages(basePackage, Optional.ofNullable(packageExclusions).orElse(Set.of()))); + } + + private ReflectionCacheBuilder subPackages(Set subPackages) { + this.subPackages = subPackages; + return this; + } + + private Set findSubPackages(String packageName, Set exclusions) { + try { + return ClassPath.from(getClass().getClassLoader()) + .getAllClasses() + .stream() + .filter(clazz -> exclusions.stream().noneMatch(excl -> clazz.getPackageName().startsWith(excl)) + && !clazz.getName().contains("$") && clazz.getName().startsWith(packageName)) + .map(ClassPath.ClassInfo::getPackageName) + .collect(Collectors.toSet()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public Method lookupMethod(Class clazz, String method, Class... parameters) { + if (clazz == null) { + return null; + } else { + return METHOD_CACHE.computeIfAbsent( + String.join("_", clazz.getName(), method), + key -> { + try { + log.debug("Lookup: " + clazz.getName() + " Method: " + method + " Parameters: " + Arrays.toString(parameters)); + return clazz.getDeclaredMethod(method, parameters); + } catch (NoSuchMethodException e) { + return null; + } + } + ); + } + } + + public Class lookupClass(String className, boolean searchSubclass) { + if (!searchSubclass) { + return lookupClass(className); + } else { + List subclasses = new LinkedList<>(); + subclasses.add(basePackage); + if (subPackages != null) { + subclasses.addAll(subPackages); + } + + for (String packageName : subclasses) { + try { + return cachedClassLookup(packageName, className); + } catch (Exception e) { + log.debug("Class not found {}.{} ... continuing search", packageName, className); + } + } + } + throw new ClassCastException(String.format("Could not locate %s in package %s", className, basePackage)); + } + + public Class lookupClass(String className) { + return cachedClassLookup(basePackage, className); + } + + private Class cachedClassLookup(String packageName, String className) { + return CLASS_CACHE.computeIfAbsent( + String.format("%s.%s", packageName, className), + key -> { + try { + log.debug("Lookup: " + key); + return Class.forName(key); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + ); + } + + /** + * Get builder instance and class + */ + public Pair, Object> getBuilder(Class toClazz) throws InvocationTargetException, IllegalAccessException { + Class toClazzBuilder = lookupClass(getBuilderName.apply(toClazz)); + return Pair.of(toClazzBuilder, lookupMethod(toClazz, "builder").invoke(null)); + } + + public Method lookupMethod(Pair, Object> builderPair, String method, Class... parameters) { + return lookupMethod(builderPair.getFirst(), method, parameters); + } + + public static String toLowerFirst(String s) { + return s.substring(0, 1).toLowerCase() + s.substring(1); + } + + public static String toUpperFirst(String s) { + return s.substring(0, 1).toUpperCase() + s.substring(1); + } +} diff --git a/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java b/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java index 229e71168557d..6c2ec108fe493 100644 --- a/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java +++ b/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java @@ -90,85 +90,86 @@ public void setup() public void testIngestDataset() { List datasetAspects = new ArrayList<>(); UpsertAspectRequest viewProperties = UpsertAspectRequest.builder() - .aspect(new ViewProperties() + .aspect(ViewProperties.builder() .viewLogic(S) .viewLanguage(S) - .materialized(true)) + .materialized(true).build()) .entityType(DATASET_ENTITY_NAME) .entityUrn(DATASET_URN) .build(); datasetAspects.add(viewProperties); UpsertAspectRequest subTypes = UpsertAspectRequest.builder() - .aspect(new SubTypes() - .typeNames(Collections.singletonList(S))) + .aspect(SubTypes.builder() + .typeNames(Collections.singletonList(S)).build()) .entityType(DATASET_ENTITY_NAME) - .entityKeyAspect(new DatasetKey() + .entityKeyAspect(DatasetKey.builder() .name("name") .platform(DATA_PLATFORM_URN) - .origin(FabricType.PROD)) + .origin(FabricType.PROD).build()) .build(); datasetAspects.add(subTypes); UpsertAspectRequest datasetProfile = UpsertAspectRequest.builder() - .aspect(new DatasetProfile().timestampMillis(0L).addFieldProfilesItem( - new DatasetFieldProfile() - .fieldPath(S) - .histogram(new Histogram() - .boundaries(Collections.singletonList(S)))) - ) - .entityType(DATASET_ENTITY_NAME) - .entityKeyAspect(new DatasetKey() - .name("name") - .platform(DATA_PLATFORM_URN) - .origin(FabricType.PROD)) - .build(); + .aspect(DatasetProfile.builder().build().timestampMillis(0L).addFieldProfilesItem( + DatasetFieldProfile.builder() + .fieldPath(S) + .histogram(Histogram.builder() + .boundaries(Collections.singletonList(S)).build()).build() + ) + ) + .entityType(DATASET_ENTITY_NAME) + .entityKeyAspect(DatasetKey.builder() + .name("name") + .platform(DATA_PLATFORM_URN) + .origin(FabricType.PROD).build()) + .build(); datasetAspects.add(datasetProfile); UpsertAspectRequest schemaMetadata = UpsertAspectRequest.builder() - .aspect(new SchemaMetadata() + .aspect(SchemaMetadata.builder() .schemaName(S) .dataset(DATASET_URN) .platform(DATA_PLATFORM_URN) .hash(S) .version(0L) - .platformSchema(new MySqlDDL().tableSchema(S)) - .fields(Collections.singletonList(new SchemaField() + .platformSchema(MySqlDDL.builder().tableSchema(S).build()) + .fields(Collections.singletonList(SchemaField.builder() .fieldPath(S) .nativeDataType(S) - .type(new SchemaFieldDataType().type(new StringType())) + .type(SchemaFieldDataType.builder().type(StringType.builder().build()).build()) .description(S) - .globalTags(new GlobalTags() - .tags(Collections.singletonList(new TagAssociation() - .tag(TAG_URN)))) - .glossaryTerms(new GlossaryTerms() - .terms(Collections.singletonList(new GlossaryTermAssociation() - .urn(GLOSSARY_TERM_URN))) - .auditStamp(new AuditStamp() + .globalTags(GlobalTags.builder() + .tags(Collections.singletonList(TagAssociation.builder() + .tag(TAG_URN).build())).build()) + .glossaryTerms(GlossaryTerms.builder() + .terms(Collections.singletonList(GlossaryTermAssociation.builder() + .urn(GLOSSARY_TERM_URN).build())) + .auditStamp(AuditStamp.builder() .time(0L) - .actor(CORPUSER_URN))) + .actor(CORPUSER_URN).build()).build()).build() ) - )) + ).build()) .entityType(DATASET_ENTITY_NAME) - .entityKeyAspect(new DatasetKey() + .entityKeyAspect(DatasetKey.builder() .name("name") .platform(DATA_PLATFORM_URN) - .origin(FabricType.PROD)) + .origin(FabricType.PROD).build()) .build(); datasetAspects.add(schemaMetadata); UpsertAspectRequest glossaryTerms = UpsertAspectRequest.builder() - .aspect(new GlossaryTerms() - .terms(Collections.singletonList(new GlossaryTermAssociation() - .urn(GLOSSARY_TERM_URN))) - .auditStamp(new AuditStamp() + .aspect(GlossaryTerms.builder() + .terms(Collections.singletonList(GlossaryTermAssociation.builder() + .urn(GLOSSARY_TERM_URN).build())) + .auditStamp(AuditStamp.builder() .time(0L) - .actor(CORPUSER_URN))) + .actor(CORPUSER_URN).build()).build()) .entityType(DATASET_ENTITY_NAME) - .entityKeyAspect(new DatasetKey() + .entityKeyAspect(DatasetKey.builder() .name("name") .platform(DATA_PLATFORM_URN) - .origin(FabricType.PROD)) + .origin(FabricType.PROD).build()) .build(); datasetAspects.add(glossaryTerms); diff --git a/metadata-service/schema-registry-api/build.gradle b/metadata-service/schema-registry-api/build.gradle index 7bf1e558c8906..290126836eb4a 100644 --- a/metadata-service/schema-registry-api/build.gradle +++ b/metadata-service/schema-registry-api/build.gradle @@ -20,7 +20,7 @@ dependencies { // End of dependencies implementation externalDependency.swaggerAnnotations - swaggerCodegen 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.46' + swaggerCodegen externalDependency.swaggerCli testImplementation externalDependency.assertJ } diff --git a/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/config/SpringWebSchemaRegistryConfig.java b/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/config/SpringWebSchemaRegistryConfig.java index c67d6d8f6fe17..d217d501630e3 100644 --- a/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/config/SpringWebSchemaRegistryConfig.java +++ b/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/config/SpringWebSchemaRegistryConfig.java @@ -1,6 +1,7 @@ package io.datahubproject.openapi.schema.registry.config; import io.swagger.v3.oas.annotations.OpenAPIDefinition; +import io.swagger.v3.oas.annotations.info.Info; import io.swagger.v3.oas.annotations.servers.Server; import java.util.List; import org.springframework.context.annotation.Configuration; @@ -14,7 +15,8 @@ @EnableWebMvc -@OpenAPIDefinition(servers = {@Server(url = "/schema-registry/", description = "Schema Registry Server URL")}) +@OpenAPIDefinition(info = @Info(title = "DataHub OpenAPI", version = "1.0.0"), + servers = {@Server(url = "/schema-registry/", description = "Schema Registry Server URL")}) @Configuration public class SpringWebSchemaRegistryConfig implements WebMvcConfigurer { diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java index 86043f4b7cd27..30cfc2e0288bd 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java @@ -35,6 +35,14 @@ public interface EntityService { + /** + * Just whether the entity/aspect exists + * @param urn urn for the entity + * @param aspectName aspect for the entity + * @return exists or not + */ + Boolean exists(Urn urn, String aspectName); + /** * Retrieves the latest aspects corresponding to a batch of {@link Urn}s based on a provided * set of aspect names. @@ -206,7 +214,7 @@ void ingestEntities(@Nonnull final List entities, @Nonnull final AuditSt @Nonnull final List systemMetadata); @Deprecated - void ingestEntity(Entity entity, AuditStamp auditStamp); + SystemMetadata ingestEntity(Entity entity, AuditStamp auditStamp); @Deprecated void ingestEntity(@Nonnull Entity entity, @Nonnull AuditStamp auditStamp, diff --git a/metadata-service/war/build.gradle b/metadata-service/war/build.gradle index 3bd2695c927a7..122c2b9d5357b 100644 --- a/metadata-service/war/build.gradle +++ b/metadata-service/war/build.gradle @@ -19,6 +19,8 @@ dependencies { runtimeOnly project(':metadata-service:graphql-servlet-impl') runtimeOnly project(':metadata-service:health-servlet') runtimeOnly project(':metadata-service:openapi-servlet') + runtimeOnly project(':metadata-service:openapi-entity-servlet') + runtimeOnly project(':metadata-service:openapi-analytics-servlet') runtimeOnly project(':metadata-service:schema-registry-servlet') runtimeOnly project(':metadata-jobs:mce-consumer') runtimeOnly project(':metadata-jobs:mae-consumer') diff --git a/metadata-service/war/src/main/resources/boot/policies.json b/metadata-service/war/src/main/resources/boot/policies.json index 3cda0269b79f1..410596cc30cbe 100644 --- a/metadata-service/war/src/main/resources/boot/policies.json +++ b/metadata-service/war/src/main/resources/boot/policies.json @@ -31,7 +31,8 @@ "GET_ES_TASK_STATUS_PRIVILEGE", "SET_WRITEABLE_PRIVILEGE", "APPLY_RETENTION_PRIVILEGE", - "MANAGE_GLOBAL_OWNERSHIP_TYPES" + "MANAGE_GLOBAL_OWNERSHIP_TYPES", + "GET_ANALYTICS_PRIVILEGE" ], "displayName":"Root User - All Platform Privileges", "description":"Grants full platform privileges to root datahub super user.", @@ -204,7 +205,8 @@ "GET_ES_TASK_STATUS_PRIVILEGE", "SET_WRITEABLE_PRIVILEGE", "APPLY_RETENTION_PRIVILEGE", - "MANAGE_GLOBAL_OWNERSHIP_TYPES" + "MANAGE_GLOBAL_OWNERSHIP_TYPES", + "GET_ANALYTICS_PRIVILEGE" ], "displayName":"Admins - Platform Policy", "description":"Admins have all platform privileges.", diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java index 0b0d462f079bf..df960808d8a41 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java @@ -47,6 +47,11 @@ public class PoliciesConfig { "View Analytics", "View the DataHub analytics dashboard."); + public static final Privilege GET_ANALYTICS_PRIVILEGE = Privilege.of( + "GET_ANALYTICS_PRIVILEGE", + "Analytics API access", + "API read access to raw analytics data."); + public static final Privilege GENERATE_PERSONAL_ACCESS_TOKENS_PRIVILEGE = Privilege.of( "GENERATE_PERSONAL_ACCESS_TOKENS", "Generate Personal Access Tokens", @@ -117,6 +122,7 @@ public class PoliciesConfig { MANAGE_POLICIES_PRIVILEGE, MANAGE_USERS_AND_GROUPS_PRIVILEGE, VIEW_ANALYTICS_PRIVILEGE, + GET_ANALYTICS_PRIVILEGE, MANAGE_DOMAINS_PRIVILEGE, MANAGE_GLOBAL_ANNOUNCEMENTS_PRIVILEGE, MANAGE_INGESTION_PRIVILEGE, diff --git a/settings.gradle b/settings.gradle index 270672e929e88..d6777b07b3fb3 100644 --- a/settings.gradle +++ b/settings.gradle @@ -14,6 +14,9 @@ include 'metadata-service:restli-client' include 'metadata-service:restli-servlet-impl' include 'metadata-service:graphql-servlet-impl' include 'metadata-service:openapi-servlet' +include 'metadata-service:openapi-entity-servlet' +include 'metadata-service:openapi-entity-servlet:generators' +include 'metadata-service:openapi-analytics-servlet' include 'metadata-service:plugin' include 'metadata-service:plugin:src:test:sample-test-plugins' include 'metadata-dao-impl:kafka-producer'