Skip to content

Commit

Permalink
feat(SDK) Add java SDK for structuredProperty entity PATCH + CRUD exa…
Browse files Browse the repository at this point in the history
…mples (#10823)
  • Loading branch information
chriscollins3456 authored Jul 2, 2024
1 parent 5e4a3af commit a7f4b71
Show file tree
Hide file tree
Showing 5 changed files with 458 additions and 43 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,11 @@
import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance;
import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME;

import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.ValueNode;
import com.linkedin.common.urn.Urn;
import com.linkedin.metadata.aspect.patch.PatchOperationType;
import java.util.List;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.commons.lang3.tuple.ImmutableTriple;
Expand All @@ -17,8 +16,8 @@ public class StructuredPropertiesPatchBuilder
extends AbstractMultiFieldPatchBuilder<StructuredPropertiesPatchBuilder> {

private static final String BASE_PATH = "/properties";
private static final String URN_KEY = "urn";
private static final String CONTEXT_KEY = "context";
private static final String URN_KEY = "propertyUrn";
private static final String VALUES_KEY = "values";

/**
* Remove a property from a structured properties aspect. If the property doesn't exist, this is a
Expand All @@ -34,63 +33,77 @@ public StructuredPropertiesPatchBuilder removeProperty(Urn propertyUrn) {
return this;
}

public StructuredPropertiesPatchBuilder setProperty(
@Nonnull Urn propertyUrn, @Nullable List<Object> propertyValues) {
propertyValues.stream()
.map(
propertyValue ->
propertyValue instanceof Integer
? this.setProperty(propertyUrn, (Integer) propertyValue)
: this.setProperty(propertyUrn, String.valueOf(propertyValue)))
.collect(Collectors.toList());
return this;
}

public StructuredPropertiesPatchBuilder setProperty(
public StructuredPropertiesPatchBuilder setNumberProperty(
@Nonnull Urn propertyUrn, @Nullable Integer propertyValue) {
ValueNode propertyValueNode = instance.numberNode((Integer) propertyValue);
ObjectNode value = instance.objectNode();
value.put(URN_KEY, propertyUrn.toString());
ObjectNode newProperty = instance.objectNode();
newProperty.put(URN_KEY, propertyUrn.toString());

ArrayNode valuesNode = instance.arrayNode();
ObjectNode propertyValueNode = instance.objectNode();
propertyValueNode.set("double", instance.numberNode(propertyValue));
valuesNode.add(propertyValueNode);
newProperty.set(VALUES_KEY, valuesNode);

pathValues.add(
ImmutableTriple.of(
PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, propertyValueNode));
PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, newProperty));
return this;
}

public StructuredPropertiesPatchBuilder setProperty(
@Nonnull Urn propertyUrn, @Nullable String propertyValue) {
ValueNode propertyValueNode = instance.textNode(String.valueOf(propertyValue));
ObjectNode value = instance.objectNode();
value.put(URN_KEY, propertyUrn.toString());
public StructuredPropertiesPatchBuilder setNumberProperty(
@Nonnull Urn propertyUrn, @Nonnull List<Integer> propertyValues) {
ObjectNode newProperty = instance.objectNode();
newProperty.put(URN_KEY, propertyUrn.toString());

ArrayNode valuesNode = instance.arrayNode();
propertyValues.forEach(
propertyValue -> {
ObjectNode propertyValueNode = instance.objectNode();
propertyValueNode.set("double", instance.numberNode(propertyValue));
valuesNode.add(propertyValueNode);
});
newProperty.set(VALUES_KEY, valuesNode);

pathValues.add(
ImmutableTriple.of(
PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, propertyValueNode));
PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, newProperty));
return this;
}

public StructuredPropertiesPatchBuilder addProperty(
@Nonnull Urn propertyUrn, @Nullable Integer propertyValue) {
ValueNode propertyValueNode = instance.numberNode((Integer) propertyValue);
ObjectNode value = instance.objectNode();
value.put(URN_KEY, propertyUrn.toString());
public StructuredPropertiesPatchBuilder setStringProperty(
@Nonnull Urn propertyUrn, @Nullable String propertyValue) {
ObjectNode newProperty = instance.objectNode();
newProperty.put(URN_KEY, propertyUrn.toString());

ArrayNode valuesNode = instance.arrayNode();
ObjectNode propertyValueNode = instance.objectNode();
propertyValueNode.set("string", instance.textNode(propertyValue));
valuesNode.add(propertyValueNode);
newProperty.set(VALUES_KEY, valuesNode);

pathValues.add(
ImmutableTriple.of(
PatchOperationType.ADD.getValue(),
BASE_PATH + "/" + propertyUrn + "/" + String.valueOf(propertyValue),
propertyValueNode));
PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, newProperty));
return this;
}

public StructuredPropertiesPatchBuilder addProperty(
@Nonnull Urn propertyUrn, @Nullable String propertyValue) {
ValueNode propertyValueNode = instance.textNode(String.valueOf(propertyValue));
ObjectNode value = instance.objectNode();
value.put(URN_KEY, propertyUrn.toString());
public StructuredPropertiesPatchBuilder setStringProperty(
@Nonnull Urn propertyUrn, @Nonnull List<String> propertyValues) {
ObjectNode newProperty = instance.objectNode();
newProperty.put(URN_KEY, propertyUrn.toString());

ArrayNode valuesNode = instance.arrayNode();
propertyValues.forEach(
propertyValue -> {
ObjectNode propertyValueNode = instance.objectNode();
propertyValueNode.set("string", instance.textNode(propertyValue));
valuesNode.add(propertyValueNode);
});
newProperty.set(VALUES_KEY, valuesNode);

pathValues.add(
ImmutableTriple.of(
PatchOperationType.ADD.getValue(),
BASE_PATH + "/" + propertyUrn + "/" + String.valueOf(propertyValue),
propertyValueNode));
PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, newProperty));
return this;
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
package com.linkedin.metadata.aspect.patch.builder;

import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance;
import static com.linkedin.metadata.Constants.*;

import com.datahub.util.RecordUtils;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.linkedin.data.template.StringArrayMap;
import com.linkedin.metadata.aspect.patch.PatchOperationType;
import com.linkedin.structured.PropertyCardinality;
import com.linkedin.structured.PropertyValue;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.commons.lang3.tuple.ImmutableTriple;

public class StructuredPropertyDefinitionPatchBuilder
extends AbstractMultiFieldPatchBuilder<StructuredPropertyDefinitionPatchBuilder> {

public static final String PATH_DELIM = "/";
public static final String QUALIFIED_NAME_FIELD = "qualifiedName";
public static final String DISPLAY_NAME_FIELD = "displayName";
public static final String VALUE_TYPE_FIELD = "valueType";
public static final String TYPE_QUALIFIER_FIELD = "typeQualifier";
public static final String ALLOWED_VALUES_FIELD = "allowedValues";
public static final String CARDINALITY_FIELD = "cardinality";
public static final String ENTITY_TYPES_FIELD = "entityTypes";
public static final String DESCRIPTION_FIELD = "description";
public static final String IMMUTABLE_FIELD = "immutable";

// can only be used when creating a new structured property
public StructuredPropertyDefinitionPatchBuilder setQualifiedName(@Nonnull String name) {
this.pathValues.add(
ImmutableTriple.of(
PatchOperationType.ADD.getValue(),
PATH_DELIM + QUALIFIED_NAME_FIELD,
instance.textNode(name)));
return this;
}

public StructuredPropertyDefinitionPatchBuilder setDisplayName(@Nonnull String displayName) {
this.pathValues.add(
ImmutableTriple.of(
PatchOperationType.ADD.getValue(),
PATH_DELIM + DISPLAY_NAME_FIELD,
instance.textNode(displayName)));
return this;
}

public StructuredPropertyDefinitionPatchBuilder setValueType(@Nonnull String valueTypeUrn) {
this.pathValues.add(
ImmutableTriple.of(
PatchOperationType.ADD.getValue(),
PATH_DELIM + VALUE_TYPE_FIELD,
instance.textNode(valueTypeUrn)));
return this;
}

// can only be used when creating a new structured property
public StructuredPropertyDefinitionPatchBuilder setTypeQualifier(
@Nonnull StringArrayMap typeQualifier) {
ObjectNode value = instance.objectNode();
typeQualifier.forEach(
(key, values) -> {
ArrayNode valuesNode = instance.arrayNode();
values.forEach(valuesNode::add);
value.set(key, valuesNode);
});
this.pathValues.add(
ImmutableTriple.of(
PatchOperationType.ADD.getValue(), PATH_DELIM + TYPE_QUALIFIER_FIELD, value));
return this;
}

public StructuredPropertyDefinitionPatchBuilder addAllowedValue(
@Nonnull PropertyValue propertyValue) {
try {
ObjectNode valueNode =
(ObjectNode) new ObjectMapper().readTree(RecordUtils.toJsonString(propertyValue));
this.pathValues.add(
ImmutableTriple.of(
PatchOperationType.ADD.getValue(),
PATH_DELIM + ALLOWED_VALUES_FIELD + PATH_DELIM + propertyValue.getValue(),
valueNode));
return this;
} catch (JsonProcessingException e) {
throw new IllegalArgumentException(
"Failed to add allowed value, failed to parse provided aspect json.", e);
}
}

public StructuredPropertyDefinitionPatchBuilder setCardinality(
@Nonnull PropertyCardinality cardinality) {
this.pathValues.add(
ImmutableTriple.of(
PatchOperationType.ADD.getValue(),
PATH_DELIM + CARDINALITY_FIELD,
instance.textNode(cardinality.toString())));
return this;
}

public StructuredPropertyDefinitionPatchBuilder addEntityType(@Nonnull String entityTypeUrn) {
this.pathValues.add(
ImmutableTriple.of(
PatchOperationType.ADD.getValue(),
PATH_DELIM + ENTITY_TYPES_FIELD + PATH_DELIM + entityTypeUrn,
instance.textNode(entityTypeUrn)));
return this;
}

public StructuredPropertyDefinitionPatchBuilder setDescription(@Nullable String description) {
if (description == null) {
this.pathValues.add(
ImmutableTriple.of(
PatchOperationType.REMOVE.getValue(), PATH_DELIM + DESCRIPTION_FIELD, null));
} else {
this.pathValues.add(
ImmutableTriple.of(
PatchOperationType.ADD.getValue(),
PATH_DELIM + DESCRIPTION_FIELD,
instance.textNode(description)));
}
return this;
}

public StructuredPropertyDefinitionPatchBuilder setImmutable(boolean immutable) {
this.pathValues.add(
ImmutableTriple.of(
PatchOperationType.ADD.getValue(),
PATH_DELIM + IMMUTABLE_FIELD,
instance.booleanNode(immutable)));
return this;
}

@Override
protected String getAspectName() {
return STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME;
}

@Override
protected String getEntityType() {
return STRUCTURED_PROPERTY_ENTITY_NAME;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
import com.linkedin.common.urn.TagUrn;
import com.linkedin.common.urn.Urn;
import com.linkedin.common.urn.UrnUtils;
import com.linkedin.data.template.StringArray;
import com.linkedin.data.template.StringArrayMap;
import com.linkedin.dataset.DatasetLineageType;
import com.linkedin.form.FormPrompt;
import com.linkedin.form.FormPromptType;
Expand All @@ -30,9 +32,14 @@
import com.linkedin.metadata.aspect.patch.builder.EditableSchemaMetadataPatchBuilder;
import com.linkedin.metadata.aspect.patch.builder.FormInfoPatchBuilder;
import com.linkedin.metadata.aspect.patch.builder.OwnershipPatchBuilder;
import com.linkedin.metadata.aspect.patch.builder.StructuredPropertiesPatchBuilder;
import com.linkedin.metadata.aspect.patch.builder.StructuredPropertyDefinitionPatchBuilder;
import com.linkedin.metadata.aspect.patch.builder.UpstreamLineagePatchBuilder;
import com.linkedin.metadata.graph.LineageDirection;
import com.linkedin.mxe.MetadataChangeProposal;
import com.linkedin.structured.PrimitivePropertyValue;
import com.linkedin.structured.PropertyCardinality;
import com.linkedin.structured.PropertyValue;
import datahub.client.MetadataWriteResponse;
import datahub.client.file.FileEmitter;
import datahub.client.file.FileEmitterConfig;
Expand Down Expand Up @@ -647,6 +654,49 @@ public void testLocalDashboardInfoRemove() {
}
}

@Test
@Ignore
public void testLocalStructuredPropertyDefinitionAdd() {
RestEmitter restEmitter = new RestEmitter(RestEmitterConfig.builder().build());
try {
StringArrayMap typeQualifier = new StringArrayMap();
typeQualifier.put(
"allowedTypes",
new StringArray(
"urn:li:entityType:datahub.corpuser", "urn:li:entityType:datahub.corpGroup"));
PropertyValue propertyValue1 = new PropertyValue();
PrimitivePropertyValue value1 = new PrimitivePropertyValue();
value1.setString("test value 1");
propertyValue1.setValue(value1);
PropertyValue propertyValue2 = new PropertyValue();
PrimitivePropertyValue value2 = new PrimitivePropertyValue();
value2.setString("test value 2");
propertyValue2.setValue(value2);

MetadataChangeProposal structuredPropertyDefinitionPatch =
new StructuredPropertyDefinitionPatchBuilder()
.urn(UrnUtils.getUrn("urn:li:structuredProperty:123456"))
.setQualifiedName("test.testing.123")
.setDisplayName("Test Display Name")
.setValueType("urn:li:dataType:datahub.urn")
.setTypeQualifier(typeQualifier)
.addAllowedValue(propertyValue1)
.addAllowedValue(propertyValue2)
.setCardinality(PropertyCardinality.MULTIPLE)
.addEntityType("urn:li:entityType:datahub.dataFlow")
.setDescription("test description")
.setImmutable(true)
.build();

Future<MetadataWriteResponse> response = restEmitter.emit(structuredPropertyDefinitionPatch);

System.out.println(response.get().getResponseContent());

} catch (IOException | ExecutionException | InterruptedException e) {
System.out.println(Arrays.asList(e.getStackTrace()));
}
}

@Test
@Ignore
public void testLocalFormInfoAdd() {
Expand Down Expand Up @@ -690,6 +740,31 @@ public void testLocalFormInfoAdd() {
}
}

@Test
@Ignore
public void testLocalStructuredPropertiesUpdate() {
try {
MetadataChangeProposal mcp =
new StructuredPropertiesPatchBuilder()
.urn(
UrnUtils.getUrn(
"urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)"))
.setNumberProperty(
UrnUtils.getUrn(
"urn:li:structuredProperty:io.acryl.dataManagement.replicationSLA"),
3456)
.build();

String token = "";
RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:8080").token(token));
Future<MetadataWriteResponse> response = emitter.emit(mcp, null);
System.out.println(response.get().getResponseContent());

} catch (IOException | ExecutionException | InterruptedException e) {
System.out.println(Arrays.asList(e.getStackTrace()));
}
}

@Test
@Ignore
public void testLocalFormInfoRemove() {
Expand Down
Loading

0 comments on commit a7f4b71

Please sign in to comment.