From 59a36c954104084821496283d13b0f0bd10264e1 Mon Sep 17 00:00:00 2001 From: George Chen Date: Thu, 24 Oct 2024 14:22:22 -0500 Subject: [PATCH 1/6] MAINT: conditional required schema Signed-off-by: George Chen --- .../annotations/ConditionalRequired.java | 23 +++++ .../schemas/JsonSchemaConverter.java | 63 +++++++++++++ .../schemas/JsonSchemaConverterTest.java | 88 +++++++++++++++++++ 3 files changed, 174 insertions(+) create mode 100644 data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/ConditionalRequired.java diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/ConditionalRequired.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/ConditionalRequired.java new file mode 100644 index 0000000000..10627d4250 --- /dev/null +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/ConditionalRequired.java @@ -0,0 +1,23 @@ +package org.opensearch.dataprepper.model.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ ElementType.FIELD, ElementType.TYPE }) +@Retention(RetentionPolicy.RUNTIME) +public @interface ConditionalRequired { + IfThenElse[] value(); + + @interface IfThenElse { + SchemaProperty[] ifFulfilled(); + SchemaProperty[] thenExpect(); + SchemaProperty[] elseExpect() default {}; + } + + @interface SchemaProperty { + String field(); + String value() default ""; + } +} diff --git a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java index 78e79c9fa0..247b6f3ca2 100644 --- a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java +++ b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java @@ -4,17 +4,21 @@ import com.fasterxml.classmate.types.ResolvedObjectType; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.github.victools.jsonschema.generator.FieldScope; import com.github.victools.jsonschema.generator.Module; +import com.github.victools.jsonschema.generator.Option; import com.github.victools.jsonschema.generator.OptionPreset; import com.github.victools.jsonschema.generator.SchemaGenerator; import com.github.victools.jsonschema.generator.SchemaGeneratorConfig; import com.github.victools.jsonschema.generator.SchemaGeneratorConfigBuilder; import com.github.victools.jsonschema.generator.SchemaGeneratorConfigPart; import com.github.victools.jsonschema.generator.SchemaGeneratorGeneralConfigPart; +import com.github.victools.jsonschema.generator.SchemaKeyword; import com.github.victools.jsonschema.generator.SchemaVersion; import org.opensearch.dataprepper.model.annotations.AlsoRequired; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired; import org.opensearch.dataprepper.model.event.EventKey; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; @@ -25,6 +29,7 @@ import java.util.Collections; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -52,6 +57,7 @@ public ObjectNode convertIntoJsonSchema( resolveDefaultValueFromJsonProperty(scopeSchemaGeneratorConfigPart); resolveDependentRequiresFields(scopeSchemaGeneratorConfigPart); overrideDataPrepperPluginTypeAttribute(configBuilder.forTypesInGeneral(), schemaVersion, optionPreset); + overrideTypeAttributeWithConditionalRequired(configBuilder.forTypesInGeneral()); resolveDataPrepperTypes(scopeSchemaGeneratorConfigPart); scopeSchemaGeneratorConfigPart.withInstanceAttributeOverride(new ExampleValuesInstanceAttributeOverride()); @@ -107,6 +113,63 @@ private void overrideDataPrepperPluginTypeAttribute( }); } + private void overrideTypeAttributeWithConditionalRequired( + final SchemaGeneratorGeneralConfigPart schemaGeneratorGeneralConfigPart) { + schemaGeneratorGeneralConfigPart.withTypeAttributeOverride((node, scope, context) -> { + final ConditionalRequired conditionalRequiredAnnotation = scope.getContext() + .getTypeAnnotationConsideringHierarchy(scope.getType(), ConditionalRequired.class); + if (conditionalRequiredAnnotation != null) { + final SchemaGeneratorConfig config = context.getGeneratorConfig(); + final ArrayNode ifThenElseArrayNode = node.putArray(config.getKeyword(SchemaKeyword.TAG_ALLOF)); + Arrays.asList(conditionalRequiredAnnotation.value()).forEach(ifThenElse -> { + ObjectNode ifThenElseNode = config.createObjectNode(); + final ObjectNode ifObjectNode = constructIfObjectNode(config, ifThenElse.ifFulfilled()); + ifThenElseNode.set(config.getKeyword(SchemaKeyword.TAG_IF), ifObjectNode); + final ObjectNode thenObjectNode = constructExpectObjectNode(config, ifThenElse.thenExpect()); + ifThenElseNode.set(config.getKeyword(SchemaKeyword.TAG_THEN), thenObjectNode); + final ObjectNode elseObjectNode = constructExpectObjectNode(config, ifThenElse.elseExpect()); + if (!elseObjectNode.isEmpty()) { + ifThenElseNode.set(config.getKeyword(SchemaKeyword.TAG_ELSE), elseObjectNode); + } + ifThenElseArrayNode.add(ifThenElseNode); + }); + } + }); + } + + private ObjectNode constructIfObjectNode(final SchemaGeneratorConfig config, + final ConditionalRequired.SchemaProperty[] schemaProperties) { + final ObjectNode ifObjectNode = config.createObjectNode(); + final ObjectNode ifPropertiesNode = ifObjectNode.putObject(config.getKeyword(SchemaKeyword.TAG_PROPERTIES)); + Arrays.asList(schemaProperties).forEach(schemaProperty -> { + ifPropertiesNode.putObject(schemaProperty.field()).put( + config.getKeyword(SchemaKeyword.TAG_CONST), schemaProperty.value()); + }); + return ifObjectNode; + } + + private ObjectNode constructExpectObjectNode(final SchemaGeneratorConfig config, + final ConditionalRequired.SchemaProperty[] schemaProperties) { + final ObjectNode expectObjectNode = config.createObjectNode(); + final ObjectNode expectPropertiesNode = config.createObjectNode(); + final ArrayNode expectRequiredNode = config.createArrayNode(); + Arrays.asList(schemaProperties).forEach(schemaProperty -> { + if (!Objects.equals(schemaProperty.value(), "")) { + expectPropertiesNode.putObject(schemaProperty.field()).put( + config.getKeyword(SchemaKeyword.TAG_CONST), schemaProperty.value()); + } else { + expectRequiredNode.add(schemaProperty.field()); + } + }); + if (!expectPropertiesNode.isEmpty()) { + expectObjectNode.set(config.getKeyword(SchemaKeyword.TAG_PROPERTIES), expectPropertiesNode); + } + if (!expectRequiredNode.isEmpty()) { + expectObjectNode.set(config.getKeyword(SchemaKeyword.TAG_REQUIRED), expectRequiredNode); + } + return expectObjectNode; + } + private void resolveDefaultValueFromJsonProperty( final SchemaGeneratorConfigPart scopeSchemaGeneratorConfigPart) { scopeSchemaGeneratorConfigPart.withDefaultResolver(field -> { diff --git a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java index 7642e93e90..0ab6247a42 100644 --- a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java +++ b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java @@ -12,6 +12,7 @@ import com.github.victools.jsonschema.generator.SchemaVersion; import org.junit.jupiter.api.Test; import org.opensearch.dataprepper.model.annotations.AlsoRequired; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired; import org.opensearch.dataprepper.model.event.EventKey; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -95,6 +96,91 @@ void testConvertIntoJsonSchemaWithEventKey() throws JsonProcessingException { assertThat(propertiesNode.get("testAttributeEventKey").get("type"), is(equalTo(TextNode.valueOf("string")))); } + @Test + void testConvertIntoJsonSchemaWithConditionalRequired() throws JsonProcessingException { + final JsonSchemaConverter jsonSchemaConverter = createObjectUnderTest(Collections.emptyList(), pluginProvider); + final ObjectNode jsonSchemaNode = jsonSchemaConverter.convertIntoJsonSchema( + SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfig.class); + final JsonNode allOfNode = jsonSchemaNode.at("/allOf"); + assertThat(allOfNode, instanceOf(ArrayNode.class)); + assertThat(allOfNode.size(), equalTo(2)); + + final JsonNode ifThenElseNode1 = allOfNode.get(0); + assertThat(ifThenElseNode1.has("else"), is(false)); + final JsonNode ifNode1 = ifThenElseNode1.at("/if"); + assertThat(ifNode1, instanceOf(ObjectNode.class)); + final JsonNode ifPropertiesNode1 = ifNode1.at("/properties"); + assertThat(ifPropertiesNode1, instanceOf(ObjectNode.class)); + final JsonNode attributeNode1 = ifPropertiesNode1.at("/test_mutually_exclusive_attribute_a"); + assertThat(attributeNode1, instanceOf(ObjectNode.class)); + final JsonNode thenNode1 = ifThenElseNode1.at("/then"); + assertThat(thenNode1, instanceOf(ObjectNode.class)); + assertThat(thenNode1.has("properties"), is(false)); + final JsonNode thenRequiredNode1 = thenNode1.at("/required"); + assertThat(thenRequiredNode1, instanceOf(ArrayNode.class)); + assertThat(thenRequiredNode1.isEmpty(), is(false)); + + final JsonNode ifThenElseNode2 = allOfNode.get(1); + final JsonNode ifNode2 = ifThenElseNode2.at("/if"); + assertThat(ifNode2, instanceOf(ObjectNode.class)); + final JsonNode ifPropertiesNode2 = ifNode2.at("/properties"); + assertThat(ifPropertiesNode2, instanceOf(ObjectNode.class)); + final JsonNode ifAttributeNode2 = ifPropertiesNode2.at("/test_mutually_exclusive_attribute_a"); + assertThat(ifAttributeNode2, instanceOf(ObjectNode.class)); + final JsonNode thenNode2 = ifThenElseNode2.at("/then"); + assertThat(thenNode2, instanceOf(ObjectNode.class)); + assertThat(thenNode2.has("required"), is(false)); + final JsonNode thenPropertiesNode2 = thenNode2.at("/properties"); + assertThat(thenPropertiesNode2, instanceOf(ObjectNode.class)); + assertThat(thenPropertiesNode2.isEmpty(), is(false)); + final JsonNode thenAttributeNode2 = thenPropertiesNode2.at("/test_mutually_exclusive_attribute_c"); + assertThat(thenAttributeNode2, instanceOf(ObjectNode.class)); + final JsonNode elseNode2 = ifThenElseNode2.at("/else"); + assertThat(elseNode2, instanceOf(ObjectNode.class)); + assertThat(elseNode2.has("required"), is(false)); + final JsonNode elsePropertiesNode2 = elseNode2.at("/properties"); + assertThat(elsePropertiesNode2, instanceOf(ObjectNode.class)); + assertThat(elsePropertiesNode2.isEmpty(), is(false)); + final JsonNode elseAttributeNode2 = thenPropertiesNode2.at("/test_mutually_exclusive_attribute_c"); + assertThat(elseAttributeNode2, instanceOf(ObjectNode.class)); +// assertThat(propertiesNode.has("testAttributeEventKey"), is(equalTo(true))); +// assertThat(propertiesNode.get("testAttributeEventKey"), is(notNullValue())); +// assertThat(propertiesNode.get("testAttributeEventKey").get("type"), is(equalTo(TextNode.valueOf("string")))); + } + + @ConditionalRequired(value = { + @ConditionalRequired.IfThenElse( + ifFulfilled = { + @ConditionalRequired.SchemaProperty( + field = "test_mutually_exclusive_attribute_a", + value = "null") + }, + thenExpect = { + @ConditionalRequired.SchemaProperty( + field = "test_mutually_exclusive_attribute_b" + ) + } + ), + @ConditionalRequired.IfThenElse( + ifFulfilled = { + @ConditionalRequired.SchemaProperty( + field = "test_mutually_exclusive_attribute_a", + value = "null") + }, + thenExpect = { + @ConditionalRequired.SchemaProperty( + field = "test_mutually_exclusive_attribute_c", + value = "\"option1\"" + ) + }, + elseExpect = { + @ConditionalRequired.SchemaProperty( + field = "test_mutually_exclusive_attribute_c", + value = "\"option2\"" + ) + } + ) + }) @JsonClassDescription("test config") static class TestConfig { private String testAttributeWithGetter; @@ -113,6 +199,8 @@ static class TestConfig { private String testMutuallyExclusiveAttributeB; + private String testMutuallyExclusiveAttributeC; + @JsonProperty @AlsoRequired(values = { @AlsoRequired.Required(name="test_mutually_exclusive_attribute_a") From c9553b196b16f552e17c470493ae5512fcec7d81 Mon Sep 17 00:00:00 2001 From: George Chen Date: Thu, 24 Oct 2024 14:40:11 -0500 Subject: [PATCH 2/6] FIX: test case Signed-off-by: George Chen --- .../dataprepper/schemas/JsonSchemaConverterTest.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java index 0ab6247a42..4abbf9c883 100644 --- a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java +++ b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java @@ -135,17 +135,20 @@ void testConvertIntoJsonSchemaWithConditionalRequired() throws JsonProcessingExc assertThat(thenPropertiesNode2.isEmpty(), is(false)); final JsonNode thenAttributeNode2 = thenPropertiesNode2.at("/test_mutually_exclusive_attribute_c"); assertThat(thenAttributeNode2, instanceOf(ObjectNode.class)); + final JsonNode thenAttributeValueNode2 = thenAttributeNode2.at("/const"); + assertThat(thenAttributeValueNode2, instanceOf(TextNode.class)); + assertThat(thenAttributeValueNode2.asText(), equalTo("\"option1\"")); final JsonNode elseNode2 = ifThenElseNode2.at("/else"); assertThat(elseNode2, instanceOf(ObjectNode.class)); assertThat(elseNode2.has("required"), is(false)); final JsonNode elsePropertiesNode2 = elseNode2.at("/properties"); assertThat(elsePropertiesNode2, instanceOf(ObjectNode.class)); assertThat(elsePropertiesNode2.isEmpty(), is(false)); - final JsonNode elseAttributeNode2 = thenPropertiesNode2.at("/test_mutually_exclusive_attribute_c"); + final JsonNode elseAttributeNode2 = elsePropertiesNode2.at("/test_mutually_exclusive_attribute_c"); assertThat(elseAttributeNode2, instanceOf(ObjectNode.class)); -// assertThat(propertiesNode.has("testAttributeEventKey"), is(equalTo(true))); -// assertThat(propertiesNode.get("testAttributeEventKey"), is(notNullValue())); -// assertThat(propertiesNode.get("testAttributeEventKey").get("type"), is(equalTo(TextNode.valueOf("string")))); + final JsonNode elseAttributeValueNode2 = elseAttributeNode2.at("/const"); + assertThat(elseAttributeValueNode2, instanceOf(TextNode.class)); + assertThat(elseAttributeValueNode2.asText(), equalTo("\"option2\"")); } @ConditionalRequired(value = { From cf4936f77bcd2000e316b1515e3792e683240b84 Mon Sep 17 00:00:00 2001 From: George Chen Date: Thu, 24 Oct 2024 16:48:26 -0500 Subject: [PATCH 3/6] ENH: backfill processors Signed-off-by: George Chen --- .../processor/date/DateProcessorConfig.java | 11 +++++++ .../mutateevent/AddEntryProcessorConfig.java | 32 +++++++++++++++++++ .../ConvertEntryTypeProcessorConfig.java | 11 +++++++ .../mutateevent/ListToMapProcessorConfig.java | 7 ++++ .../SplitStringProcessorConfig.java | 11 +++++++ .../splitevent/SplitEventProcessorConfig.java | 11 +++++++ 6 files changed, 83 insertions(+) diff --git a/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java b/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java index c81a3a1ffb..9960ae33b2 100644 --- a/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java +++ b/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; import jakarta.validation.constraints.AssertTrue; import org.opensearch.dataprepper.model.annotations.AlsoRequired; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired; import org.opensearch.dataprepper.model.annotations.ExampleValues; import org.opensearch.dataprepper.model.annotations.ExampleValues.Example; @@ -20,6 +21,16 @@ import java.util.Locale; import java.time.format.DateTimeFormatter; +@ConditionalRequired(value = { + @ConditionalRequired.IfThenElse( + ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "match", value = "null")}, + thenExpect = {@ConditionalRequired.SchemaProperty(field = "from_time_received", value = "true")} + ), + @ConditionalRequired.IfThenElse( + ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "from_time_received", value = "false")}, + thenExpect = {@ConditionalRequired.SchemaProperty(field = "match")} + ) +}) @JsonPropertyOrder @JsonClassDescription("The date processor adds a default timestamp to an event, parses timestamp fields, " + "and converts timestamp information to the International Organization for Standardization (ISO) 8601 format. " + diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java index fc0df97e06..49cf29d345 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java @@ -14,10 +14,42 @@ import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import org.opensearch.dataprepper.model.annotations.AlsoRequired; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired; import java.util.List; import java.util.stream.Stream; +@ConditionalRequired(value = { + @ConditionalRequired.IfThenElse( + ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "key", value = "null")}, + thenExpect = {@ConditionalRequired.SchemaProperty(field = "metadata_key")} + ), + @ConditionalRequired.IfThenElse( + ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "metadata_key", value = "null")}, + thenExpect = {@ConditionalRequired.SchemaProperty(field = "key")} + ), + @ConditionalRequired.IfThenElse( + ifFulfilled = { + @ConditionalRequired.SchemaProperty(field = "format", value = "null"), + @ConditionalRequired.SchemaProperty(field = "value", value = "null"), + }, + thenExpect = {@ConditionalRequired.SchemaProperty(field = "value_expression")} + ), + @ConditionalRequired.IfThenElse( + ifFulfilled = { + @ConditionalRequired.SchemaProperty(field = "format", value = "null"), + @ConditionalRequired.SchemaProperty(field = "value_expression", value = "null"), + }, + thenExpect = {@ConditionalRequired.SchemaProperty(field = "value")} + ), + @ConditionalRequired.IfThenElse( + ifFulfilled = { + @ConditionalRequired.SchemaProperty(field = "value", value = "null"), + @ConditionalRequired.SchemaProperty(field = "value_expression", value = "null"), + }, + thenExpect = {@ConditionalRequired.SchemaProperty(field = "format")} + ) +}) @JsonPropertyOrder @JsonClassDescription("The add_entries processor adds entries to an event.") public class AddEntryProcessorConfig { diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java index 93f1c4bc91..a2e403c2b2 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java @@ -10,11 +10,22 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonPropertyDescription; import org.opensearch.dataprepper.model.annotations.AlsoRequired; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired; import org.opensearch.dataprepper.typeconverter.ConverterArguments; import java.util.List; import java.util.Optional; +@ConditionalRequired(value = { + @ConditionalRequired.IfThenElse( + ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "key", value = "null")}, + thenExpect = {@ConditionalRequired.SchemaProperty(field = "keys")} + ), + @ConditionalRequired.IfThenElse( + ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "keys", value = "null")}, + thenExpect = {@ConditionalRequired.SchemaProperty(field = "key")} + ) +}) @JsonPropertyOrder @JsonClassDescription("The convert_type processor converts a value associated with the specified key in " + "a event to the specified type. It is a casting processor that changes the types of specified fields in events.") diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessorConfig.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessorConfig.java index d998684444..065e56d7bb 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessorConfig.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessorConfig.java @@ -13,12 +13,19 @@ import com.fasterxml.jackson.annotation.JsonValue; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +@ConditionalRequired(value = { + @ConditionalRequired.IfThenElse( + ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "use_source_key", value = "false")}, + thenExpect = {@ConditionalRequired.SchemaProperty(field = "key")} + ) +}) @JsonPropertyOrder @JsonClassDescription("The list_to_map processor converts a list of objects from an event, " + "where each object contains a key field, into a map of target keys.") diff --git a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessorConfig.java b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessorConfig.java index fd18d1aa8c..568abc6461 100644 --- a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessorConfig.java +++ b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessorConfig.java @@ -15,10 +15,21 @@ import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.Size; import org.opensearch.dataprepper.model.annotations.AlsoRequired; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired; import org.opensearch.dataprepper.model.event.EventKey; import java.util.List; +@ConditionalRequired(value = { + @ConditionalRequired.IfThenElse( + ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "delimiter", value = "null")}, + thenExpect = {@ConditionalRequired.SchemaProperty(field = "delimiter_regex")} + ), + @ConditionalRequired.IfThenElse( + ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "delimiter_regex", value = "null")}, + thenExpect = {@ConditionalRequired.SchemaProperty(field = "delimiter")} + ) +}) @JsonPropertyOrder @JsonClassDescription("The split_string processor splits a field into an array using a delimiting character.") public class SplitStringProcessorConfig implements StringProcessorConfig { diff --git a/data-prepper-plugins/split-event-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/splitevent/SplitEventProcessorConfig.java b/data-prepper-plugins/split-event-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/splitevent/SplitEventProcessorConfig.java index 2c7b009e78..b4acbff42b 100644 --- a/data-prepper-plugins/split-event-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/splitevent/SplitEventProcessorConfig.java +++ b/data-prepper-plugins/split-event-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/splitevent/SplitEventProcessorConfig.java @@ -18,7 +18,18 @@ import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.Size; import org.opensearch.dataprepper.model.annotations.AlsoRequired; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired; +@ConditionalRequired(value = { + @ConditionalRequired.IfThenElse( + ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "delimiter", value = "null")}, + thenExpect = {@ConditionalRequired.SchemaProperty(field = "delimiter_regex")} + ), + @ConditionalRequired.IfThenElse( + ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "delimiter_regex", value = "null")}, + thenExpect = {@ConditionalRequired.SchemaProperty(field = "delimiter")} + ) +}) @JsonPropertyOrder @JsonClassDescription("The split_event processor is used to split events based on a delimiter and " + "generates multiple events from a user-specified field.") From fc1c6ce38f0273d5bf44c6c4728069e8f71b21ec Mon Sep 17 00:00:00 2001 From: George Chen Date: Thu, 24 Oct 2024 16:58:13 -0500 Subject: [PATCH 4/6] MAINT: doc string Signed-off-by: George Chen --- .../annotations/ConditionalRequired.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/ConditionalRequired.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/ConditionalRequired.java index 10627d4250..7808b0d321 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/ConditionalRequired.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/ConditionalRequired.java @@ -5,19 +5,46 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +/** + * Annotation used in schema generation to define the if-then-else requirements. + */ @Target({ ElementType.FIELD, ElementType.TYPE }) @Retention(RetentionPolicy.RUNTIME) public @interface ConditionalRequired { + /** + * Array of if-then-else requirements. + */ IfThenElse[] value(); + /** + * Annotation to represent an if-then-else requirement. + */ @interface IfThenElse { + /** + * Array of property schemas involved in if condition. + */ SchemaProperty[] ifFulfilled(); + /** + * Array of property schemas involved in then expectation. + */ SchemaProperty[] thenExpect(); + /** + * Array of property schemas involved in else expectation. + */ SchemaProperty[] elseExpect() default {}; } + /** + * Annotation to represent a property schema. + */ @interface SchemaProperty { + /** + * Name of the property. + */ String field(); + /** + * Value of the property. Empty string means any non-null value is allowed. + */ String value() default ""; } } From 6d040602cbf181d3a1e2e77ef28f262a1ba7757e Mon Sep 17 00:00:00 2001 From: George Chen Date: Thu, 24 Oct 2024 17:21:03 -0500 Subject: [PATCH 5/6] STY: remove unused import Signed-off-by: George Chen --- .../org/opensearch/dataprepper/schemas/JsonSchemaConverter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java index 247b6f3ca2..ef447264ba 100644 --- a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java +++ b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java @@ -8,7 +8,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.github.victools.jsonschema.generator.FieldScope; import com.github.victools.jsonschema.generator.Module; -import com.github.victools.jsonschema.generator.Option; import com.github.victools.jsonschema.generator.OptionPreset; import com.github.victools.jsonschema.generator.SchemaGenerator; import com.github.victools.jsonschema.generator.SchemaGeneratorConfig; From 53be872ad57c23c9c5cfe044662cd92047e36436 Mon Sep 17 00:00:00 2001 From: George Chen Date: Mon, 28 Oct 2024 14:01:51 -0500 Subject: [PATCH 6/6] MAINT: simplify reference Signed-off-by: George Chen --- .../processor/date/DateProcessorConfig.java | 14 ++++--- .../mutateevent/AddEntryProcessorConfig.java | 38 ++++++++++--------- .../ConvertEntryTypeProcessorConfig.java | 14 ++++--- .../mutateevent/ListToMapProcessorConfig.java | 8 ++-- .../SplitStringProcessorConfig.java | 14 ++++--- .../splitevent/SplitEventProcessorConfig.java | 14 ++++--- 6 files changed, 57 insertions(+), 45 deletions(-) diff --git a/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java b/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java index 9960ae33b2..f66303a170 100644 --- a/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java +++ b/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java @@ -13,6 +13,8 @@ import jakarta.validation.constraints.AssertTrue; import org.opensearch.dataprepper.model.annotations.AlsoRequired; import org.opensearch.dataprepper.model.annotations.ConditionalRequired; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired.IfThenElse; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired.SchemaProperty; import org.opensearch.dataprepper.model.annotations.ExampleValues; import org.opensearch.dataprepper.model.annotations.ExampleValues.Example; @@ -22,13 +24,13 @@ import java.time.format.DateTimeFormatter; @ConditionalRequired(value = { - @ConditionalRequired.IfThenElse( - ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "match", value = "null")}, - thenExpect = {@ConditionalRequired.SchemaProperty(field = "from_time_received", value = "true")} + @IfThenElse( + ifFulfilled = {@SchemaProperty(field = "match", value = "null")}, + thenExpect = {@SchemaProperty(field = "from_time_received", value = "true")} ), - @ConditionalRequired.IfThenElse( - ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "from_time_received", value = "false")}, - thenExpect = {@ConditionalRequired.SchemaProperty(field = "match")} + @IfThenElse( + ifFulfilled = {@SchemaProperty(field = "from_time_received", value = "false")}, + thenExpect = {@SchemaProperty(field = "match")} ) }) @JsonPropertyOrder diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java index 49cf29d345..008572a2a3 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java @@ -15,39 +15,41 @@ import jakarta.validation.constraints.NotNull; import org.opensearch.dataprepper.model.annotations.AlsoRequired; import org.opensearch.dataprepper.model.annotations.ConditionalRequired; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired.IfThenElse; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired.SchemaProperty; import java.util.List; import java.util.stream.Stream; @ConditionalRequired(value = { - @ConditionalRequired.IfThenElse( - ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "key", value = "null")}, - thenExpect = {@ConditionalRequired.SchemaProperty(field = "metadata_key")} + @IfThenElse( + ifFulfilled = {@SchemaProperty(field = "key", value = "null")}, + thenExpect = {@SchemaProperty(field = "metadata_key")} ), - @ConditionalRequired.IfThenElse( - ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "metadata_key", value = "null")}, - thenExpect = {@ConditionalRequired.SchemaProperty(field = "key")} + @IfThenElse( + ifFulfilled = {@SchemaProperty(field = "metadata_key", value = "null")}, + thenExpect = {@SchemaProperty(field = "key")} ), - @ConditionalRequired.IfThenElse( + @IfThenElse( ifFulfilled = { - @ConditionalRequired.SchemaProperty(field = "format", value = "null"), - @ConditionalRequired.SchemaProperty(field = "value", value = "null"), + @SchemaProperty(field = "format", value = "null"), + @SchemaProperty(field = "value", value = "null"), }, - thenExpect = {@ConditionalRequired.SchemaProperty(field = "value_expression")} + thenExpect = {@SchemaProperty(field = "value_expression")} ), - @ConditionalRequired.IfThenElse( + @IfThenElse( ifFulfilled = { - @ConditionalRequired.SchemaProperty(field = "format", value = "null"), - @ConditionalRequired.SchemaProperty(field = "value_expression", value = "null"), + @SchemaProperty(field = "format", value = "null"), + @SchemaProperty(field = "value_expression", value = "null"), }, - thenExpect = {@ConditionalRequired.SchemaProperty(field = "value")} + thenExpect = {@SchemaProperty(field = "value")} ), - @ConditionalRequired.IfThenElse( + @IfThenElse( ifFulfilled = { - @ConditionalRequired.SchemaProperty(field = "value", value = "null"), - @ConditionalRequired.SchemaProperty(field = "value_expression", value = "null"), + @SchemaProperty(field = "value", value = "null"), + @SchemaProperty(field = "value_expression", value = "null"), }, - thenExpect = {@ConditionalRequired.SchemaProperty(field = "format")} + thenExpect = {@SchemaProperty(field = "format")} ) }) @JsonPropertyOrder diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java index a2e403c2b2..7b401ec843 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java @@ -11,19 +11,21 @@ import com.fasterxml.jackson.annotation.JsonPropertyDescription; import org.opensearch.dataprepper.model.annotations.AlsoRequired; import org.opensearch.dataprepper.model.annotations.ConditionalRequired; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired.IfThenElse; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired.SchemaProperty; import org.opensearch.dataprepper.typeconverter.ConverterArguments; import java.util.List; import java.util.Optional; @ConditionalRequired(value = { - @ConditionalRequired.IfThenElse( - ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "key", value = "null")}, - thenExpect = {@ConditionalRequired.SchemaProperty(field = "keys")} + @IfThenElse( + ifFulfilled = {@SchemaProperty(field = "key", value = "null")}, + thenExpect = {@SchemaProperty(field = "keys")} ), - @ConditionalRequired.IfThenElse( - ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "keys", value = "null")}, - thenExpect = {@ConditionalRequired.SchemaProperty(field = "key")} + @IfThenElse( + ifFulfilled = {@SchemaProperty(field = "keys", value = "null")}, + thenExpect = {@SchemaProperty(field = "key")} ) }) @JsonPropertyOrder diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessorConfig.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessorConfig.java index 065e56d7bb..91fb38a1f1 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessorConfig.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessorConfig.java @@ -14,6 +14,8 @@ import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import org.opensearch.dataprepper.model.annotations.ConditionalRequired; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired.IfThenElse; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired.SchemaProperty; import java.util.Arrays; import java.util.List; @@ -21,9 +23,9 @@ import java.util.stream.Collectors; @ConditionalRequired(value = { - @ConditionalRequired.IfThenElse( - ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "use_source_key", value = "false")}, - thenExpect = {@ConditionalRequired.SchemaProperty(field = "key")} + @IfThenElse( + ifFulfilled = {@SchemaProperty(field = "use_source_key", value = "false")}, + thenExpect = {@SchemaProperty(field = "key")} ) }) @JsonPropertyOrder diff --git a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessorConfig.java b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessorConfig.java index 568abc6461..11116859aa 100644 --- a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessorConfig.java +++ b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessorConfig.java @@ -16,18 +16,20 @@ import jakarta.validation.constraints.Size; import org.opensearch.dataprepper.model.annotations.AlsoRequired; import org.opensearch.dataprepper.model.annotations.ConditionalRequired; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired.IfThenElse; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired.SchemaProperty; import org.opensearch.dataprepper.model.event.EventKey; import java.util.List; @ConditionalRequired(value = { - @ConditionalRequired.IfThenElse( - ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "delimiter", value = "null")}, - thenExpect = {@ConditionalRequired.SchemaProperty(field = "delimiter_regex")} + @IfThenElse( + ifFulfilled = {@SchemaProperty(field = "delimiter", value = "null")}, + thenExpect = {@SchemaProperty(field = "delimiter_regex")} ), - @ConditionalRequired.IfThenElse( - ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "delimiter_regex", value = "null")}, - thenExpect = {@ConditionalRequired.SchemaProperty(field = "delimiter")} + @IfThenElse( + ifFulfilled = {@SchemaProperty(field = "delimiter_regex", value = "null")}, + thenExpect = {@SchemaProperty(field = "delimiter")} ) }) @JsonPropertyOrder diff --git a/data-prepper-plugins/split-event-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/splitevent/SplitEventProcessorConfig.java b/data-prepper-plugins/split-event-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/splitevent/SplitEventProcessorConfig.java index b4acbff42b..188d3cfc9f 100644 --- a/data-prepper-plugins/split-event-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/splitevent/SplitEventProcessorConfig.java +++ b/data-prepper-plugins/split-event-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/splitevent/SplitEventProcessorConfig.java @@ -19,15 +19,17 @@ import jakarta.validation.constraints.Size; import org.opensearch.dataprepper.model.annotations.AlsoRequired; import org.opensearch.dataprepper.model.annotations.ConditionalRequired; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired.IfThenElse; +import org.opensearch.dataprepper.model.annotations.ConditionalRequired.SchemaProperty; @ConditionalRequired(value = { - @ConditionalRequired.IfThenElse( - ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "delimiter", value = "null")}, - thenExpect = {@ConditionalRequired.SchemaProperty(field = "delimiter_regex")} + @IfThenElse( + ifFulfilled = {@SchemaProperty(field = "delimiter", value = "null")}, + thenExpect = {@SchemaProperty(field = "delimiter_regex")} ), - @ConditionalRequired.IfThenElse( - ifFulfilled = {@ConditionalRequired.SchemaProperty(field = "delimiter_regex", value = "null")}, - thenExpect = {@ConditionalRequired.SchemaProperty(field = "delimiter")} + @IfThenElse( + ifFulfilled = {@SchemaProperty(field = "delimiter_regex", value = "null")}, + thenExpect = {@SchemaProperty(field = "delimiter")} ) }) @JsonPropertyOrder