From dbfb1d36f8ee2d3b934614dca5d6f26179c4025d Mon Sep 17 00:00:00 2001 From: Taylor Gray Date: Thu, 14 Nov 2024 11:34:33 -0600 Subject: [PATCH] Adds example annotations to a handful of processors (#5140) Signed-off-by: Taylor Gray --- .../aggregate/AggregateProcessorConfig.java | 13 ++++++++++ .../AnomalyDetectorProcessorConfig.java | 7 ++++- .../plugins/processor/DelayProcessor.java | 7 +++++ .../processor/csv/CsvProcessorConfig.java | 8 ++++++ .../decompress/DecompressProcessorConfig.java | 5 ++++ .../mutateevent/AddEntryProcessorConfig.java | 26 +++++++++++++++++++ .../ConvertEntryTypeProcessorConfig.java | 8 +++++- .../mutateevent/CopyValueProcessorConfig.java | 6 +++++ .../DeleteEntryProcessorConfig.java | 5 ++++ 9 files changed, 83 insertions(+), 2 deletions(-) diff --git a/data-prepper-plugins/aggregate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/aggregate/AggregateProcessorConfig.java b/data-prepper-plugins/aggregate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/aggregate/AggregateProcessorConfig.java index b9ea6ead84..bd08de94c6 100644 --- a/data-prepper-plugins/aggregate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/aggregate/AggregateProcessorConfig.java +++ b/data-prepper-plugins/aggregate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/aggregate/AggregateProcessorConfig.java @@ -8,6 +8,8 @@ import com.fasterxml.jackson.annotation.JsonClassDescription; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import org.opensearch.dataprepper.model.annotations.AlsoRequired; +import org.opensearch.dataprepper.model.annotations.ExampleValues; +import org.opensearch.dataprepper.model.annotations.ExampleValues.Example; import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; import org.opensearch.dataprepper.model.configuration.PluginModel; import com.fasterxml.jackson.annotation.JsonPropertyDescription; @@ -32,6 +34,9 @@ public class AggregateProcessorConfig { "At least one identification_key is required. And example configuration is [\"sourceIp\", \"destinationIp\", \"port\"].") @JsonProperty("identification_keys") @NotEmpty + @ExampleValues({ + @Example(value = "group_id", description = "Aggregate based on the values of a group_id key in the Events") + }) private List identificationKeys; @JsonPropertyDescription("The action to be performed on each group. One of the available aggregate actions must be provided.") @@ -42,6 +47,11 @@ public class AggregateProcessorConfig { @JsonPropertyDescription("The amount of time that a group should exist before it is concluded automatically. Supports ISO_8601 notation strings (\"PT20.345S\", \"PT15M\", etc.) as well as simple notation for seconds (\"60s\") and milliseconds (\"1500ms\"). Default value is 180s.") @JsonProperty(value = "group_duration", defaultValue = DEFAULT_GROUP_DURATION_SECONDS + "s") + @ExampleValues({ + @Example(value = "180s", description = "Aggregated groups will be flushed after 180 seconds"), + @Example(value = "1000ms", description = "Aggregated groups will be flushed after 1,000 milliseconds"), + @Example(value = "PT2H", description = "Aggregated groups will be flushed after 2 hours"), + }) private Duration groupDuration = Duration.ofSeconds(DEFAULT_GROUP_DURATION_SECONDS); @JsonPropertyDescription("When local_mode is set to true, the aggregation is performed locally on each node instead of forwarding events to a specific node based on the identification_keys using a hash function. Default is false.") @@ -62,6 +72,9 @@ public class AggregateProcessorConfig { @JsonPropertyDescription("A conditional expression, such as /some-key == \"test\", that will be evaluated to determine whether the processor will be run on the event.") @JsonProperty("aggregate_when") + @ExampleValues({ + @Example(value = "/some_key == null", description = "Only includes events in the aggregations if the key some_key is null or does not exist.") + }) private String whenCondition; public List getIdentificationKeys() { diff --git a/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/AnomalyDetectorProcessorConfig.java b/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/AnomalyDetectorProcessorConfig.java index 2122a0f318..69d4f8c8a4 100644 --- a/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/AnomalyDetectorProcessorConfig.java +++ b/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/AnomalyDetectorProcessorConfig.java @@ -7,6 +7,8 @@ import com.fasterxml.jackson.annotation.JsonClassDescription; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import org.opensearch.dataprepper.model.annotations.ExampleValues; +import org.opensearch.dataprepper.model.annotations.ExampleValues.Example; import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; import org.opensearch.dataprepper.model.configuration.PluginModel; import com.fasterxml.jackson.annotation.JsonPropertyDescription; @@ -34,6 +36,9 @@ public class AnomalyDetectorProcessorConfig { @JsonPropertyDescription("If provided, anomalies will be detected within each unique instance of these keys. For example, if you provide the ip field, anomalies will be detected separately for each unique IP address.") @JsonProperty("identification_keys") + @ExampleValues({ + @Example(value = "ip_address", description = "Anomalies will be detected separately for each unique IP address from the existing ip_address key of the Event.") + }) private List identificationKeys = Collections.emptyList(); @JsonPropertyDescription("RCF will try to automatically learn and reduce the number of anomalies detected. For example, if latency is consistently between 50 and 100, and then suddenly jumps to around 1000, only the first one or two data points after the transition will be detected (unless there are other spikes/anomalies). Similarly, for repeated spikes to the same level, RCF will likely eliminate many of the spikes after a few initial ones. This is because the default setting is to minimize the number of alerts detected. Setting the verbose setting to true will cause RCF to consistently detect these repeated cases, which may be useful for detecting anomalous behavior that lasts an extended period of time. Default is false.") @@ -41,7 +46,7 @@ public class AnomalyDetectorProcessorConfig { private Boolean verbose = false; @JsonPropertyDescription("If using the identification_keys settings, a new ML model will be created for every degree of cardinality. This can cause a large amount of memory usage, so it is helpful to set a limit on the number of models. Default limit is 5000.") - @JsonProperty("cardinality_limit") + @JsonProperty(value = "cardinality_limit", defaultValue = "5000") private int cardinalityLimit = 5000; public PluginModel getDetectorMode() { diff --git a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/DelayProcessor.java b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/DelayProcessor.java index 2f42016c95..31150fd5a3 100644 --- a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/DelayProcessor.java +++ b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/DelayProcessor.java @@ -12,6 +12,8 @@ import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; +import org.opensearch.dataprepper.model.annotations.ExampleValues; +import org.opensearch.dataprepper.model.annotations.ExampleValues.Example; import org.opensearch.dataprepper.model.processor.Processor; import org.opensearch.dataprepper.model.record.Record; import org.slf4j.Logger; @@ -63,6 +65,11 @@ public void shutdown() { public static class Configuration { @JsonProperty("for") @JsonPropertyDescription("The duration of time to delay. Defaults to 1s.") + @ExampleValues({ + @Example(value = "1s", description = "Delays for 1 second."), + @Example(value = "1000ms", description = "Delays for 1,000 milliseconds."), + @Example(value = "PT10M", description = "Delays for 10 minutes."), + }) private Duration delayFor = Duration.ofSeconds(1); public Duration getDelayFor() { diff --git a/data-prepper-plugins/csv-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/csv/CsvProcessorConfig.java b/data-prepper-plugins/csv-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/csv/CsvProcessorConfig.java index 3cb0d699e1..940c6e3a9e 100644 --- a/data-prepper-plugins/csv-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/csv/CsvProcessorConfig.java +++ b/data-prepper-plugins/csv-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/csv/CsvProcessorConfig.java @@ -11,6 +11,8 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.NotBlank; +import org.opensearch.dataprepper.model.annotations.ExampleValues; +import org.opensearch.dataprepper.model.annotations.ExampleValues.Example; import java.util.List; @@ -61,11 +63,17 @@ public class CsvProcessorConfig { "the header in column_names_source_key generates the event fields. If too few columns are specified " + "in this field, the remaining column names are automatically generated. " + "If too many column names are specified in this field, the CSV processor omits the extra column names.") + @ExampleValues({ + @Example(value = "column_one", description = "Specifies column_one as one of the column names to create") + }) private List columnNames; @JsonProperty("csv_when") @JsonPropertyDescription("A conditional expression such as /some_key == \"test\". " + "If specified, the csv processor will only run on events when the expression evaluates to true. ") + @ExampleValues({ + @Example(value = "/some_key == null", description = "Only runs the csv processor if the key some_key is null or does not exist.") + }) private String csvWhen; @JsonPropertyDescription("If true, the configured source field will be deleted after the CSV data is parsed into separate fields.") diff --git a/data-prepper-plugins/decompress-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/decompress/DecompressProcessorConfig.java b/data-prepper-plugins/decompress-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/decompress/DecompressProcessorConfig.java index a6b8202879..d7b9d81e3c 100644 --- a/data-prepper-plugins/decompress-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/decompress/DecompressProcessorConfig.java +++ b/data-prepper-plugins/decompress-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/decompress/DecompressProcessorConfig.java @@ -12,6 +12,8 @@ import com.fasterxml.jackson.annotation.JsonPropertyDescription; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; +import org.opensearch.dataprepper.model.annotations.ExampleValues; +import org.opensearch.dataprepper.model.annotations.ExampleValues.Example; import org.opensearch.dataprepper.plugins.processor.decompress.encoding.EncodingType; import org.opensearch.dataprepper.plugins.processor.decompress.encoding.DecoderEngineFactory; @@ -39,6 +41,9 @@ public class DecompressProcessorConfig { @JsonPropertyDescription("A conditional expression, such as /is_compressed == true, that determines when the decompress processor will run on certain events.") @JsonProperty("decompress_when") + @ExampleValues({ + @Example(value = "/some_key == null", description = "Only runs the decompress processor on the Event if the key some_key is null or does not exist.") + }) private String decompressWhen; @JsonIgnore diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java index 008572a2a3..63e73be1b4 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java @@ -17,6 +17,8 @@ import org.opensearch.dataprepper.model.annotations.ConditionalRequired; import org.opensearch.dataprepper.model.annotations.ConditionalRequired.IfThenElse; import org.opensearch.dataprepper.model.annotations.ConditionalRequired.SchemaProperty; +import org.opensearch.dataprepper.model.annotations.ExampleValues; +import org.opensearch.dataprepper.model.annotations.ExampleValues.Example; import java.util.List; import java.util.stream.Stream; @@ -68,6 +70,11 @@ public static class Entry { @AlsoRequired(values = { @AlsoRequired.Required(name=METADATA_KEY_KEY, allowedValues = {"null"}) }) + @ExampleValues({ + @Example(value = "my_key", description = "Adds 'my_key' to the Event"), + @Example(value = "${/key_one}-${/key_two}", description = "Evaluates existing Event keys key_one and key_two and adds the result to the Event"), + @Example(value = "/nested/key", description = "Adds a nested key of { \"nested\": { \"key\": \"some_value\" }"), + }) private String key; @JsonProperty(METADATA_KEY_KEY) @@ -76,6 +83,9 @@ public static class Entry { @AlsoRequired(values = { @AlsoRequired.Required(name="key", allowedValues = {"null"}) }) + @ExampleValues({ + @Example(value = "some_metadata", description = "The Event will contain a metadata key called 'some_metadata' that can be used in expressions with sending the key to the sinks.") + }) private String metadataKey; @JsonPropertyDescription("The value of the new entry to be added, which can be used with any of the " + @@ -84,6 +94,12 @@ public static class Entry { @AlsoRequired.Required(name="format", allowedValues = {"null"}), @AlsoRequired.Required(name=VALUE_EXPRESSION_KEY, allowedValues = {"null"}) }) + @ExampleValues({ + @Example(value = "my_string_value", description = "Adds a value of 'my_string_value' to the key or metadata_key"), + @Example(value = "false", description = "Adds a value of false to the key or metadata_key"), + @Example(value = "10", description = "Adds a value of 10 to the key or metadata_key"), + @Example(value = "[ \"element_one\", \"element_two\" ]", description = "Adds an array value with two elements to the key or metadata_key"), + }) private Object value; @JsonPropertyDescription("A format string to use as the value of the new entry, for example, " + @@ -93,6 +109,9 @@ public static class Entry { @AlsoRequired.Required(name="value", allowedValues = {"null"}), @AlsoRequired.Required(name=VALUE_EXPRESSION_KEY, allowedValues = {"null"}) }) + @ExampleValues({ + @Example(value = "${/key_one}-${/key_two}", description = "Adds a value as a combination of the existing key_one and key_two values to the key or metadata_key"), + }) private String format; @JsonProperty(VALUE_EXPRESSION_KEY) @@ -105,6 +124,10 @@ public static class Entry { @AlsoRequired.Required(name="value", allowedValues = {"null"}), @AlsoRequired.Required(name="format", allowedValues = {"null"}) }) + @ExampleValues({ + @Example(value = "length(/my_key)", description = "Adds an integer value based on the length of the existing key 'my_key' to the new key or metadata_key"), + @Example(value = "/my_key", description = "Adds a value based on the existing value of my_key to the new key or metadata_key"), + }) private String valueExpression; @JsonProperty(OVERWRITE_IF_KEY_EXISTS_KEY) @@ -126,6 +149,9 @@ public static class Entry { @JsonProperty("add_when") @JsonPropertyDescription("A conditional expression, " + "such as /some-key == \"test\", that will be evaluated to determine whether the processor will be run on the event.") + @ExampleValues({ + @Example(value = "/some_key == null", description = "Only runs the add_entries processor if the key some_key is null or does not exist.") + }) private String addWhen; public String getKey() { diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java index 7b401ec843..48a0d0634d 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java @@ -13,6 +13,8 @@ import org.opensearch.dataprepper.model.annotations.ConditionalRequired; import org.opensearch.dataprepper.model.annotations.ConditionalRequired.IfThenElse; import org.opensearch.dataprepper.model.annotations.ConditionalRequired.SchemaProperty; +import org.opensearch.dataprepper.model.annotations.ExampleValues; +import org.opensearch.dataprepper.model.annotations.ExampleValues.Example; import org.opensearch.dataprepper.typeconverter.ConverterArguments; import java.util.List; @@ -69,7 +71,11 @@ public class ConvertEntryTypeProcessorConfig implements ConverterArguments { private List tagsOnFailure; @JsonProperty("convert_when") - @JsonPropertyDescription("Specifies a condition using a conditional expression for performing the convert_entry_type operation. If specified, the convert_entry_type operation runs only when the expression evaluates to true. Example: /mykey != \"---\"") + @JsonPropertyDescription("Specifies a condition using a conditional expression for performing the convert_type operation. If specified, the convert_type operation runs only when the expression evaluates to true. Example: /mykey != \"test\"") + @ExampleValues({ + @Example(value = "/some_key == null", description = "Only runs the convert_type processor on the Event if the existing key some_key is null or does not exist."), + @Example(value = "/some_key typeof integer", description = "Only runs the convert_type processor on the Event if the key some_key is an integer.") + }) private String convertWhen; public String getKey() { diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/CopyValueProcessorConfig.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/CopyValueProcessorConfig.java index fb47207d09..c4e7dae876 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/CopyValueProcessorConfig.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/CopyValueProcessorConfig.java @@ -14,6 +14,8 @@ import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import org.opensearch.dataprepper.model.annotations.AlsoRequired; +import org.opensearch.dataprepper.model.annotations.ExampleValues; +import org.opensearch.dataprepper.model.annotations.ExampleValues.Example; import java.util.List; @@ -45,6 +47,10 @@ public static class Entry { @JsonProperty("copy_when") @JsonPropertyDescription("A conditional expression, " + "such as /some-key == \"test\", that will be evaluated to determine whether the processor will be run on the event.") + @ExampleValues({ + @Example(value = "/some_key != null", description = "Only runs the copy_values processor on the Event if the existing key some_key is not null."), + @Example(value = "/some_key typeof integer", description = "Only runs the copy_values processor on the Event if the key some_key is an integer.") + }) private String copyWhen; public String getFromKey() { diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessorConfig.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessorConfig.java index f8a2b07513..d1863faa2c 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessorConfig.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessorConfig.java @@ -11,6 +11,8 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; +import org.opensearch.dataprepper.model.annotations.ExampleValues; +import org.opensearch.dataprepper.model.annotations.ExampleValues.Example; import org.opensearch.dataprepper.model.event.EventKey; import org.opensearch.dataprepper.model.event.EventKeyConfiguration; import org.opensearch.dataprepper.model.event.EventKeyFactory; @@ -32,6 +34,9 @@ public class DeleteEntryProcessorConfig { @JsonProperty("delete_when") @JsonPropertyDescription("Specifies under what condition the delete_entries processor should perform deletion. " + "By default, keys are always deleted. Example: /mykey == \"---\"") + @ExampleValues({ + @Example(value = "/some_key == null", description = "Only runs the delete_entries processor on the Event if the key some_key is null or does not exist.") + }) private String deleteWhen; public List getWithKeys() {