Documentation improvements for a number of Data Prepper processors. (#…

…5023)

Documentation improvements for a number of Data Prepper processors.

Updates some enumerations used in processors to support the @JsonValue. Corrects @JsonClassDescription to use HTML rather than Markdown.

Signed-off-by: David Venable <[email protected]>

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/ConditionalRoute.java

@@ -31,7 +31,7 @@
  */
 @JsonPropertyOrder
 @JsonClassDescription("The key-value pair defines routing condition, where the key is the name of a route and the " +
-        "value is a Data Prepper expression representing the routing condition.")
+        "value is an expression representing the routing condition.")
 @JsonSerialize(using = ConditionalRoute.ConditionalRouteSerializer.class)
 @JsonDeserialize(using = ConditionalRoute.ConditionalRouteDeserializer.class)
 public class ConditionalRoute {

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/HandleFailedEventsOption.java

@@ -6,6 +6,7 @@
 package org.opensearch.dataprepper.model.event;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
 
 import java.util.Arrays;
 import java.util.Map;
@@ -45,4 +46,9 @@ public boolean shouldLog() {
     static HandleFailedEventsOption fromOptionValue(final String option) {
         return OPTIONS_MAP.get(option.toLowerCase());
     }
+
+    @JsonValue
+    public String toOptionValue() {
+        return option;
+    }
 }

data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/HandleFailedEventsOptionTest.java

@@ -5,29 +5,84 @@
 
 package org.opensearch.dataprepper.model.event;
 
-import org.hamcrest.CoreMatchers;
+import org.junit.jupiter.api.extension.ExtensionContext;
 import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.ArgumentsProvider;
+import org.junit.jupiter.params.provider.ArgumentsSource;
 import org.junit.jupiter.params.provider.EnumSource;
 
+import java.util.stream.Stream;
+
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.jupiter.params.provider.Arguments.arguments;
 
 class HandleFailedEventsOptionTest {
+    @ParameterizedTest
+    @ArgumentsSource(EnumToShouldLogArgumentsProvider.class)
+    void shouldLog_returns_expected_value(final HandleFailedEventsOption option, final boolean shouldLog) {
+        assertThat(option.shouldLog(), equalTo(shouldLog));
+    }
+
+    @ParameterizedTest
+    @ArgumentsSource(EnumToShouldShouldDropArgumentsProvider.class)
+    void shouldDropEvent_returns_expected_value(final HandleFailedEventsOption option, final boolean shouldDrop) {
+        assertThat(option.shouldDropEvent(), equalTo(shouldDrop));
+    }
+
+    @ParameterizedTest
+    @ArgumentsSource(EnumToOptionValueArgumentsProvider.class)
+    void toOptionValue_returns_expected_value(final HandleFailedEventsOption option, final String optionValue) {
+        assertThat(option.toOptionValue(), equalTo(optionValue));
+    }
+
+    @ParameterizedTest
+    @ArgumentsSource(EnumToOptionValueArgumentsProvider.class)
+    void fromOptionValue_returns_expected_option(final HandleFailedEventsOption option, final String optionValue) {
+        assertThat(HandleFailedEventsOption.fromOptionValue(optionValue), equalTo(option));
+    }
+
     @ParameterizedTest
     @EnumSource(HandleFailedEventsOption.class)
-    void fromOptionValue(final HandleFailedEventsOption option) {
-        assertThat(HandleFailedEventsOption.fromOptionValue(option.name()), CoreMatchers.is(option));
+    void toOptionValue_returns_non_null_for_all(final HandleFailedEventsOption option) {
+        assertThat(option.toOptionValue(), notNullValue());
+    }
 
-        if (option == HandleFailedEventsOption.SKIP || option == HandleFailedEventsOption.SKIP_SILENTLY) {
-            assertThat(option.shouldDropEvent(), equalTo(false));
-        } else {
-            assertThat(option.shouldDropEvent(), equalTo(true));
+    private static class EnumToOptionValueArgumentsProvider implements ArgumentsProvider {
+        @Override
+        public Stream<? extends Arguments> provideArguments(final ExtensionContext context) {
+            return Stream.of(
+                    arguments(HandleFailedEventsOption.SKIP, "skip"),
+                    arguments(HandleFailedEventsOption.SKIP_SILENTLY, "skip_silently"),
+                    arguments(HandleFailedEventsOption.DROP, "drop"),
+                    arguments(HandleFailedEventsOption.DROP_SILENTLY, "drop_silently")
+            );
         }
+    }
+
+    private static class EnumToShouldShouldDropArgumentsProvider implements ArgumentsProvider {
+        @Override
+        public Stream<? extends Arguments> provideArguments(final ExtensionContext context) {
+            return Stream.of(
+                    arguments(HandleFailedEventsOption.SKIP, false),
+                    arguments(HandleFailedEventsOption.SKIP_SILENTLY, false),
+                    arguments(HandleFailedEventsOption.DROP, true),
+                    arguments(HandleFailedEventsOption.DROP_SILENTLY, true)
+            );
+        }
+    }
 
-        if (option == HandleFailedEventsOption.SKIP_SILENTLY || option == HandleFailedEventsOption.DROP_SILENTLY) {
-            assertThat(option.shouldLog(), equalTo(false));
-        } else {
-            assertThat(option.shouldLog(), equalTo(true));
+    private static class EnumToShouldLogArgumentsProvider implements ArgumentsProvider {
+        @Override
+        public Stream<? extends Arguments> provideArguments(final ExtensionContext context) {
+            return Stream.of(
+                    arguments(HandleFailedEventsOption.SKIP, true),
+                    arguments(HandleFailedEventsOption.DROP, true),
+                    arguments(HandleFailedEventsOption.SKIP_SILENTLY, false),
+                    arguments(HandleFailedEventsOption.DROP_SILENTLY, false)
+            );
         }
     }
 }

data-prepper-plugins/aggregate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/aggregate/AggregateProcessorConfig.java

@@ -19,14 +19,16 @@
 import java.util.List;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `aggregate` processor groups events based on the values of identification_keys. " +
+@JsonClassDescription("The <code>aggregate</code> processor groups events based on the values of identification_keys. " +
         "Then, the processor performs an action on each group, helping reduce unnecessary log volume and " +
         "creating aggregated logs over time.")
 public class AggregateProcessorConfig {
 
     static int DEFAULT_GROUP_DURATION_SECONDS = 180;
 
-    @JsonPropertyDescription("An unordered list by which to group events. Events with the same values as these keys are put into the same group. If an event does not contain one of the identification_keys, then the value of that key is considered to be equal to null. At least one identification_key is required (for example, [\"sourceIp\", \"destinationIp\", \"port\"].")
+    @JsonPropertyDescription("An unordered list by which to group events. Events with the same values as these keys are put into the same group. " +
+            "If an event does not contain one of the <code>identification_keys</code>, then the value of that key is considered to be equal to <code>null</code>. " +
+            "At least one <code>identification_key</code> is required. And example configuration is [\"sourceIp\", \"destinationIp\", \"port\"].")
     @JsonProperty("identification_keys")
     @NotEmpty
     private List<String> identificationKeys;
@@ -41,20 +43,20 @@ public class AggregateProcessorConfig {
     @UsesDataPrepperPlugin(pluginType = AggregateAction.class)
     private PluginModel aggregateAction;
 
-    @JsonPropertyDescription("When local_mode is set to true, the aggregation is performed locally on each Data Prepper node instead of forwarding events to a specific node based on the identification_keys using a hash function. Default is false.")
+    @JsonPropertyDescription("When <code>local_mode<code> is set to true, the aggregation is performed locally on each node instead of forwarding events to a specific node based on the <code>identification_keys</code> using a hash function. Default is false.")
     @JsonProperty("local_mode")
     @NotNull
     private Boolean localMode = false;
 
-    @JsonPropertyDescription("A boolean indicating if the unaggregated events should be forwarded to the next processor/sink in the chain.")
+    @JsonPropertyDescription("A boolean indicating if the unaggregated events should be forwarded to the next processor or sink in the chain.")
     @JsonProperty("output_unaggregated_events")
     private Boolean outputUnaggregatedEvents = false;
 
     @JsonPropertyDescription("Tag to be used for aggregated events to distinguish aggregated events from unaggregated events.")
     @JsonProperty("aggregated_events_tag")
     private String aggregatedEventsTag;
 
-    @JsonPropertyDescription("A Data Prepper <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, such as '/some-key == \"test\"', that will be evaluated to determine whether the processor will be run on the event.")
+    @JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, such as '/some-key == \"test\"', that will be evaluated to determine whether the processor will be run on the event.")
     @JsonProperty("aggregate_when")
     private String whenCondition;
 

data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/DelayProcessor.java

@@ -61,7 +61,7 @@ public void shutdown() {
             "Typically, you should use this only for testing, experimenting, and debugging.")
     public static class Configuration {
         @JsonProperty("for")
-        @JsonPropertyDescription("The duration of time to delay. Defaults to `1s`.")
+        @JsonPropertyDescription("The duration of time to delay. Defaults to <code>1s</code>.")
         private Duration delayFor = Duration.ofSeconds(1);
 
         public Duration getDelayFor() {

data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/StringProcessor.java

@@ -43,7 +43,7 @@ public class StringProcessor implements Processor<Record<Event>, Record<Event>>
     private final boolean upperCase;
 
     @JsonPropertyOrder
-    @JsonClassDescription("The `string_converter` processor converts a string to uppercase or lowercase.")
+    @JsonClassDescription("The <code>string_converter</code> processor converts a string to uppercase or lowercase.")
     public static class Configuration {
         @JsonPropertyDescription("Whether to convert to uppercase (<code>true</code>) or lowercase (<code>false</code>).")
         private boolean upperCase = true;

data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java

@@ -18,7 +18,7 @@
 import java.time.format.DateTimeFormatter;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `date` processor adds a default timestamp to an event, parses timestamp fields, " +
+@JsonClassDescription("The <code>date</code> processor adds a default timestamp to an event, parses timestamp fields, " +
         "and converts timestamp information to the International Organization for Standardization (ISO) 8601 format. " +
         "This timestamp information can be used as an event timestamp.")
 public class DateProcessorConfig {
@@ -34,10 +34,13 @@ public static class DateMatch {
         @JsonPropertyDescription("Represents the event key against which to match patterns. " +
                 "Required if <code>match</code> is configured.")
         private String key;
+
         @JsonProperty("patterns")
         @JsonPropertyDescription("A list of possible patterns that the timestamp value of the key can have. The patterns " +
                 "are based on a sequence of letters and symbols. The <code>patterns</code> support all the patterns listed in the " +
                 "Java DateTimeFormatter (https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html) reference. " +
+                "To match ISO 8601 formatted strings, use, <code>yyyy-MM-dd'T'HH:mm:ss.SSSXXX</code>. " +
+                "To match Apache Common Log Format, use <code>dd/MMM/yyyy:HH:mm:ss Z</code>. " +
                 "The timestamp value also supports <code>epoch_second</code>, <code>epoch_milli</code>, and <code>epoch_nano</code> values, " +
                 "which represent the timestamp as the number of seconds, milliseconds, and nanoseconds since the epoch. " +
                 "Epoch values always use the UTC time zone.")
@@ -110,7 +113,9 @@ public static boolean isValidPattern(final String pattern) {
 
     @JsonProperty("match")
     @JsonPropertyDescription("The date match configuration. " +
-            "This option cannot be defined at the same time as <code>from_time_received</code>. There is no default value.")
+            "This option cannot be defined at the same time as <code>from_time_received</code>. " +
+            "The date processor will use the first pattern that matches each event's timestamp field. " +
+            "You must provide at least one pattern unless you have <code>from_time_received</code>.")
     private List<DateMatch> match;
 
     @JsonProperty("destination")

data-prepper-plugins/decompress-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/decompress/DecompressProcessorConfig.java

@@ -18,7 +18,7 @@
 import java.util.List;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `decompress` processor decompresses any Base64-encoded " +
+@JsonClassDescription("The <code>decompress</code> processor decompresses any Base64-encoded " +
         "compressed fields inside of an event.")
 public class DecompressProcessorConfig {
 
@@ -28,16 +28,16 @@ public class DecompressProcessorConfig {
     @NotNull
     private List<String> keys;
 
-    @JsonPropertyDescription("The type of decompression to use for the keys in the event. Only gzip is supported.")
+    @JsonPropertyDescription("The type of decompression to use for the keys in the event. Only <code>gzip</code> is supported.")
     @JsonProperty("type")
     @NotNull
     private DecompressionType decompressionType;
 
-    @JsonPropertyDescription("A conditional expression that determines when the decompress processor will run on certain events.")
+    @JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, such as <code>'/is_compressed == true'</code>, that determines when the decompress processor will run on certain events.")
     @JsonProperty("decompress_when")
     private String decompressWhen;
 
-    @JsonPropertyDescription("A list of strings with which to tag events when the processor fails to decompress the keys inside an event. Defaults to _decompression_failure.")
+    @JsonPropertyDescription("A list of strings with which to tag events when the processor fails to decompress the keys inside an event. Defaults to <code>_decompression_failure</code>.")
     @JsonProperty("tags_on_failure")
     private List<String> tagsOnFailure = List.of("_decompression_failure");
 

data-prepper-plugins/decompress-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/decompress/DecompressionType.java

@@ -6,6 +6,7 @@
 package org.opensearch.dataprepper.plugins.processor.decompress;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
 import org.opensearch.dataprepper.model.codec.DecompressionEngine;
 import org.opensearch.dataprepper.plugins.codec.GZipDecompressionEngine;
 
@@ -37,6 +38,11 @@ static DecompressionType fromOptionValue(final String option) {
         return OPTIONS_MAP.get(option);
     }
 
+    @JsonValue
+    public String getOptionValue() {
+        return option;
+    }
+
     @Override
     public DecompressionEngine getDecompressionEngine() {
         return DECOMPRESSION_ENGINE_MAP.get(this.option);

data-prepper-plugins/decompress-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/decompress/DecompressionTypeTest.java

@@ -34,6 +34,12 @@ void getDecompressionEngine_returns_expected_DecompressionEngine(final Decompres
         assertThat(enumValue.getDecompressionEngine(), instanceOf(decompressionEngineClass));
     }
 
+    @ParameterizedTest
+    @ArgumentsSource(EnumToStringNameArgumentsProvider.class)
+    void getOptionValue_returns_data_type_name(final DecompressionType decompressionType, final String optionValue) {
+        assertThat(decompressionType.getOptionValue(), equalTo(optionValue));
+    }
+
     private static class EnumToStringNameArgumentsProvider implements ArgumentsProvider {
         @Override
         public Stream<? extends Arguments> provideArguments(final ExtensionContext context) {

data-prepper-plugins/dissect-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/dissect/DissectProcessorConfig.java

@@ -10,23 +10,31 @@
 import java.util.Map;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `dissect` processor extracts values from an event and maps them to individual fields " +
-        "based on user-defined `dissect` patterns. The processor is well suited for field extraction from log " +
+@JsonClassDescription("The <code>dissect</code> processor extracts values from an event and maps them to individual fields " +
+        "based on user-defined <code>dissect</code> patterns. The processor is well suited for field extraction from log " +
         "messages with a known structure.")
 public class DissectProcessorConfig {
     @NotNull
     @JsonProperty("map")
-    @JsonPropertyDescription("Defines the <code>dissect</code> patterns for specific keys. For details on how to define fields " +
-            "in the <code>dissect</code> pattern, see (https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/processors/dissect/#field-notations).")
+    @JsonPropertyDescription("Defines the <code>dissect</code> patterns for specific keys. " +
+            "Each key is a field name, and the value is the dissect pattern to use for dissecting it. " +
+            "For details on how to define fields " +
+            "in the <code>dissect</code> pattern, see (https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/processors/dissect/#field-notations). " +
+            "An example dissect pattern is <code>%{Date} %{Time} %{Log_Type}: %{Message}</code>, which will dissect into four fields.")
     private Map<String, String> map;
+
     @JsonProperty("target_types")
-    @JsonPropertyDescription("Specifies the data types for extract fields. Valid options are <code>integer</code>, " +
-            "<code>double</code>, <code>string</code>, <code>long</code>, <code>big_decimal</code>, and <code>boolean</code>. By default, all fields are of the <code>string</code> type.")
+    @JsonPropertyDescription("Specifies the data types for extract fields. " +
+            "Each key is a field name, and the value is the data type to use for that field. " +
+            "Valid data types are <code>integer</code>, <code>double</code>, <code>string</code>, <code>long</code>, <code>big_decimal</code>, and <code>boolean</code>. " +
+            "By default, all fields are treated as <code>string</code>.")
     private Map<String, TargetType> targetTypes;
+
     @JsonProperty("dissect_when")
     @JsonPropertyDescription("Specifies a condition for performing the <code>dissect</code> operation using a " +
-            "<a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">Data Prepper Expression Syntax</a>. " +
-            "If specified, the <code>dissect</code> operation will only run when the expression evaluates to true.")
+            "<a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>. " +
+            "If specified, the <code>dissect</code> operation will only run when the expression evaluates to true. " +
+            "For example, <code>'/some_value == \"log\"'</code>.")
     private String dissectWhen;
 
     public String getDissectWhen(){