Added tests for Date Processor

Signed-off-by: Krishna Kondaka <[email protected]>
kkondaka · Nov 3, 2023 · 72eda21 · 72eda21
1 parent 16cfa3f
commit 72eda21
Show file tree

Hide file tree

Showing 4 changed files with 39 additions and 5 deletions.
diff --git a/data-prepper-plugins/date-processor/README.md b/data-prepper-plugins/date-processor/README.md
@@ -104,6 +104,8 @@ processor:
     * Type: String
     * Default: `Locale.ROOT`
 
+* `to_origination_metadata` (Optional): When this option is used, matched time is put into the event's metadata as an instance of `Instant`.
+
 ## Metrics
 
 * `dateProcessingMatchSuccessCounter`: Number of records that match with at least one pattern specified in match configuration option.

diff --git a/...cessor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessor.java b/...cessor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessor.java
@@ -78,10 +78,11 @@ else if (keyToParse != null && !keyToParse.isEmpty()) {
                     Instant timeStamp = result.getRight();
                     if (dateProcessorConfig.getToOriginationMetadata()) {
                         Event event = (Event)record.getData();
+                        event.getMetadata().setExternalOriginationTime(timeStamp);
                         event.getEventHandle().setExternalOriginationTime(timeStamp);
                     }
-                    populateDateProcessorMetrics(zonedDateTime);
                 }
+                populateDateProcessorMetrics(zonedDateTime);
             }
 
             if (zonedDateTime != null)

diff --git a/.../test/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfigTest.java b/.../test/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfigTest.java
@@ -51,8 +51,9 @@ void isValidMatchAndFromTimestampReceived_should_return_true_if_from_time_receiv
         }
 
         @Test
-        void isValidMatchAndFromTimestampReceived_should_return_false_if_from_time_received_and_match_are_not_configured() {
-            assertThat(dateProcessorConfig.isValidMatchAndFromTimestampReceived(), equalTo(false));
+        void testToOriginationMetadata_should_return_true() throws NoSuchFieldException, IllegalAccessException {
+            reflectivelySetField(dateProcessorConfig, "toOriginationMetadata", true);
+            assertThat(dateProcessorConfig.getToOriginationMetadata(), equalTo(true));
         }
 
         @Test
@@ -178,4 +179,4 @@ private void reflectivelySetField(final DateProcessorConfig dateProcessorConfig,
             field.setAccessible(false);
         }
     }
-}
+}
diff --git a/...r/src/test/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorTests.java b/...r/src/test/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorTests.java
@@ -83,7 +83,8 @@ void setup() {
         lenient().when(pluginMetrics.counter(DateProcessor.DATE_PROCESSING_MATCH_SUCCESS)).thenReturn(dateProcessingMatchSuccessCounter);
         lenient().when(pluginMetrics.counter(DateProcessor.DATE_PROCESSING_MATCH_FAILURE)).thenReturn(dateProcessingMatchFailureCounter);
         when(mockDateProcessorConfig.getDateWhen()).thenReturn(null);
-        expectedDateTime = LocalDateTime.now();
+        expectedInstant = Instant.now();
+        expectedDateTime = LocalDateTime.ofInstant(expectedInstant, ZoneId.systemDefault());
     }
 
     @AfterEach
@@ -361,6 +362,35 @@ void match_with_different_year_formats_test(String pattern) {
         verify(dateProcessingMatchSuccessCounter, times(1)).increment();
     }
 
+    @ParameterizedTest
+    @ValueSource(strings = {"yyyy MM dd HH mm ss"})
+    void match_with_to_origination_metadata(String pattern) {
+        when(mockDateMatch.getKey()).thenReturn("logDate");
+        when(mockDateMatch.getPatterns()).thenReturn(Collections.singletonList(pattern));
+
+        List<DateProcessorConfig.DateMatch> dateMatches = Collections.singletonList(mockDateMatch);
+        when(mockDateProcessorConfig.getMatch()).thenReturn(dateMatches);
+        when(mockDateProcessorConfig.getSourceZoneId()).thenReturn(ZoneId.systemDefault());
+        when(mockDateProcessorConfig.getDestinationZoneId()).thenReturn(ZoneId.systemDefault());
+        when(mockDateProcessorConfig.getSourceLocale()).thenReturn(Locale.ROOT);
+        when(mockDateProcessorConfig.getToOriginationMetadata()).thenReturn(true);
+
+        dateProcessor = createObjectUnderTest();
+
+        Map<String, Object> testData = getTestData();
+        testData.put("logDate", expectedDateTime.format(DateTimeFormatter.ofPattern(pattern)));
+
+        final Record<Event> record = buildRecordWithEvent(testData);
+        final List<Record<Event>> processedRecords = (List<Record<Event>>) dateProcessor.doExecute(Collections.singletonList(record));
+
+        Event event = (Event)processedRecords.get(0).getData();
+        Assertions.assertTrue(event.getMetadata().getExternalOriginationTime() != null);
+        Assertions.assertTrue(event.getEventHandle().getExternalOriginationTime() != null);
+        ZonedDateTime expectedZonedDatetime = expectedDateTime.atZone(mockDateProcessorConfig.getSourceZoneId()).truncatedTo(ChronoUnit.SECONDS);
+        Assertions.assertTrue(expectedZonedDatetime.toString().equals(event.getMetadata().getExternalOriginationTime().toString()+"[UTC]"));
+        verify(dateProcessingMatchSuccessCounter, times(1)).increment();
+    }
+
     @ParameterizedTest
     @ValueSource(strings = {"MMM/dd", "MM dd"})
     void match_without_year_test(String pattern) {