diff --git a/build.gradle b/build.gradle index f4bbccbcc2..3dccd497cf 100644 --- a/build.gradle +++ b/build.gradle @@ -69,7 +69,7 @@ subprojects { } } dependencies { - implementation platform('com.fasterxml.jackson:jackson-bom:2.16.1') + implementation platform('com.fasterxml.jackson:jackson-bom:2.17.2') implementation platform('org.eclipse.jetty:jetty-bom:9.4.53.v20231009') implementation platform('io.micrometer:micrometer-bom:1.10.5') implementation libs.guava.core @@ -226,6 +226,9 @@ subprojects { test { useJUnitPlatform() + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.current() + } reports { junitXml.required html.required diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/sink/AbstractSink.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/sink/AbstractSink.java index 1c3e596265..26dd7e98a6 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/sink/AbstractSink.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/sink/AbstractSink.java @@ -28,6 +28,7 @@ public abstract class AbstractSink> implements Sink { private Thread retryThread; private int maxRetries; private int waitTimeMs; + private SinkThread sinkThread; public AbstractSink(final PluginSetting pluginSetting, int numRetries, int waitTimeMs) { this.pluginMetrics = PluginMetrics.fromPluginSetting(pluginSetting); @@ -51,7 +52,8 @@ public void initialize() { // the exceptions which are not retryable. doInitialize(); if (!isReady() && retryThread == null) { - retryThread = new Thread(new SinkThread(this, maxRetries, waitTimeMs)); + sinkThread = new SinkThread(this, maxRetries, waitTimeMs); + retryThread = new Thread(sinkThread); retryThread.start(); } } @@ -76,7 +78,7 @@ public void output(Collection records) { @Override public void shutdown() { if (retryThread != null) { - retryThread.stop(); + sinkThread.stop(); } } diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/sink/SinkThread.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/sink/SinkThread.java index c304de37af..451cef7dff 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/sink/SinkThread.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/sink/SinkThread.java @@ -10,6 +10,8 @@ class SinkThread implements Runnable { private int maxRetries; private int waitTimeMs; + private volatile boolean isStopped = false; + public SinkThread(AbstractSink sink, int maxRetries, int waitTimeMs) { this.sink = sink; this.maxRetries = maxRetries; @@ -19,11 +21,15 @@ public SinkThread(AbstractSink sink, int maxRetries, int waitTimeMs) { @Override public void run() { int numRetries = 0; - while (!sink.isReady() && numRetries++ < maxRetries) { + while (!sink.isReady() && numRetries++ < maxRetries && !isStopped) { try { Thread.sleep(waitTimeMs); sink.doInitialize(); } catch (InterruptedException e){} } } + + public void stop() { + isStopped = true; + } } diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/metrics/MetricsTestUtil.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/metrics/MetricsTestUtil.java index a77d9de349..f6c0602f9e 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/metrics/MetricsTestUtil.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/metrics/MetricsTestUtil.java @@ -6,25 +6,37 @@ package org.opensearch.dataprepper.metrics; import io.micrometer.core.instrument.Measurement; +import io.micrometer.core.instrument.Meter; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.Statistic; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.StreamSupport; public class MetricsTestUtil { - public static void initMetrics() { - Metrics.globalRegistry.getRegistries().forEach(meterRegistry -> Metrics.globalRegistry.remove(meterRegistry)); - Metrics.globalRegistry.getMeters().forEach(meter -> Metrics.globalRegistry.remove(meter)); + public static synchronized void initMetrics() { + final Set registries = new HashSet<>(Metrics.globalRegistry.getRegistries()); + registries.forEach(Metrics.globalRegistry::remove); + + final List meters = new ArrayList<>(Metrics.globalRegistry.getMeters()); + meters.forEach(Metrics.globalRegistry::remove); + Metrics.addRegistry(new SimpleMeterRegistry()); } - public static List getMeasurementList(final String meterName) { - return StreamSupport.stream(getRegistry().find(meterName).meter().measure().spliterator(), false) + public static synchronized List getMeasurementList(final String meterName) { + final Meter meter = getRegistry().find(meterName).meter(); + if(meter == null) + throw new RuntimeException("No metrics meter is available for " + meterName); + + return StreamSupport.stream(meter.measure().spliterator(), false) .collect(Collectors.toList()); } diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/sink/AbstractSinkTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/sink/AbstractSinkTest.java index 3b9fe7c007..8d1af7ea44 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/sink/AbstractSinkTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/sink/AbstractSinkTest.java @@ -11,15 +11,10 @@ import org.opensearch.dataprepper.metrics.MetricNames; import org.opensearch.dataprepper.metrics.MetricsTestUtil; import org.opensearch.dataprepper.model.configuration.PluginSetting; -import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.event.JacksonEvent; import org.opensearch.dataprepper.model.event.EventHandle; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.when; -import static org.mockito.Mockito.mock; +import org.opensearch.dataprepper.model.event.JacksonEvent; +import org.opensearch.dataprepper.model.record.Record; import java.time.Duration; import java.util.Arrays; @@ -30,6 +25,12 @@ import java.util.UUID; import static org.awaitility.Awaitility.await; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; class AbstractSinkTest { private int count; @@ -71,13 +72,13 @@ void testMetrics() { } @Test - void testSinkNotReady() { + void testSinkNotReady() throws InterruptedException { final String sinkName = "testSink"; final String pipelineName = "pipelineName"; MetricsTestUtil.initMetrics(); PluginSetting pluginSetting = new PluginSetting(sinkName, Collections.emptyMap()); pluginSetting.setPipelineName(pipelineName); - AbstractSink> abstractSink = new AbstractSinkNotReadyImpl(pluginSetting); + AbstractSinkNotReadyImpl abstractSink = new AbstractSinkNotReadyImpl(pluginSetting); abstractSink.initialize(); assertEquals(abstractSink.isReady(), false); assertEquals(abstractSink.getRetryThreadState(), Thread.State.RUNNABLE); @@ -87,7 +88,10 @@ void testSinkNotReady() { await().atMost(Duration.ofSeconds(5)) .until(abstractSink::isReady); assertEquals(abstractSink.getRetryThreadState(), Thread.State.TERMINATED); + int initCountBeforeShutdown = abstractSink.initCount; abstractSink.shutdown(); + Thread.sleep(200); + assertThat(abstractSink.initCount, equalTo(initCountBeforeShutdown)); } @Test diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 429e07069c..c939129a1c 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -48,7 +48,6 @@ dependencies { exclude group: 'commons-logging', module: 'commons-logging' } implementation 'software.amazon.cloudwatchlogs:aws-embedded-metrics:2.0.0-beta-1' - testImplementation 'org.apache.logging.log4j:log4j-jpl:2.23.0' testImplementation testLibs.spring.test implementation libs.armeria.core implementation libs.armeria.grpc @@ -60,7 +59,6 @@ dependencies { implementation 'software.amazon.awssdk:servicediscovery' implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' testImplementation testLibs.junit.vintage - testImplementation testLibs.mockito.inline testImplementation libs.commons.lang3 testImplementation project(':data-prepper-test-event') testImplementation project(':data-prepper-test-common') @@ -90,8 +88,6 @@ task integrationTest(type: Test) { classpath = sourceSets.integrationTest.runtimeClasspath - systemProperty 'log4j.configurationFile', 'src/test/resources/log4j2.properties' - filter { includeTestsMatching '*IT' } diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/peerforwarder/discovery/DnsPeerListProviderTest.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/peerforwarder/discovery/DnsPeerListProviderTest.java index 1083eea9f0..3bdee15368 100644 --- a/data-prepper-core/src/test/java/org/opensearch/dataprepper/peerforwarder/discovery/DnsPeerListProviderTest.java +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/peerforwarder/discovery/DnsPeerListProviderTest.java @@ -7,30 +7,33 @@ import com.linecorp.armeria.client.Endpoint; import com.linecorp.armeria.client.endpoint.dns.DnsAddressEndpointGroup; -import io.micrometer.core.instrument.Measurement; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; -import org.opensearch.dataprepper.metrics.MetricNames; -import org.opensearch.dataprepper.metrics.MetricsTestUtil; +import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.peerforwarder.HashRing; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.StringJoiner; import java.util.concurrent.CompletableFuture; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import java.util.function.ToDoubleFunction; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.peerforwarder.discovery.PeerListProvider.PEER_ENDPOINTS; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class DnsPeerListProviderTest { private static final String ENDPOINT_1 = "10.1.1.1"; @@ -39,8 +42,6 @@ public class DnsPeerListProviderTest { Endpoint.of(ENDPOINT_1), Endpoint.of(ENDPOINT_2) ); - private static final String COMPONENT_SCOPE = "testComponentScope"; - private static final String COMPONENT_ID = "testComponentId"; @Mock private DnsAddressEndpointGroup dnsAddressEndpointGroup; @@ -48,34 +49,33 @@ public class DnsPeerListProviderTest { @Mock private HashRing hashRing; + @Mock private PluginMetrics pluginMetrics; private CompletableFuture completableFuture; private DnsPeerListProvider dnsPeerListProvider; - @Before + @BeforeEach public void setup() { - MetricsTestUtil.initMetrics(); completableFuture = CompletableFuture.completedFuture(null); when(dnsAddressEndpointGroup.whenReady()).thenReturn(completableFuture); - pluginMetrics = PluginMetrics.fromNames(COMPONENT_ID, COMPONENT_SCOPE); dnsPeerListProvider = new DnsPeerListProvider(dnsAddressEndpointGroup, pluginMetrics); } - @Test(expected = NullPointerException.class) + @Test public void testDefaultListProviderWithNullHostname() { - new DnsPeerListProvider(null, pluginMetrics); + assertThrows(NullPointerException.class, () -> new DnsPeerListProvider(null, pluginMetrics)); } - @Test(expected = RuntimeException.class) + @Test public void testConstructWithInterruptedException() throws Exception { CompletableFuture mockFuture = mock(CompletableFuture.class); when(mockFuture.get()).thenThrow(new InterruptedException()); when(dnsAddressEndpointGroup.whenReady()).thenReturn(mockFuture); - new DnsPeerListProvider(dnsAddressEndpointGroup, pluginMetrics); + assertThrows(RuntimeException.class, () -> new DnsPeerListProvider(dnsAddressEndpointGroup, pluginMetrics)); } @Test @@ -90,17 +90,27 @@ public void testGetPeerList() { } @Test - public void testActivePeerCounter() { + public void testActivePeerCounter_with_list() { when(dnsAddressEndpointGroup.endpoints()).thenReturn(ENDPOINT_LIST); - final List endpointsMeasures = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(COMPONENT_SCOPE).add(COMPONENT_ID) - .add(PeerListProvider.PEER_ENDPOINTS).toString()); - assertEquals(1, endpointsMeasures.size()); - final Measurement endpointsMeasure = endpointsMeasures.get(0); - assertEquals(2.0, endpointsMeasure.getValue(), 0); + final ArgumentCaptor> gaugeFunctionCaptor = ArgumentCaptor.forClass(ToDoubleFunction.class); + verify(pluginMetrics).gauge(eq(PEER_ENDPOINTS), eq(dnsAddressEndpointGroup), gaugeFunctionCaptor.capture()); + + final ToDoubleFunction gaugeFunction = gaugeFunctionCaptor.getValue(); + assertThat(gaugeFunction.applyAsDouble(dnsAddressEndpointGroup), equalTo(2.0)); + } + + @Test + public void testActivePeerCounter_with_single() { when(dnsAddressEndpointGroup.endpoints()).thenReturn(Collections.singletonList(Endpoint.of(ENDPOINT_1))); - assertEquals(1.0, endpointsMeasure.getValue(), 0); + + final ArgumentCaptor> gaugeFunctionCaptor = ArgumentCaptor.forClass(ToDoubleFunction.class); + verify(pluginMetrics).gauge(eq(PEER_ENDPOINTS), eq(dnsAddressEndpointGroup), gaugeFunctionCaptor.capture()); + + final ToDoubleFunction gaugeFunction = gaugeFunctionCaptor.getValue(); + + assertThat(gaugeFunction.applyAsDouble(dnsAddressEndpointGroup), equalTo(1.0)); } @Test diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/peerforwarder/discovery/StaticPeerListProviderTest.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/peerforwarder/discovery/StaticPeerListProviderTest.java index 14bc836e36..589329b108 100644 --- a/data-prepper-core/src/test/java/org/opensearch/dataprepper/peerforwarder/discovery/StaticPeerListProviderTest.java +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/peerforwarder/discovery/StaticPeerListProviderTest.java @@ -5,56 +5,58 @@ package org.opensearch.dataprepper.peerforwarder.discovery; -import io.micrometer.core.instrument.Measurement; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; -import org.opensearch.dataprepper.metrics.MetricNames; -import org.opensearch.dataprepper.metrics.MetricsTestUtil; +import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.peerforwarder.HashRing; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.StringJoiner; - -import static org.junit.Assert.assertEquals; +import java.util.function.ToDoubleFunction; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; +import static org.opensearch.dataprepper.peerforwarder.discovery.PeerListProvider.PEER_ENDPOINTS; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class StaticPeerListProviderTest { private static final String ENDPOINT_1 = "10.10.0.1"; private static final String ENDPOINT_2 = "10.10.0.2"; private static final List ENDPOINT_LIST = Arrays.asList(ENDPOINT_1, ENDPOINT_2); - private static final String COMPONENT_SCOPE = "testComponentScope"; - private static final String COMPONENT_ID = "testComponentId"; @Mock private HashRing hashRing; + @Mock private PluginMetrics pluginMetrics; private StaticPeerListProvider staticPeerListProvider; - @Before + @BeforeEach public void setup() { - MetricsTestUtil.initMetrics(); - pluginMetrics = PluginMetrics.fromNames(COMPONENT_ID, COMPONENT_SCOPE); staticPeerListProvider = new StaticPeerListProvider(ENDPOINT_LIST, pluginMetrics); } - @Test(expected = RuntimeException.class) + @Test public void testListProviderWithEmptyList() { - new StaticPeerListProvider(Collections.emptyList(), pluginMetrics); + assertThrows(RuntimeException.class, () -> new StaticPeerListProvider(Collections.emptyList(), pluginMetrics)); } - @Test(expected = RuntimeException.class) + @Test public void testListProviderWithNullList() { - new StaticPeerListProvider(null, pluginMetrics); + assertThrows(RuntimeException.class, () -> new StaticPeerListProvider(null, pluginMetrics)); } @Test @@ -65,11 +67,12 @@ public void testListProviderWithNonEmptyList() { @Test public void testActivePeerCounter() { - final List endpointsMeasures = MetricsTestUtil.getMeasurementList( - new StringJoiner(MetricNames.DELIMITER).add(COMPONENT_SCOPE).add(COMPONENT_ID).add(PeerListProvider.PEER_ENDPOINTS).toString()); - assertEquals(1, endpointsMeasures.size()); - final Measurement endpointsMeasure = endpointsMeasures.get(0); - assertEquals(2.0, endpointsMeasure.getValue(), 0); + final ArgumentCaptor>> gaugeFunctionCaptor = ArgumentCaptor.forClass(ToDoubleFunction.class); + verify(pluginMetrics).gauge(eq(PEER_ENDPOINTS), any(List.class), gaugeFunctionCaptor.capture()); + + final ToDoubleFunction> gaugeFunction = gaugeFunctionCaptor.getValue(); + + assertThat(gaugeFunction.applyAsDouble(ENDPOINT_LIST), equalTo(2.0)); } @Test diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/pipeline/PipelineConnectorTest.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/pipeline/PipelineConnectorTest.java index fb54d532b7..e2af218c25 100644 --- a/data-prepper-core/src/test/java/org/opensearch/dataprepper/pipeline/PipelineConnectorTest.java +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/pipeline/PipelineConnectorTest.java @@ -23,7 +23,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.junit.MockitoJUnitRunner; import org.opensearch.dataprepper.plugins.buffer.blockingbuffer.BlockingBuffer; diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/pipeline/common/FutureHelperTest.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/pipeline/common/FutureHelperTest.java index c572766ac2..ba8a9714de 100644 --- a/data-prepper-core/src/test/java/org/opensearch/dataprepper/pipeline/common/FutureHelperTest.java +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/pipeline/common/FutureHelperTest.java @@ -9,7 +9,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.junit.MockitoJUnitRunner; import java.util.Arrays; import java.util.concurrent.ExecutionException; diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/pipeline/server/CloudWatchMeterRegistryProviderTest.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/pipeline/server/CloudWatchMeterRegistryProviderTest.java index 53db40d1a6..9dc744981b 100644 --- a/data-prepper-core/src/test/java/org/opensearch/dataprepper/pipeline/server/CloudWatchMeterRegistryProviderTest.java +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/pipeline/server/CloudWatchMeterRegistryProviderTest.java @@ -9,7 +9,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.junit.MockitoJUnitRunner; import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; import static org.hamcrest.CoreMatchers.notNullValue; diff --git a/data-prepper-core/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-core/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 23c33feb6d..0000000000 --- a/data-prepper-core/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1,3 +0,0 @@ -# To enable mocking of final classes with vanilla Mockito -# https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods -mock-maker-inline diff --git a/data-prepper-expression/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-expression/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 1f0955d450..0000000000 --- a/data-prepper-expression/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1 +0,0 @@ -mock-maker-inline diff --git a/data-prepper-logstash-configuration/build.gradle b/data-prepper-logstash-configuration/build.gradle index 6e328b7adc..002ae15516 100644 --- a/data-prepper-logstash-configuration/build.gradle +++ b/data-prepper-logstash-configuration/build.gradle @@ -25,7 +25,6 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-databind' implementation libs.commons.lang3 testImplementation testLibs.slf4j.simple - testImplementation testLibs.mockito.inline } generateGrammarSource { diff --git a/data-prepper-pipeline-parser/build.gradle b/data-prepper-pipeline-parser/build.gradle index 53b27d1e99..a94f63fc1d 100644 --- a/data-prepper-pipeline-parser/build.gradle +++ b/data-prepper-pipeline-parser/build.gradle @@ -30,12 +30,7 @@ dependencies { testImplementation testLibs.bundles.junit testImplementation testLibs.bundles.mockito testImplementation testLibs.hamcrest - testImplementation 'org.powermock:powermock-module-junit4:2.0.9' - testImplementation 'org.powermock:powermock-api-mockito2:2.0.9' testImplementation 'org.assertj:assertj-core:3.20.2' - testImplementation 'junit:junit:4.13.2' - testImplementation 'org.powermock:powermock-module-junit4:2.0.9' - testImplementation 'org.powermock:powermock-api-mockito2:2.0.9' compileOnly 'org.projectlombok:lombok:1.18.20' annotationProcessor 'org.projectlombok:lombok:1.18.20' } \ No newline at end of file diff --git a/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/EventKeyDeserializerTest.java b/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/EventKeyDeserializerTest.java index c727f0529a..240c14dd37 100644 --- a/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/EventKeyDeserializerTest.java +++ b/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/EventKeyDeserializerTest.java @@ -30,8 +30,8 @@ import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.sameInstance; import static org.hamcrest.MatcherAssert.assertThat; -import static org.powermock.api.mockito.PowerMockito.mock; -import static org.powermock.api.mockito.PowerMockito.when; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) class EventKeyDeserializerTest { diff --git a/data-prepper-plugin-framework/build.gradle b/data-prepper-plugin-framework/build.gradle index f77212a6b2..14f03fe15d 100644 --- a/data-prepper-plugin-framework/build.gradle +++ b/data-prepper-plugin-framework/build.gradle @@ -24,5 +24,4 @@ dependencies { } implementation libs.reflections.core implementation 'com.fasterxml.jackson.core:jackson-databind' - testImplementation testLibs.mockito.inline } \ No newline at end of file diff --git a/data-prepper-plugins/aggregate-processor/build.gradle b/data-prepper-plugins/aggregate-processor/build.gradle index 744986e924..9a3eb4551a 100644 --- a/data-prepper-plugins/aggregate-processor/build.gradle +++ b/data-prepper-plugins/aggregate-processor/build.gradle @@ -19,7 +19,6 @@ dependencies { implementation libs.opentelemetry.proto implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'io.micrometer:micrometer-core' - testImplementation testLibs.mockito.inline } jacocoTestCoverageVerification { diff --git a/data-prepper-plugins/armeria-common/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-plugins/armeria-common/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index ca6ee9cea8..0000000000 --- a/data-prepper-plugins/armeria-common/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1 +0,0 @@ -mock-maker-inline \ No newline at end of file diff --git a/data-prepper-plugins/avro-codecs/src/test/java/org/opensearch/dataprepper/avro/AvroAutoSchemaGeneratorTest.java b/data-prepper-plugins/avro-codecs/src/test/java/org/opensearch/dataprepper/avro/AvroAutoSchemaGeneratorTest.java index 622eb56a1b..1b66b62c37 100644 --- a/data-prepper-plugins/avro-codecs/src/test/java/org/opensearch/dataprepper/avro/AvroAutoSchemaGeneratorTest.java +++ b/data-prepper-plugins/avro-codecs/src/test/java/org/opensearch/dataprepper/avro/AvroAutoSchemaGeneratorTest.java @@ -17,7 +17,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Random; +import java.util.Timer; import java.util.UUID; import java.util.stream.Stream; @@ -218,7 +218,7 @@ static class SomeUnknownTypesArgumentsProvider implements ArgumentsProvider { @Override public Stream provideArguments(ExtensionContext context) { return Stream.of( - arguments(Random.class), + arguments(Timer.class), arguments(InputStream.class), arguments(File.class) ); diff --git a/data-prepper-plugins/aws-plugin/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-plugins/aws-plugin/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 23c33feb6d..0000000000 --- a/data-prepper-plugins/aws-plugin/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1,3 +0,0 @@ -# To enable mocking of final classes with vanilla Mockito -# https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods -mock-maker-inline diff --git a/data-prepper-plugins/blocking-buffer/src/test/java/org/opensearch/dataprepper/plugins/buffer/blockingbuffer/BlockingBufferTests.java b/data-prepper-plugins/blocking-buffer/src/test/java/org/opensearch/dataprepper/plugins/buffer/blockingbuffer/BlockingBufferTests.java index 194c810ec4..f3f28db174 100644 --- a/data-prepper-plugins/blocking-buffer/src/test/java/org/opensearch/dataprepper/plugins/buffer/blockingbuffer/BlockingBufferTests.java +++ b/data-prepper-plugins/blocking-buffer/src/test/java/org/opensearch/dataprepper/plugins/buffer/blockingbuffer/BlockingBufferTests.java @@ -328,7 +328,7 @@ public Stream provideArguments(final ExtensionContext conte return Stream.of( Arguments.of(0, randomInt + 1, 0.0), Arguments.of(1, 100, 1.0), - Arguments.of(randomInt, randomInt, 100.0), + Arguments.of(randomInt + 1, randomInt + 1, 100.0), Arguments.of(randomInt, randomInt + 250, ((double) randomInt / (randomInt + 250)) * 100), Arguments.of(6, 9, 66.66666666666666), Arguments.of(531, 1000, 53.1), diff --git a/data-prepper-plugins/cloudwatch-logs/build.gradle b/data-prepper-plugins/cloudwatch-logs/build.gradle index dc374997f0..3bbb24f443 100644 --- a/data-prepper-plugins/cloudwatch-logs/build.gradle +++ b/data-prepper-plugins/cloudwatch-logs/build.gradle @@ -16,7 +16,6 @@ dependencies { implementation 'org.projectlombok:lombok:1.18.26' implementation 'org.hibernate.validator:hibernate-validator:8.0.0.Final' testImplementation project(path: ':data-prepper-test-common') - testImplementation testLibs.mockito.inline compileOnly 'org.projectlombok:lombok:1.18.24' annotationProcessor 'org.projectlombok:lombok:1.18.24' } diff --git a/data-prepper-plugins/cloudwatch-metrics-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-plugins/cloudwatch-metrics-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 23c33feb6d..0000000000 --- a/data-prepper-plugins/cloudwatch-metrics-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1,3 +0,0 @@ -# To enable mocking of final classes with vanilla Mockito -# https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods -mock-maker-inline diff --git a/data-prepper-plugins/common/build.gradle b/data-prepper-plugins/common/build.gradle index aec7d7bddc..cdfdeab9ef 100644 --- a/data-prepper-plugins/common/build.gradle +++ b/data-prepper-plugins/common/build.gradle @@ -24,7 +24,6 @@ dependencies { testImplementation project(':data-prepper-plugins:blocking-buffer') testImplementation project(':data-prepper-test-event') testImplementation libs.commons.io - testImplementation testLibs.mockito.inline } jacocoTestCoverageVerification { diff --git a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/StringProcessor.java b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/StringProcessor.java index aa2930e634..3cf2953e06 100644 --- a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/StringProcessor.java +++ b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/StringProcessor.java @@ -5,6 +5,7 @@ package org.opensearch.dataprepper.plugins.processor; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.configuration.PluginSetting; @@ -40,6 +41,7 @@ public class StringProcessor implements Processor, Record> private final boolean upperCase; public static class Configuration { + @JsonPropertyDescription("Whether to convert to uppercase (`true`) or lowercase (`false`).") private boolean upperCase = true; public boolean getUpperCase() { diff --git a/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java b/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java index a74b2e9d38..aed3a38674 100644 --- a/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java +++ b/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import jakarta.validation.constraints.AssertTrue; import java.time.ZoneId; @@ -24,8 +25,16 @@ public class DateProcessorConfig { public static class DateMatch { @JsonProperty("key") + @JsonPropertyDescription("Represents the event key against which to match patterns. " + + "Required if `match` is configured. ") private String key; @JsonProperty("patterns") + @JsonPropertyDescription("A list of possible patterns that the timestamp value of the key can have. The patterns " + + "are based on a sequence of letters and symbols. The `patterns` support all the patterns listed in the " + + "Java [DatetimeFormatter](https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html) reference. " + + "The timestamp value also supports `epoch_second`, `epoch_milli`, and `epoch_nano` values, " + + "which represent the timestamp as the number of seconds, milliseconds, and nanoseconds since the epoch. " + + "Epoch values always use the UTC time zone.") private List patterns; public DateMatch() { @@ -82,30 +91,57 @@ public static boolean isValidPattern(final String pattern) { } @JsonProperty("from_time_received") + @JsonPropertyDescription("When `true`, the timestamp from the event metadata, " + + "which is the time at which the source receives the event, is added to the event data. " + + "This option cannot be defined at the same time as `match`. Default is `false`.") private Boolean fromTimeReceived = DEFAULT_FROM_TIME_RECEIVED; @JsonProperty("to_origination_metadata") + @JsonPropertyDescription("When `true`, the matched time is also added to the event's metadata as an instance of " + + "`Instant`. Default is `false`.") private Boolean toOriginationMetadata = DEFAULT_TO_ORIGINATION_METADATA; @JsonProperty("match") + @JsonPropertyDescription("The date match configuration. " + + "This option cannot be defined at the same time as `from_time_received`. There is no default value.") private List match; @JsonProperty("destination") + @JsonPropertyDescription("The field used to store the timestamp parsed by the date processor. " + + "Can be used with both `match` and `from_time_received`. Default is `@timestamp`.") private String destination = DEFAULT_DESTINATION; @JsonProperty("output_format") + @JsonPropertyDescription("Determines the format of the timestamp added to an event. " + + "Default is `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`.") private String outputFormat = DEFAULT_OUTPUT_FORMAT; @JsonProperty("source_timezone") + @JsonPropertyDescription("The time zone used to parse dates, including when the zone or offset cannot be extracted " + + "from the value. If the zone or offset are part of the value, then the time zone is ignored. " + + "A list of all the available time zones is contained in the **TZ database name** column of " + + "[the list of database time zones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List).") private String sourceTimezone = DEFAULT_SOURCE_TIMEZONE; @JsonProperty("destination_timezone") + @JsonPropertyDescription("The time zone used for storing the timestamp in the `destination` field. " + + "A list of all the available time zones is contained in the **TZ database name** column of " + + "[the list of database time zones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List).") private String destinationTimezone = DEFAULT_DESTINATION_TIMEZONE; @JsonProperty("locale") + @JsonPropertyDescription("The location used for parsing dates. Commonly used for parsing month names (`MMM`). " + + "The value can contain language, country, or variant fields in IETF BCP 47, such as `en-US`, " + + "or a string representation of the " + + "[locale](https://docs.oracle.com/javase/8/docs/api/java/util/Locale.html) object, such as `en_US`. " + + "A full list of locale fields, including language, country, and variant, can be found in " + + "[the language subtag registry](https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry). " + + "Default is `Locale.ROOT`.") private String locale; @JsonProperty("date_when") + @JsonPropertyDescription("Specifies under what condition the `date` processor should perform matching. " + + "Default is no condition.") private String dateWhen; @JsonIgnore diff --git a/data-prepper-plugins/decompress-processor/build.gradle b/data-prepper-plugins/decompress-processor/build.gradle index 9d67cffc3b..1068830a59 100644 --- a/data-prepper-plugins/decompress-processor/build.gradle +++ b/data-prepper-plugins/decompress-processor/build.gradle @@ -9,5 +9,4 @@ dependencies { implementation project(':data-prepper-plugins:common') implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'io.micrometer:micrometer-core' - testImplementation testLibs.mockito.inline } \ No newline at end of file diff --git a/data-prepper-plugins/dynamodb-source-coordination-store/build.gradle b/data-prepper-plugins/dynamodb-source-coordination-store/build.gradle index 4b9fb2a8f4..1912c2ae9b 100644 --- a/data-prepper-plugins/dynamodb-source-coordination-store/build.gradle +++ b/data-prepper-plugins/dynamodb-source-coordination-store/build.gradle @@ -10,7 +10,6 @@ dependencies { implementation 'software.amazon.awssdk:dynamodb' implementation 'software.amazon.awssdk:dynamodb-enhanced' implementation 'software.amazon.awssdk:sts' - testImplementation testLibs.mockito.inline } test { diff --git a/data-prepper-plugins/dynamodb-source/build.gradle b/data-prepper-plugins/dynamodb-source/build.gradle index 8fdc037470..3b3046434a 100644 --- a/data-prepper-plugins/dynamodb-source/build.gradle +++ b/data-prepper-plugins/dynamodb-source/build.gradle @@ -25,6 +25,5 @@ dependencies { implementation project(path: ':data-prepper-plugins:buffer-common') - testImplementation testLibs.mockito.inline testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' } \ No newline at end of file diff --git a/data-prepper-plugins/event-json-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/event_json/EventJsonInputCodecTest.java b/data-prepper-plugins/event-json-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/event_json/EventJsonInputCodecTest.java index f85d1c6605..a4b0377963 100644 --- a/data-prepper-plugins/event-json-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/event_json/EventJsonInputCodecTest.java +++ b/data-prepper-plugins/event-json-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/event_json/EventJsonInputCodecTest.java @@ -11,9 +11,12 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; + import static org.mockito.Mockito.when; import static org.mockito.Mockito.mock; + import org.mockito.Mock; + import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.CoreMatchers.not; @@ -28,6 +31,7 @@ import java.io.ByteArrayInputStream; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.List; import java.util.LinkedList; import java.util.Map; @@ -56,7 +60,7 @@ public EventJsonInputCodec createInputCodec() { @ParameterizedTest @ValueSource(strings = {"", "{}"}) public void emptyTest(String input) throws Exception { - input = "{\""+EventJsonDefines.VERSION+"\":\""+DataPrepperVersion.getCurrentVersion().toString()+"\", \""+EventJsonDefines.EVENTS+"\":["+input+"]}"; + input = "{\"" + EventJsonDefines.VERSION + "\":\"" + DataPrepperVersion.getCurrentVersion().toString() + "\", \"" + EventJsonDefines.EVENTS + "\":[" + input + "]}"; ByteArrayInputStream inputStream = new ByteArrayInputStream(input.getBytes()); inputCodec = createInputCodec(); Consumer> consumer = mock(Consumer.class); @@ -70,15 +74,15 @@ public void inCompatibleVersionTest() throws Exception { final String key = UUID.randomUUID().toString(); final String value = UUID.randomUUID().toString(); Map data = Map.of(key, value); - Instant startTime = Instant.now(); + Instant startTime = Instant.now().truncatedTo(ChronoUnit.MICROS); Event event = createEvent(data, startTime); Map dataMap = event.toMap(); Map metadataMap = objectMapper.convertValue(event.getMetadata(), Map.class); - String input = "{\""+EventJsonDefines.VERSION+"\":\"3.0\", \""+EventJsonDefines.EVENTS+"\":["; + String input = "{\"" + EventJsonDefines.VERSION + "\":\"3.0\", \"" + EventJsonDefines.EVENTS + "\":["; String comma = ""; for (int i = 0; i < 2; i++) { - input += comma+"{\"data\":"+objectMapper.writeValueAsString(dataMap)+","+"\"metadata\":"+objectMapper.writeValueAsString(metadataMap)+"}"; + input += comma + "{\"data\":" + objectMapper.writeValueAsString(dataMap) + "," + "\"metadata\":" + objectMapper.writeValueAsString(metadataMap) + "}"; comma = ","; } input += "]}"; @@ -95,15 +99,15 @@ public void basicTest() throws Exception { final String key = UUID.randomUUID().toString(); final String value = UUID.randomUUID().toString(); Map data = Map.of(key, value); - Instant startTime = Instant.now(); + Instant startTime = Instant.now().truncatedTo(ChronoUnit.MICROS); Event event = createEvent(data, startTime); Map dataMap = event.toMap(); Map metadataMap = objectMapper.convertValue(event.getMetadata(), Map.class); - String input = "{\""+EventJsonDefines.VERSION+"\":\""+DataPrepperVersion.getCurrentVersion().toString()+"\", \""+EventJsonDefines.EVENTS+"\":["; + String input = "{\"" + EventJsonDefines.VERSION + "\":\"" + DataPrepperVersion.getCurrentVersion().toString() + "\", \"" + EventJsonDefines.EVENTS + "\":["; String comma = ""; for (int i = 0; i < 2; i++) { - input += comma+"{\"data\":"+objectMapper.writeValueAsString(dataMap)+","+"\"metadata\":"+objectMapper.writeValueAsString(metadataMap)+"}"; + input += comma + "{\"data\":" + objectMapper.writeValueAsString(dataMap) + "," + "\"metadata\":" + objectMapper.writeValueAsString(metadataMap) + "}"; comma = ","; } input += "]}"; @@ -111,8 +115,8 @@ public void basicTest() throws Exception { List> records = new LinkedList<>(); inputCodec.parse(inputStream, records::add); assertThat(records.size(), equalTo(2)); - for(Record record : records) { - Event e = (Event)record.getData(); + for (Record record : records) { + Event e = (Event) record.getData(); assertThat(e.get(key, String.class), equalTo(value)); assertThat(e.getMetadata().getTimeReceived(), equalTo(startTime)); assertThat(e.getMetadata().getTags().size(), equalTo(0)); @@ -126,15 +130,15 @@ public void test_with_timeReceivedOverridden() throws Exception { final String key = UUID.randomUUID().toString(); final String value = UUID.randomUUID().toString(); Map data = Map.of(key, value); - Instant startTime = Instant.now().minusSeconds(5); + Instant startTime = Instant.now().truncatedTo(ChronoUnit.MICROS).minusSeconds(5); Event event = createEvent(data, startTime); Map dataMap = event.toMap(); Map metadataMap = objectMapper.convertValue(event.getMetadata(), Map.class); - String input = "{\""+EventJsonDefines.VERSION+"\":\""+DataPrepperVersion.getCurrentVersion().toString()+"\", \""+EventJsonDefines.EVENTS+"\":["; + String input = "{\"" + EventJsonDefines.VERSION + "\":\"" + DataPrepperVersion.getCurrentVersion().toString() + "\", \"" + EventJsonDefines.EVENTS + "\":["; String comma = ""; for (int i = 0; i < 2; i++) { - input += comma+"{\"data\":"+objectMapper.writeValueAsString(dataMap)+","+"\"metadata\":"+objectMapper.writeValueAsString(metadataMap)+"}"; + input += comma + "{\"data\":" + objectMapper.writeValueAsString(dataMap) + "," + "\"metadata\":" + objectMapper.writeValueAsString(metadataMap) + "}"; comma = ","; } input += "]}"; @@ -142,8 +146,8 @@ public void test_with_timeReceivedOverridden() throws Exception { List> records = new LinkedList<>(); inputCodec.parse(inputStream, records::add); assertThat(records.size(), equalTo(2)); - for(Record record : records) { - Event e = (Event)record.getData(); + for (Record record : records) { + Event e = (Event) record.getData(); assertThat(e.get(key, String.class), equalTo(value)); assertThat(e.getMetadata().getTimeReceived(), not(equalTo(startTime))); assertThat(e.getMetadata().getTags().size(), equalTo(0)); @@ -159,7 +163,7 @@ private Event createEvent(final Map json, final Instant timeRece if (timeReceived != null) { logBuilder.withTimeReceived(timeReceived); } - final JacksonEvent event = (JacksonEvent)logBuilder.build(); + final JacksonEvent event = (JacksonEvent) logBuilder.build(); return event; } diff --git a/data-prepper-plugins/event-json-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/event_json/EventJsonInputOutputCodecTest.java b/data-prepper-plugins/event-json-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/event_json/EventJsonInputOutputCodecTest.java index 85e91e5a55..7ea8c49cd0 100644 --- a/data-prepper-plugins/event-json-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/event_json/EventJsonInputOutputCodecTest.java +++ b/data-prepper-plugins/event-json-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/event_json/EventJsonInputOutputCodecTest.java @@ -6,9 +6,12 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; + import static org.mockito.Mockito.when; import static org.mockito.Mockito.mock; + import org.mockito.Mock; + import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; @@ -22,6 +25,7 @@ import org.opensearch.dataprepper.model.log.JacksonLog; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.List; import java.util.LinkedList; import java.util.Map; @@ -64,7 +68,7 @@ public void basicTest() throws Exception { final String value = UUID.randomUUID().toString(); Map data = Map.of(key, value); - Instant startTime = Instant.now(); + Instant startTime = Instant.now().truncatedTo(ChronoUnit.MICROS); Event event = createEvent(data, startTime); outputCodec = createOutputCodec(); inputCodec = createInputCodec(); @@ -75,8 +79,8 @@ public void basicTest() throws Exception { inputCodec.parse(new ByteArrayInputStream(outputStream.toByteArray()), records::add); assertThat(records.size(), equalTo(1)); - for(Record record : records) { - Event e = (Event)record.getData(); + for (Record record : records) { + Event e = (Event) record.getData(); assertThat(e.get(key, String.class), equalTo(value)); assertThat(e.getMetadata().getTimeReceived(), equalTo(startTime)); assertThat(e.getMetadata().getTags().size(), equalTo(0)); @@ -90,7 +94,7 @@ public void multipleEventsTest() throws Exception { final String value = UUID.randomUUID().toString(); Map data = Map.of(key, value); - Instant startTime = Instant.now(); + Instant startTime = Instant.now().truncatedTo(ChronoUnit.MICROS); Event event = createEvent(data, startTime); outputCodec = createOutputCodec(); inputCodec = createInputCodec(); @@ -103,8 +107,8 @@ public void multipleEventsTest() throws Exception { inputCodec.parse(new ByteArrayInputStream(outputStream.toByteArray()), records::add); assertThat(records.size(), equalTo(3)); - for(Record record : records) { - Event e = (Event)record.getData(); + for (Record record : records) { + Event e = (Event) record.getData(); assertThat(e.get(key, String.class), equalTo(value)); assertThat(e.getMetadata().getTimeReceived(), equalTo(startTime)); assertThat(e.getMetadata().getTags().size(), equalTo(0)); @@ -122,7 +126,7 @@ public void extendedTest() throws Exception { Set tags = Set.of(UUID.randomUUID().toString(), UUID.randomUUID().toString()); List tagsList = tags.stream().collect(Collectors.toList()); - Instant startTime = Instant.now(); + Instant startTime = Instant.now().truncatedTo(ChronoUnit.MICROS); Event event = createEvent(data, startTime); Instant origTime = startTime.minusSeconds(5); event.getMetadata().setExternalOriginationTime(origTime); @@ -135,11 +139,11 @@ public void extendedTest() throws Exception { outputCodec.complete(outputStream); assertThat(outputCodec.getExtension(), equalTo(EventJsonOutputCodec.EVENT_JSON)); List> records = new LinkedList<>(); -inputCodec.parse(new ByteArrayInputStream(outputStream.toByteArray()), records::add); + inputCodec.parse(new ByteArrayInputStream(outputStream.toByteArray()), records::add); assertThat(records.size(), equalTo(1)); - for(Record record : records) { - Event e = (Event)record.getData(); + for (Record record : records) { + Event e = (Event) record.getData(); assertThat(e.get(key, String.class), equalTo(value)); assertThat(e.getMetadata().getTimeReceived(), equalTo(startTime)); assertThat(e.getMetadata().getTags(), equalTo(tags)); @@ -157,7 +161,7 @@ private Event createEvent(final Map json, final Instant timeRece if (timeReceived != null) { logBuilder.withTimeReceived(timeReceived); } - final JacksonEvent event = (JacksonEvent)logBuilder.build(); + final JacksonEvent event = (JacksonEvent) logBuilder.build(); return event; } diff --git a/data-prepper-plugins/event-json-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/event_json/EventJsonOutputCodecTest.java b/data-prepper-plugins/event-json-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/event_json/EventJsonOutputCodecTest.java index 51dda545cb..b32d2b62e9 100644 --- a/data-prepper-plugins/event-json-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/event_json/EventJsonOutputCodecTest.java +++ b/data-prepper-plugins/event-json-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/event_json/EventJsonOutputCodecTest.java @@ -11,6 +11,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; + import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; @@ -22,6 +23,7 @@ import org.opensearch.dataprepper.model.log.JacksonLog; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Map; import java.util.UUID; @@ -49,7 +51,7 @@ public void basicTest() throws Exception { final String value = UUID.randomUUID().toString(); Map data = Map.of(key, value); - Instant startTime = Instant.now(); + Instant startTime = Instant.now().truncatedTo(ChronoUnit.MICROS); Event event = createEvent(data, startTime); outputCodec = createOutputCodec(); outputCodec.start(outputStream, null, null); @@ -59,10 +61,10 @@ public void basicTest() throws Exception { Map dataMap = event.toMap(); Map metadataMap = objectMapper.convertValue(event.getMetadata(), Map.class); //String expectedOutput = "{\"version\":\""+DataPrepperVersion.getCurrentVersion().toString()+"\",\""+EventJsonDefines.EVENTS+"\":["; - String expectedOutput = "{\""+EventJsonDefines.VERSION+"\":\""+DataPrepperVersion.getCurrentVersion().toString()+"\",\""+EventJsonDefines.EVENTS+"\":["; + String expectedOutput = "{\"" + EventJsonDefines.VERSION + "\":\"" + DataPrepperVersion.getCurrentVersion().toString() + "\",\"" + EventJsonDefines.EVENTS + "\":["; String comma = ""; for (int i = 0; i < 2; i++) { - expectedOutput += comma+"{\""+EventJsonDefines.DATA+"\":"+objectMapper.writeValueAsString(dataMap)+","+"\""+EventJsonDefines.METADATA+"\":"+objectMapper.writeValueAsString(metadataMap)+"}"; + expectedOutput += comma + "{\"" + EventJsonDefines.DATA + "\":" + objectMapper.writeValueAsString(dataMap) + "," + "\"" + EventJsonDefines.METADATA + "\":" + objectMapper.writeValueAsString(metadataMap) + "}"; comma = ","; } expectedOutput += "]}"; @@ -78,7 +80,7 @@ private Event createEvent(final Map json, final Instant timeRece if (timeReceived != null) { logBuilder.withTimeReceived(timeReceived); } - final JacksonEvent event = (JacksonEvent)logBuilder.build(); + final JacksonEvent event = (JacksonEvent) logBuilder.build(); return event; } diff --git a/data-prepper-plugins/geoip-processor/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-plugins/geoip-processor/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 23c33feb6d..0000000000 --- a/data-prepper-plugins/geoip-processor/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1,3 +0,0 @@ -# To enable mocking of final classes with vanilla Mockito -# https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods -mock-maker-inline diff --git a/data-prepper-plugins/grok-processor/build.gradle b/data-prepper-plugins/grok-processor/build.gradle index 82a8306a5d..ae4a82a0ee 100644 --- a/data-prepper-plugins/grok-processor/build.gradle +++ b/data-prepper-plugins/grok-processor/build.gradle @@ -12,7 +12,6 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-databind' implementation "io.krakens:java-grok:0.1.9" implementation 'io.micrometer:micrometer-core' - testImplementation testLibs.mockito.inline testImplementation project(':data-prepper-test-common') } diff --git a/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java b/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java index 8b8b7f2e90..8cc9c6a716 100644 --- a/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java +++ b/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java @@ -12,10 +12,10 @@ import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Timer; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.annotations.SingleThread; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.processor.AbstractProcessor; import org.opensearch.dataprepper.model.processor.Processor; @@ -59,7 +59,7 @@ @SingleThread -@DataPrepperPlugin(name = "grok", pluginType = Processor.class) +@DataPrepperPlugin(name = "grok", pluginType = Processor.class, pluginConfigurationType = GrokProcessorConfig.class) public class GrokProcessor extends AbstractProcessor, Record> { static final long EXECUTOR_SERVICE_SHUTDOWN_TIMEOUT = 300L; @@ -89,20 +89,28 @@ public class GrokProcessor extends AbstractProcessor, Record(grokProcessorConfig.getkeysToOverwrite()); this.grokCompiler = grokCompiler; this.fieldToGrok = new LinkedHashMap<>(); this.executorService = executorService; this.expressionEvaluator = expressionEvaluator; this.tagsOnMatchFailure = grokProcessorConfig.getTagsOnMatchFailure(); - this.tagsOnTimeout = grokProcessorConfig.getTagsOnTimeout(); + this.tagsOnTimeout = grokProcessorConfig.getTagsOnTimeout().isEmpty() ? + grokProcessorConfig.getTagsOnMatchFailure() : grokProcessorConfig.getTagsOnTimeout(); grokProcessingMatchCounter = pluginMetrics.counter(GROK_PROCESSING_MATCH); grokProcessingMismatchCounter = pluginMetrics.counter(GROK_PROCESSING_MISMATCH); grokProcessingErrorsCounter = pluginMetrics.counter(GROK_PROCESSING_ERRORS); diff --git a/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorConfig.java b/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorConfig.java index de9daf91d5..2d2ae1ef41 100644 --- a/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorConfig.java +++ b/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorConfig.java @@ -5,8 +5,10 @@ package org.opensearch.dataprepper.plugins.processor.grok; -import org.opensearch.dataprepper.model.configuration.PluginSetting; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -39,69 +41,57 @@ public class GrokProcessorConfig { static final int DEFAULT_TIMEOUT_MILLIS = 30000; static final String DEFAULT_TARGET_KEY = null; - private final boolean breakOnMatch; - private final boolean keepEmptyCaptures; - private final Map> match; - private final boolean namedCapturesOnly; - private final List keysToOverwrite; - private final List patternsDirectories; - private final String patternsFilesGlob; - private final Map patternDefinitions; - private final int timeoutMillis; - private final String targetKey; - private final String grokWhen; - private final List tagsOnMatchFailure; - private final List tagsOnTimeout; - - private final boolean includePerformanceMetadata; - - private GrokProcessorConfig(final boolean breakOnMatch, - final boolean keepEmptyCaptures, - final Map> match, - final boolean namedCapturesOnly, - final List keysToOverwrite, - final List patternsDirectories, - final String patternsFilesGlob, - final Map patternDefinitions, - final int timeoutMillis, - final String targetKey, - final String grokWhen, - final List tagsOnMatchFailure, - final List tagsOnTimeout, - final boolean includePerformanceMetadata) { - - this.breakOnMatch = breakOnMatch; - this.keepEmptyCaptures = keepEmptyCaptures; - this.match = match; - this.namedCapturesOnly = namedCapturesOnly; - this.keysToOverwrite = keysToOverwrite; - this.patternsDirectories = patternsDirectories; - this.patternsFilesGlob = patternsFilesGlob; - this.patternDefinitions = patternDefinitions; - this.timeoutMillis = timeoutMillis; - this.targetKey = targetKey; - this.grokWhen = grokWhen; - this.tagsOnMatchFailure = tagsOnMatchFailure; - this.tagsOnTimeout = tagsOnTimeout.isEmpty() ? tagsOnMatchFailure : tagsOnTimeout; - this.includePerformanceMetadata = includePerformanceMetadata; - } - - public static GrokProcessorConfig buildConfig(final PluginSetting pluginSetting) { - return new GrokProcessorConfig(pluginSetting.getBooleanOrDefault(BREAK_ON_MATCH, DEFAULT_BREAK_ON_MATCH), - pluginSetting.getBooleanOrDefault(KEEP_EMPTY_CAPTURES, DEFAULT_KEEP_EMPTY_CAPTURES), - pluginSetting.getTypedListMap(MATCH, String.class, String.class), - pluginSetting.getBooleanOrDefault(NAMED_CAPTURES_ONLY, DEFAULT_NAMED_CAPTURES_ONLY), - pluginSetting.getTypedList(KEYS_TO_OVERWRITE, String.class), - pluginSetting.getTypedList(PATTERNS_DIRECTORIES, String.class), - pluginSetting.getStringOrDefault(PATTERNS_FILES_GLOB, DEFAULT_PATTERNS_FILES_GLOB), - pluginSetting.getTypedMap(PATTERN_DEFINITIONS, String.class, String.class), - pluginSetting.getIntegerOrDefault(TIMEOUT_MILLIS, DEFAULT_TIMEOUT_MILLIS), - pluginSetting.getStringOrDefault(TARGET_KEY, DEFAULT_TARGET_KEY), - pluginSetting.getStringOrDefault(GROK_WHEN, null), - pluginSetting.getTypedList(TAGS_ON_MATCH_FAILURE, String.class), - pluginSetting.getTypedList(TAGS_ON_TIMEOUT, String.class), - pluginSetting.getBooleanOrDefault(INCLUDE_PERFORMANCE_METADATA, false)); - } + @JsonProperty(BREAK_ON_MATCH) + @JsonPropertyDescription("Specifies whether to match all patterns (`false`) or stop once the first successful " + + "match is found (`true`). Default is `true`.") + private boolean breakOnMatch = DEFAULT_BREAK_ON_MATCH; + @JsonProperty(KEEP_EMPTY_CAPTURES) + @JsonPropertyDescription("Enables the preservation of `null` captures from the processed output. Default is `false`.") + private boolean keepEmptyCaptures = DEFAULT_KEEP_EMPTY_CAPTURES; + @JsonProperty(MATCH) + @JsonPropertyDescription("Specifies which keys should match specific patterns. Default is an empty response body.") + private Map> match = Collections.emptyMap(); + @JsonProperty(NAMED_CAPTURES_ONLY) + @JsonPropertyDescription("Specifies whether to keep only named captures. Default is `true`.") + private boolean namedCapturesOnly = DEFAULT_NAMED_CAPTURES_ONLY; + @JsonProperty(KEYS_TO_OVERWRITE) + @JsonPropertyDescription("Specifies which existing keys will be overwritten if there is a capture with the same key value. " + + "Default is `[]`.") + private List keysToOverwrite = Collections.emptyList(); + @JsonProperty(PATTERNS_DIRECTORIES) + @JsonPropertyDescription("Specifies which directory paths contain the custom pattern files. Default is an empty list.") + private List patternsDirectories = Collections.emptyList(); + @JsonProperty(PATTERNS_FILES_GLOB) + @JsonPropertyDescription("Specifies which pattern files to use from the directories specified for " + + "`pattern_directories`. Default is `*`.") + private String patternsFilesGlob = DEFAULT_PATTERNS_FILES_GLOB; + @JsonProperty(PATTERN_DEFINITIONS) + @JsonPropertyDescription("Allows for a custom pattern that can be used inline inside the response body. " + + "Default is an empty response body.") + private Map patternDefinitions = Collections.emptyMap(); + @JsonProperty(TIMEOUT_MILLIS) + @JsonPropertyDescription("The maximum amount of time during which matching occurs. " + + "Setting to `0` prevents any matching from occurring. Default is `30,000`.") + private int timeoutMillis = DEFAULT_TIMEOUT_MILLIS; + @JsonProperty(TARGET_KEY) + @JsonPropertyDescription("Specifies a parent-level key used to store all captures. Default value is `null`.") + private String targetKey = DEFAULT_TARGET_KEY; + @JsonProperty(GROK_WHEN) + @JsonPropertyDescription("Specifies under what condition the `grok` processor should perform matching. " + + "Default is no condition.") + private String grokWhen; + @JsonProperty(TAGS_ON_MATCH_FAILURE) + @JsonPropertyDescription("A `List` of `String`s that specifies the tags to be set in the event when grok fails to " + + "match or an unknown exception occurs while matching. This tag may be used in conditional expressions in " + + "other parts of the configuration") + private List tagsOnMatchFailure = Collections.emptyList(); + @JsonProperty(TAGS_ON_TIMEOUT) + @JsonPropertyDescription("A `List` of `String`s that specifies the tags to be set in the event when grok match times out.") + private List tagsOnTimeout = Collections.emptyList(); + @JsonProperty(INCLUDE_PERFORMANCE_METADATA) + @JsonPropertyDescription("A `Boolean` on whether to include performance metadata into event metadata, " + + "e.g. _total_grok_patterns_attempted, _total_grok_processing_time.") + private boolean includePerformanceMetadata = false; public boolean isBreakOnMatch() { return breakOnMatch; diff --git a/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorConfigTests.java b/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorConfigTests.java index eb69968a96..37c5ec9cb1 100644 --- a/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorConfigTests.java +++ b/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorConfigTests.java @@ -5,6 +5,7 @@ package org.opensearch.dataprepper.plugins.processor.grok; +import com.fasterxml.jackson.databind.ObjectMapper; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -27,6 +28,7 @@ import static org.opensearch.dataprepper.plugins.processor.grok.GrokProcessorConfig.DEFAULT_TIMEOUT_MILLIS; public class GrokProcessorConfigTests { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final String PLUGIN_NAME = "grok"; private static final Map> TEST_MATCH = new HashMap<>(); @@ -62,7 +64,8 @@ public static void setUp() { @Test public void testDefault() { - final GrokProcessorConfig grokProcessorConfig = GrokProcessorConfig.buildConfig(new PluginSetting(PLUGIN_NAME, null)); + final GrokProcessorConfig grokProcessorConfig = OBJECT_MAPPER.convertValue( + Collections.emptyMap(), GrokProcessorConfig.class); assertThat(grokProcessorConfig.isBreakOnMatch(), equalTo(DEFAULT_BREAK_ON_MATCH)); assertThat(grokProcessorConfig.isKeepEmptyCaptures(), equalTo(DEFAULT_KEEP_EMPTY_CAPTURES)); @@ -95,7 +98,8 @@ public void testValidConfig() { TEST_TARGET_KEY, true); - final GrokProcessorConfig grokProcessorConfig = GrokProcessorConfig.buildConfig(validPluginSetting); + final GrokProcessorConfig grokProcessorConfig = OBJECT_MAPPER.convertValue( + validPluginSetting.getSettings(), GrokProcessorConfig.class); assertThat(grokProcessorConfig.isBreakOnMatch(), equalTo(false)); assertThat(grokProcessorConfig.isKeepEmptyCaptures(), equalTo(true)); @@ -127,7 +131,8 @@ public void testInvalidConfig() { invalidPluginSetting.getSettings().put(GrokProcessorConfig.MATCH, TEST_INVALID_MATCH); - assertThrows(IllegalArgumentException.class, () -> GrokProcessorConfig.buildConfig(invalidPluginSetting)); + assertThrows(IllegalArgumentException.class, () -> OBJECT_MAPPER.convertValue( + invalidPluginSetting.getSettings(), GrokProcessorConfig.class)); } private PluginSetting completePluginSettingForGrokProcessor(final boolean breakOnMatch, @@ -160,33 +165,22 @@ private PluginSetting completePluginSettingForGrokProcessor(final boolean breakO @Test void getTagsOnMatchFailure_returns_tagOnMatch() { final List tagsOnMatch = List.of(UUID.randomUUID().toString(), UUID.randomUUID().toString()); - final GrokProcessorConfig objectUnderTest = GrokProcessorConfig.buildConfig(new PluginSetting(PLUGIN_NAME, - Map.of(GrokProcessorConfig.TAGS_ON_MATCH_FAILURE, tagsOnMatch) - )); + final GrokProcessorConfig objectUnderTest = OBJECT_MAPPER.convertValue( + Map.of(GrokProcessorConfig.TAGS_ON_MATCH_FAILURE, tagsOnMatch), GrokProcessorConfig.class); assertThat(objectUnderTest.getTagsOnMatchFailure(), equalTo(tagsOnMatch)); } - @Test - void getTagsOnTimeout_returns_tagsOnMatch_if_no_tagsOnTimeout() { - final List tagsOnMatch = List.of(UUID.randomUUID().toString(), UUID.randomUUID().toString()); - final GrokProcessorConfig objectUnderTest = GrokProcessorConfig.buildConfig(new PluginSetting(PLUGIN_NAME, - Map.of(GrokProcessorConfig.TAGS_ON_MATCH_FAILURE, tagsOnMatch) - )); - - assertThat(objectUnderTest.getTagsOnTimeout(), equalTo(tagsOnMatch)); - } - @Test void getTagsOnTimeout_returns_tagsOnTimeout_if_present() { final List tagsOnMatch = List.of(UUID.randomUUID().toString(), UUID.randomUUID().toString()); final List tagsOnTimeout = List.of(UUID.randomUUID().toString(), UUID.randomUUID().toString()); - final GrokProcessorConfig objectUnderTest = GrokProcessorConfig.buildConfig(new PluginSetting(PLUGIN_NAME, + final GrokProcessorConfig objectUnderTest = OBJECT_MAPPER.convertValue( Map.of( GrokProcessorConfig.TAGS_ON_MATCH_FAILURE, tagsOnMatch, GrokProcessorConfig.TAGS_ON_TIMEOUT, tagsOnTimeout - ) - )); + ), + GrokProcessorConfig.class); assertThat(objectUnderTest.getTagsOnTimeout(), equalTo(tagsOnTimeout)); } @@ -194,9 +188,8 @@ void getTagsOnTimeout_returns_tagsOnTimeout_if_present() { @Test void getTagsOnTimeout_returns_tagsOnTimeout_if_present_and_no_tagsOnMatch() { final List tagsOnTimeout = List.of(UUID.randomUUID().toString(), UUID.randomUUID().toString()); - final GrokProcessorConfig objectUnderTest = GrokProcessorConfig.buildConfig(new PluginSetting(PLUGIN_NAME, - Map.of(GrokProcessorConfig.TAGS_ON_TIMEOUT, tagsOnTimeout) - )); + final GrokProcessorConfig objectUnderTest = OBJECT_MAPPER.convertValue( + Map.of(GrokProcessorConfig.TAGS_ON_TIMEOUT, tagsOnTimeout), GrokProcessorConfig.class); assertThat(objectUnderTest.getTagsOnTimeout(), equalTo(tagsOnTimeout)); } diff --git a/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorIT.java b/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorIT.java index 1c8d0036c2..f6fa090405 100644 --- a/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorIT.java +++ b/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorIT.java @@ -16,6 +16,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mock; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.record.Record; @@ -38,6 +39,8 @@ public class GrokProcessorIT { private PluginSetting pluginSetting; + private PluginMetrics pluginMetrics; + private GrokProcessorConfig grokProcessorConfig; private GrokProcessor grokProcessor; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference>() {}; @@ -65,6 +68,8 @@ public void setup() { null); pluginSetting.setPipelineName("grokPipeline"); + grokProcessorConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), GrokProcessorConfig.class); + pluginMetrics = PluginMetrics.fromPluginSetting(pluginSetting); // This is a COMMONAPACHELOG pattern with the following format // COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) @@ -115,7 +120,8 @@ public void testMatchNoCapturesWithExistingAndNonExistingKey() throws JsonProces matchConfig.put("bad_key", Collections.singletonList(nonMatchingPattern)); pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); - grokProcessor = new GrokProcessor(pluginSetting, expressionEvaluator); + grokProcessorConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), GrokProcessorConfig.class); + grokProcessor = new GrokProcessor(pluginMetrics, grokProcessorConfig, expressionEvaluator); final Map testData = new HashMap(); testData.put("message", messageInput); @@ -135,7 +141,8 @@ public void testSingleMatchSinglePatternWithDefaults() throws JsonProcessingExce matchConfig.put("message", Collections.singletonList("%{COMMONAPACHELOG}")); pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); - grokProcessor = new GrokProcessor(pluginSetting, expressionEvaluator); + grokProcessorConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), GrokProcessorConfig.class); + grokProcessor = new GrokProcessor(pluginMetrics, grokProcessorConfig, expressionEvaluator); final Map testData = new HashMap(); testData.put("message", messageInput); @@ -173,7 +180,8 @@ public void testSingleMatchMultiplePatternWithBreakOnMatchFalse() throws JsonPro pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); pluginSetting.getSettings().put(GrokProcessorConfig.BREAK_ON_MATCH, false); - grokProcessor = new GrokProcessor(pluginSetting, expressionEvaluator); + grokProcessorConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), GrokProcessorConfig.class); + grokProcessor = new GrokProcessor(pluginMetrics, grokProcessorConfig, expressionEvaluator); final Map testData = new HashMap(); testData.put("message", messageInput); @@ -208,7 +216,8 @@ public void testSingleMatchTypeConversionWithDefaults() throws JsonProcessingExc matchConfig.put("message", Collections.singletonList("\"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response:int} (?:%{NUMBER:bytes:float}|-)")); pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); - grokProcessor = new GrokProcessor(pluginSetting, expressionEvaluator); + grokProcessorConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), GrokProcessorConfig.class); + grokProcessor = new GrokProcessor(pluginMetrics, grokProcessorConfig, expressionEvaluator); final Map testData = new HashMap(); testData.put("message", messageInput); @@ -240,7 +249,8 @@ public void testMultipleMatchWithBreakOnMatchFalse() throws JsonProcessingExcept pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); pluginSetting.getSettings().put(GrokProcessorConfig.BREAK_ON_MATCH, false); - grokProcessor = new GrokProcessor(pluginSetting, expressionEvaluator); + grokProcessorConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), GrokProcessorConfig.class); + grokProcessor = new GrokProcessor(pluginMetrics, grokProcessorConfig, expressionEvaluator); final Map testData = new HashMap(); testData.put("message", messageInput); @@ -278,7 +288,8 @@ public void testMatchWithKeepEmptyCapturesTrue() throws JsonProcessingException pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); pluginSetting.getSettings().put(GrokProcessorConfig.KEEP_EMPTY_CAPTURES, true); - grokProcessor = new GrokProcessor(pluginSetting, expressionEvaluator); + grokProcessorConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), GrokProcessorConfig.class); + grokProcessor = new GrokProcessor(pluginMetrics, grokProcessorConfig, expressionEvaluator); final Map testData = new HashMap(); testData.put("message", messageInput); @@ -314,7 +325,8 @@ public void testMatchWithNamedCapturesOnlyFalse() throws JsonProcessingException pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); pluginSetting.getSettings().put(GrokProcessorConfig.NAMED_CAPTURES_ONLY, false); - grokProcessor = new GrokProcessor(pluginSetting, expressionEvaluator); + grokProcessorConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), GrokProcessorConfig.class); + grokProcessor = new GrokProcessor(pluginMetrics, grokProcessorConfig, expressionEvaluator); final Map testData = new HashMap(); testData.put("message", "This is my greedy data before matching 192.0.2.1 123456"); @@ -346,7 +358,8 @@ public void testPatternDefinitions() throws JsonProcessingException { pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); pluginSetting.getSettings().put(GrokProcessorConfig.PATTERN_DEFINITIONS, patternDefinitions); - grokProcessor = new GrokProcessor(pluginSetting, expressionEvaluator); + grokProcessorConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), GrokProcessorConfig.class); + grokProcessor = new GrokProcessor(pluginMetrics, grokProcessorConfig, expressionEvaluator); final Map testData = new HashMap(); testData.put("message", "This is my greedy data before matching with my phone number 123-456-789"); @@ -389,7 +402,8 @@ public void testPatternsDirWithDefaultPatternsFilesGlob() throws JsonProcessingE pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); pluginSetting.getSettings().put(GrokProcessorConfig.PATTERNS_DIRECTORIES, patternsDirectories); - grokProcessor = new GrokProcessor(pluginSetting, expressionEvaluator); + grokProcessorConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), GrokProcessorConfig.class); + grokProcessor = new GrokProcessor(pluginMetrics, grokProcessorConfig, expressionEvaluator); final Record resultRecord = buildRecordWithEvent(resultData); @@ -422,7 +436,8 @@ public void testPatternsDirWithCustomPatternsFilesGlob() throws JsonProcessingEx pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); pluginSetting.getSettings().put(GrokProcessorConfig.PATTERNS_DIRECTORIES, patternsDirectories); pluginSetting.getSettings().put(GrokProcessorConfig.PATTERNS_FILES_GLOB, "*1.txt"); - grokProcessor = new GrokProcessor(pluginSetting, expressionEvaluator); + grokProcessorConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), GrokProcessorConfig.class); + grokProcessor = new GrokProcessor(pluginMetrics, grokProcessorConfig, expressionEvaluator); final Record resultRecord = buildRecordWithEvent(resultData); @@ -436,8 +451,10 @@ public void testPatternsDirWithCustomPatternsFilesGlob() throws JsonProcessingEx matchConfigWithPatterns2Pattern.put("message", Collections.singletonList("My birthday is %{CUSTOMBIRTHDAYPATTERN:my_birthday}")); pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfigWithPatterns2Pattern); + grokProcessorConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), GrokProcessorConfig.class); - Throwable throwable = assertThrows(IllegalArgumentException.class, () -> new GrokProcessor(pluginSetting, expressionEvaluator)); + Throwable throwable = assertThrows(IllegalArgumentException.class, () -> new GrokProcessor( + pluginMetrics, grokProcessorConfig, expressionEvaluator)); assertThat("No definition for key 'CUSTOMBIRTHDAYPATTERN' found, aborting", equalTo(throwable.getMessage())); } @@ -447,7 +464,8 @@ public void testMatchWithNamedCapturesSyntax() throws JsonProcessingException { matchConfig.put("message", Collections.singletonList("%{GREEDYDATA:greedy_data} (?\\d\\d\\d-\\d\\d\\d-\\d\\d\\d)")); pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); - grokProcessor = new GrokProcessor(pluginSetting, expressionEvaluator); + grokProcessorConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), GrokProcessorConfig.class); + grokProcessor = new GrokProcessor(pluginMetrics, grokProcessorConfig, expressionEvaluator); final Map testData = new HashMap(); testData.put("message", "This is my greedy data before matching with my phone number 123-456-789"); @@ -477,7 +495,8 @@ public void testMatchWithNoCapturesAndTags() throws JsonProcessingException { pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); pluginSetting.getSettings().put(GrokProcessorConfig.TAGS_ON_MATCH_FAILURE, List.of(tagOnMatchFailure1, tagOnMatchFailure2)); - grokProcessor = new GrokProcessor(pluginSetting, expressionEvaluator); + grokProcessorConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), GrokProcessorConfig.class); + grokProcessor = new GrokProcessor(pluginMetrics, grokProcessorConfig, expressionEvaluator); final Map testData = new HashMap(); testData.put("log", "This is my greedy data before matching with my phone number 123-456-789"); @@ -495,14 +514,16 @@ public void testMatchWithNoCapturesAndTags() throws JsonProcessingException { @Test public void testCompileNonRegisteredPatternThrowsIllegalArgumentException() { - grokProcessor = new GrokProcessor(pluginSetting, expressionEvaluator); + grokProcessor = new GrokProcessor(pluginMetrics, grokProcessorConfig, expressionEvaluator); final Map> matchConfig = new HashMap<>(); matchConfig.put("message", Collections.singletonList("%{NONEXISTENTPATTERN}")); pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); + grokProcessorConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), GrokProcessorConfig.class); - assertThrows(IllegalArgumentException.class, () -> new GrokProcessor(pluginSetting, expressionEvaluator)); + assertThrows(IllegalArgumentException.class, () -> new GrokProcessor( + pluginMetrics, grokProcessorConfig, expressionEvaluator)); } @ParameterizedTest @@ -512,7 +533,8 @@ void testDataPrepperBuiltInGrokPatterns(final String matchPattern, final String matchConfig.put("message", Collections.singletonList(matchPattern)); pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); - grokProcessor = new GrokProcessor(pluginSetting, expressionEvaluator); + grokProcessorConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), GrokProcessorConfig.class); + grokProcessor = new GrokProcessor(pluginMetrics, grokProcessorConfig, expressionEvaluator); final Map testData = new HashMap(); testData.put("message", logInput); diff --git a/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorTests.java b/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorTests.java index e9d17121d8..aedad1fe5c 100644 --- a/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorTests.java +++ b/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorTests.java @@ -20,11 +20,9 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; -import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.expression.ExpressionEvaluator; import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.JacksonEvent; import org.opensearch.dataprepper.model.record.Record; @@ -52,7 +50,6 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.any; import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -109,23 +106,22 @@ public class GrokProcessorTests { @Mock private ExpressionEvaluator expressionEvaluator; - - private PluginSetting pluginSetting; + @Mock + private GrokProcessorConfig grokProcessorConfig; private final String PLUGIN_NAME = "grok"; private Map capture; private final Map> matchConfig = new HashMap<>(); @BeforeEach public void setup() throws TimeoutException, ExecutionException, InterruptedException { - pluginSetting = getDefaultPluginSetting(); - pluginSetting.setPipelineName("grokPipeline"); + configureDefaultGrokProcessorConfig(); final List matchPatterns = new ArrayList<>(); matchPatterns.add("%{PATTERN1}"); matchPatterns.add("%{PATTERN2}"); matchConfig.put("message", matchPatterns); - pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); + when(grokProcessorConfig.getMatch()).thenReturn(matchConfig); lenient().when(pluginMetrics.counter(GrokProcessor.GROK_PROCESSING_MATCH)).thenReturn(grokProcessingMatchCounter); lenient().when(pluginMetrics.counter(GrokProcessor.GROK_PROCESSING_MISMATCH)).thenReturn(grokProcessingMismatchCounter); @@ -155,15 +151,13 @@ public void setup() throws TimeoutException, ExecutionException, InterruptedExce } private GrokProcessor createObjectUnderTest() { - try (MockedStatic pluginMetricsMockedStatic = mockStatic(PluginMetrics.class)) { - pluginMetricsMockedStatic.when(() -> PluginMetrics.fromPluginSetting(pluginSetting)).thenReturn(pluginMetrics); - return new GrokProcessor(pluginSetting, grokCompiler, executorService, expressionEvaluator); - } + return new GrokProcessor( + pluginMetrics, grokProcessorConfig, grokCompiler, executorService, expressionEvaluator); } @Test public void testMatchMerge() throws JsonProcessingException, ExecutionException, InterruptedException, TimeoutException { - pluginSetting.getSettings().put(GrokProcessorConfig.INCLUDE_PERFORMANCE_METADATA, false); + when(grokProcessorConfig.getIncludePerformanceMetadata()).thenReturn(false); grokProcessor = createObjectUnderTest(); @@ -202,7 +196,7 @@ public void testMatchMerge() throws JsonProcessingException, ExecutionException, @Test public void testTarget() throws JsonProcessingException, ExecutionException, InterruptedException, TimeoutException { - pluginSetting.getSettings().put(GrokProcessorConfig.TARGET_KEY, "test_target"); + when(grokProcessorConfig.getTargetKey()).thenReturn("test_target"); grokProcessor = createObjectUnderTest(); capture.put("key_capture_1", "value_capture_1"); @@ -238,7 +232,7 @@ public void testTarget() throws JsonProcessingException, ExecutionException, Int @Test public void testOverwrite() throws JsonProcessingException { - pluginSetting.getSettings().put(GrokProcessorConfig.KEYS_TO_OVERWRITE, Collections.singletonList("message")); + when(grokProcessorConfig.getkeysToOverwrite()).thenReturn(Collections.singletonList("message")); grokProcessor = createObjectUnderTest(); capture.put("key_capture_1", "value_capture_1"); @@ -423,7 +417,7 @@ public void testThatTimeoutExceptionIsCaughtAndProcessingContinues() throws Json @Test public void testThatProcessingWithTimeoutMillisOfZeroDoesNotInteractWithExecutorServiceAndReturnsCorrectResult() throws JsonProcessingException { - pluginSetting.getSettings().put(GrokProcessorConfig.TIMEOUT_MILLIS, 0); + when(grokProcessorConfig.getTimeoutMillis()).thenReturn(0); grokProcessor = createObjectUnderTest(); capture.put("key_capture_1", "value_capture_1"); @@ -528,7 +522,7 @@ public void testNoCaptures() throws JsonProcessingException { @Test public void testMatchOnSecondPattern() throws JsonProcessingException { - pluginSetting.getSettings().put(GrokProcessorConfig.INCLUDE_PERFORMANCE_METADATA, true); + when(grokProcessorConfig.getIncludePerformanceMetadata()).thenReturn(true); when(match.capture()).thenReturn(Collections.emptyMap()); when(grokSecondMatch.match(messageInput)).thenReturn(secondMatch); @@ -556,7 +550,7 @@ public void testMatchOnSecondPattern() throws JsonProcessingException { @Test public void testMatchOnSecondPatternWithExistingMetadataForTotalPatternMatches() throws JsonProcessingException { - pluginSetting.getSettings().put(GrokProcessorConfig.INCLUDE_PERFORMANCE_METADATA, true); + when(grokProcessorConfig.getIncludePerformanceMetadata()).thenReturn(true); when(match.capture()).thenReturn(Collections.emptyMap()); when(grokSecondMatch.match(messageInput)).thenReturn(secondMatch); @@ -598,8 +592,10 @@ void setUp() { tagOnMatchFailure2 = UUID.randomUUID().toString(); tagOnTimeout1 = UUID.randomUUID().toString(); tagOnTimeout2 = UUID.randomUUID().toString(); - pluginSetting.getSettings().put(GrokProcessorConfig.TAGS_ON_MATCH_FAILURE, List.of(tagOnMatchFailure1, tagOnMatchFailure2)); - pluginSetting.getSettings().put(GrokProcessorConfig.TAGS_ON_TIMEOUT, List.of(tagOnTimeout1, tagOnTimeout2)); + when(grokProcessorConfig.getTagsOnMatchFailure()).thenReturn( + List.of(tagOnMatchFailure1, tagOnMatchFailure2)); + when(grokProcessorConfig.getTagsOnTimeout()).thenReturn( + List.of(tagOnTimeout1, tagOnTimeout2)); } @Test @@ -654,6 +650,34 @@ public void timeout_exception_tags_the_event() throws JsonProcessingException, T verifyNoInteractions(grokProcessingErrorsCounter, grokProcessingMismatchCounter); } + @Test + public void timeout_exception_tags_the_event_with_tags_on_match_failure() + throws JsonProcessingException, TimeoutException, ExecutionException, InterruptedException { + when(grokProcessorConfig.getTagsOnTimeout()).thenReturn(Collections.emptyList()); + when(task.get(GrokProcessorConfig.DEFAULT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)).thenThrow(TimeoutException.class); + + grokProcessor = createObjectUnderTest(); + + capture.put("key_capture_1", "value_capture_1"); + capture.put("key_capture_2", "value_capture_2"); + capture.put("key_capture_3", "value_capture_3"); + + final Map testData = new HashMap(); + testData.put("message", messageInput); + final Record record = buildRecordWithEvent(testData); + + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); + + assertThat(grokkedRecords.size(), equalTo(1)); + assertThat(grokkedRecords.get(0), notNullValue()); + assertRecordsAreEqual(grokkedRecords.get(0), record); + assertThat(record.getData().getMetadata().getTags(), hasItem(tagOnMatchFailure1)); + assertThat(record.getData().getMetadata().getTags(), hasItem(tagOnMatchFailure2)); + verify(grokProcessingTimeoutsCounter, times(1)).increment(); + verify(grokProcessingTime, times(1)).record(any(Runnable.class)); + verifyNoInteractions(grokProcessingErrorsCounter, grokProcessingMismatchCounter); + } + @ParameterizedTest @ValueSource(classes = {ExecutionException.class, InterruptedException.class, RuntimeException.class}) public void execution_exception_tags_the_event(Class exceptionClass) throws JsonProcessingException, TimeoutException, ExecutionException, InterruptedException { @@ -720,7 +744,7 @@ public void testBreakOnMatchTrue() throws JsonProcessingException { @Test public void testBreakOnMatchFalse() throws JsonProcessingException { - pluginSetting.getSettings().put(GrokProcessorConfig.BREAK_ON_MATCH, false); + when(grokProcessorConfig.isBreakOnMatch()).thenReturn(false); grokProcessor = createObjectUnderTest(); when(grokSecondMatch.match(messageInput)).thenReturn(secondMatch); @@ -756,10 +780,8 @@ public void testBreakOnMatchFalse() throws JsonProcessingException { } } - private PluginSetting getDefaultPluginSetting() { - - return completePluginSettingForGrokProcessor( - GrokProcessorConfig.DEFAULT_BREAK_ON_MATCH, + private void configureDefaultGrokProcessorConfig() { + completeMockGrokProcessorConfig(GrokProcessorConfig.DEFAULT_BREAK_ON_MATCH, GrokProcessorConfig.DEFAULT_KEEP_EMPTY_CAPTURES, matchConfig, GrokProcessorConfig.DEFAULT_NAMED_CAPTURES_ONLY, @@ -775,7 +797,7 @@ private PluginSetting getDefaultPluginSetting() { @Test public void testNoGrok_when_GrokWhen_returns_false() throws JsonProcessingException { final String grokWhen = UUID.randomUUID().toString(); - pluginSetting.getSettings().put(GrokProcessorConfig.GROK_WHEN, grokWhen); + when(grokProcessorConfig.getGrokWhen()).thenReturn(grokWhen); grokProcessor = createObjectUnderTest(); capture.put("key_capture_1", "value_capture_1"); @@ -796,31 +818,28 @@ public void testNoGrok_when_GrokWhen_returns_false() throws JsonProcessingExcept verifyNoInteractions(grok, grokSecondMatch); } - private PluginSetting completePluginSettingForGrokProcessor(final boolean breakOnMatch, - final boolean keepEmptyCaptures, - final Map> match, - final boolean namedCapturesOnly, - final List keysToOverwrite, - final List patternsDirectories, - final String patternsFilesGlob, - final Map patternDefinitions, - final int timeoutMillis, - final String targetKey, - final String grokWhen) { - final Map settings = new HashMap<>(); - settings.put(GrokProcessorConfig.BREAK_ON_MATCH, breakOnMatch); - settings.put(GrokProcessorConfig.NAMED_CAPTURES_ONLY, namedCapturesOnly); - settings.put(GrokProcessorConfig.MATCH, match); - settings.put(GrokProcessorConfig.KEEP_EMPTY_CAPTURES, keepEmptyCaptures); - settings.put(GrokProcessorConfig.KEYS_TO_OVERWRITE, keysToOverwrite); - settings.put(GrokProcessorConfig.PATTERNS_DIRECTORIES, patternsDirectories); - settings.put(GrokProcessorConfig.PATTERN_DEFINITIONS, patternDefinitions); - settings.put(GrokProcessorConfig.PATTERNS_FILES_GLOB, patternsFilesGlob); - settings.put(GrokProcessorConfig.TIMEOUT_MILLIS, timeoutMillis); - settings.put(GrokProcessorConfig.TARGET_KEY, targetKey); - settings.put(GrokProcessorConfig.GROK_WHEN, grokWhen); - - return new PluginSetting(PLUGIN_NAME, settings); + private void completeMockGrokProcessorConfig(final boolean breakOnMatch, + final boolean keepEmptyCaptures, + final Map> match, + final boolean namedCapturesOnly, + final List keysToOverwrite, + final List patternsDirectories, + final String patternsFilesGlob, + final Map patternDefinitions, + final int timeoutMillis, + final String targetKey, + final String grokWhen) { + lenient().when(grokProcessorConfig.isBreakOnMatch()).thenReturn(breakOnMatch); + lenient().when(grokProcessorConfig.isNamedCapturesOnly()).thenReturn(namedCapturesOnly); + lenient().when(grokProcessorConfig.getMatch()).thenReturn(match); + lenient().when(grokProcessorConfig.isKeepEmptyCaptures()).thenReturn(keepEmptyCaptures); + lenient().when(grokProcessorConfig.getkeysToOverwrite()).thenReturn(keysToOverwrite); + lenient().when(grokProcessorConfig.getPatternsDirectories()).thenReturn(patternsDirectories); + lenient().when(grokProcessorConfig.getPatternDefinitions()).thenReturn(patternDefinitions); + lenient().when(grokProcessorConfig.getPatternsFilesGlob()).thenReturn(patternsFilesGlob); + lenient().when(grokProcessorConfig.getTimeoutMillis()).thenReturn(timeoutMillis); + lenient().when(grokProcessorConfig.getTargetKey()).thenReturn(targetKey); + lenient().when(grokProcessorConfig.getGrokWhen()).thenReturn(grokWhen); } private void assertRecordsAreEqual(final Record first, final Record second) throws JsonProcessingException { diff --git a/data-prepper-plugins/http-common/build.gradle b/data-prepper-plugins/http-common/build.gradle index fa0e1c3efb..54fa5d346d 100644 --- a/data-prepper-plugins/http-common/build.gradle +++ b/data-prepper-plugins/http-common/build.gradle @@ -6,7 +6,6 @@ dependencies { implementation 'org.apache.httpcomponents:httpcore:4.4.16' testImplementation testLibs.bundles.junit - testImplementation testLibs.mockito.inline } jacocoTestCoverageVerification { diff --git a/data-prepper-plugins/http-sink/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-plugins/http-sink/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 23c33feb6d..0000000000 --- a/data-prepper-plugins/http-sink/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1,3 +0,0 @@ -# To enable mocking of final classes with vanilla Mockito -# https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods -mock-maker-inline diff --git a/data-prepper-plugins/http-source-common/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-plugins/http-source-common/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 78ccc25012..0000000000 --- a/data-prepper-plugins/http-source-common/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1,3 +0,0 @@ -# To enable mocking of final classes with vanilla Mockito -# https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods -mock-maker-inline \ No newline at end of file diff --git a/data-prepper-plugins/http-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-plugins/http-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 78ccc25012..0000000000 --- a/data-prepper-plugins/http-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1,3 +0,0 @@ -# To enable mocking of final classes with vanilla Mockito -# https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods -mock-maker-inline \ No newline at end of file diff --git a/data-prepper-plugins/kafka-plugins/build.gradle b/data-prepper-plugins/kafka-plugins/build.gradle index 0032bed806..046aef949a 100644 --- a/data-prepper-plugins/kafka-plugins/build.gradle +++ b/data-prepper-plugins/kafka-plugins/build.gradle @@ -53,7 +53,6 @@ dependencies { implementation 'software.amazon.awssdk:s3' implementation 'software.amazon.awssdk:apache-client' - testImplementation testLibs.mockito.inline testImplementation 'org.yaml:snakeyaml:2.2' testImplementation testLibs.spring.test testImplementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' @@ -62,12 +61,10 @@ dependencies { testImplementation project(':data-prepper-core') testImplementation project(':data-prepper-plugin-framework') testImplementation project(':data-prepper-pipeline-parser') - testImplementation testLibs.mockito.inline testImplementation 'org.apache.kafka:kafka_2.13:3.6.1' testImplementation 'org.apache.kafka:kafka_2.13:3.6.1:test' testImplementation 'org.apache.curator:curator-test:5.5.0' testImplementation('com.kjetland:mbknor-jackson-jsonschema_2.13:1.0.39') - testImplementation group: 'org.powermock', name: 'powermock-api-mockito2', version: '2.0.9' testImplementation project(':data-prepper-plugins:otel-metrics-source') testImplementation project(':data-prepper-plugins:otel-proto-common') testImplementation libs.opentelemetry.proto diff --git a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java index 84cdb868e9..bcc8eb0a27 100644 --- a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java +++ b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java @@ -6,6 +6,7 @@ package org.opensearch.dataprepper.plugins.processor.keyvalue; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.AssertTrue; @@ -35,87 +36,163 @@ public class KeyValueProcessorConfig { static final boolean DEFAULT_RECURSIVE = false; @NotEmpty + @JsonPropertyDescription("The message field to be parsed. Optional. Default value is `message`.") private String source = DEFAULT_SOURCE; + @JsonPropertyDescription("The destination field for the parsed source. The parsed source overwrites the " + + "preexisting data for that key. Optional. If `destination` is set to `null`, the parsed fields will be " + + "written to the root of the event. Default value is `parsed_message`.") private String destination = DEFAULT_DESTINATION; @JsonProperty("field_delimiter_regex") + @JsonPropertyDescription("A regular expression specifying the delimiter that separates key-value pairs. " + + "Special regular expression characters such as `[` and `]` must be escaped with `\\\\`. " + + "Cannot be defined at the same time as `field_split_characters`. Optional. " + + "If this option is not defined, `field_split_characters` is used.") private String fieldDelimiterRegex; @JsonProperty("field_split_characters") + @JsonPropertyDescription("A string of characters specifying the delimiter that separates key-value pairs. " + + "Special regular expression characters such as `[` and `]` must be escaped with `\\\\`. " + + "Cannot be defined at the same time as `field_delimiter_regex`. Optional. Default value is `&`.") private String fieldSplitCharacters = DEFAULT_FIELD_SPLIT_CHARACTERS; @JsonProperty("include_keys") + @JsonPropertyDescription("An array specifying the keys that should be added for parsing. " + + "By default, all keys will be added.") @NotNull private List includeKeys = DEFAULT_INCLUDE_KEYS; @JsonProperty("exclude_keys") + @JsonPropertyDescription("An array specifying the parsed keys that should not be added to the event. " + + "By default, no keys will be excluded.") @NotNull private List excludeKeys = DEFAULT_EXCLUDE_KEYS; @JsonProperty("default_values") + @JsonPropertyDescription("A map specifying the default keys and their values that should be added " + + "to the event in case these keys do not exist in the source field being parsed. " + + "If the default key already exists in the message, the value is not changed. " + + "The `include_keys` filter will be applied to the message before `default_values`.") @NotNull private Map defaultValues = DEFAULT_DEFAULT_VALUES; @JsonProperty("key_value_delimiter_regex") + @JsonPropertyDescription("A regular expression specifying the delimiter that separates the key and value " + + "within a key-value pair. Special regular expression characters such as `[` and `]` must be escaped with " + + "`\\\\`. This option cannot be defined at the same time as `value_split_characters`. Optional. " + + "If this option is not defined, `value_split_characters` is used.") private String keyValueDelimiterRegex; @JsonProperty("value_split_characters") + @JsonPropertyDescription("A string of characters specifying the delimiter that separates the key and value within " + + "a key-value pair. Special regular expression characters such as `[` and `]` must be escaped with `\\\\`. " + + "Cannot be defined at the same time as `key_value_delimiter_regex`. Optional. Default value is `=`.") private String valueSplitCharacters = DEFAULT_VALUE_SPLIT_CHARACTERS; @JsonProperty("non_match_value") + @JsonPropertyDescription("When a key-value pair cannot be successfully split, the key-value pair is " + + "placed in the `key` field, and the specified value is placed in the `value` field. " + + "Optional. Default value is `null`.") private Object nonMatchValue = DEFAULT_NON_MATCH_VALUE; + @JsonPropertyDescription("A prefix to append before all keys. Optional. Default value is an empty string.") @NotNull private String prefix = DEFAULT_PREFIX; @JsonProperty("delete_key_regex") + @JsonPropertyDescription("A regular expression specifying the characters to delete from the key. " + + "Special regular expression characters such as `[` and `]` must be escaped with `\\\\`. Cannot be an " + + "empty string. Optional. No default value.") @NotNull private String deleteKeyRegex = DEFAULT_DELETE_KEY_REGEX; @JsonProperty("delete_value_regex") + @JsonPropertyDescription("A regular expression specifying the characters to delete from the value. " + + "Special regular expression characters such as `[` and `]` must be escaped with `\\\\`. " + + "Cannot be an empty string. Optional. No default value.") @NotNull private String deleteValueRegex = DEFAULT_DELETE_VALUE_REGEX; @JsonProperty("transform_key") + @JsonPropertyDescription("When to lowercase, uppercase, or capitalize keys.") @NotNull private String transformKey = DEFAULT_TRANSFORM_KEY; @JsonProperty("whitespace") + @JsonPropertyDescription("Specifies whether to be lenient or strict with the acceptance of " + + "unnecessary white space surrounding the configured value-split sequence. Default is `lenient`.") @NotNull private String whitespace = DEFAULT_WHITESPACE; @JsonProperty("skip_duplicate_values") + @JsonPropertyDescription("A Boolean option for removing duplicate key-value pairs. When set to `true`, " + + "only one unique key-value pair will be preserved. Default is `false`.") @NotNull private boolean skipDuplicateValues = DEFAULT_SKIP_DUPLICATE_VALUES; @JsonProperty("remove_brackets") + @JsonPropertyDescription("Specifies whether to treat square brackets, angle brackets, and parentheses " + + "as value “wrappers” that should be removed from the value. Default is `false`.") @NotNull private boolean removeBrackets = DEFAULT_REMOVE_BRACKETS; @JsonProperty("value_grouping") + @JsonPropertyDescription("Specifies whether to group values using predefined value grouping delimiters: " + + "`{...}`, `[...]`, `<...>`, `(...)`, `\"...\"`, `'...'`, `http://... (space)`, and `https:// (space)`. " + + "If this flag is enabled, then the content between the delimiters is considered to be one entity and " + + "is not parsed for key-value pairs. Default is `false`. If `value_grouping` is `true`, then " + + "`{\"key1=[a=b,c=d]&key2=value2\"}` parses to `{\"key1\": \"[a=b,c=d]\", \"key2\": \"value2\"}`.") private boolean valueGrouping = DEFAULT_VALUE_GROUPING; @JsonProperty("recursive") + @JsonPropertyDescription("Specifies whether to recursively obtain additional key-value pairs from values. " + + "The extra key-value pairs will be stored as sub-keys of the root key. Default is `false`. " + + "The levels of recursive parsing must be defined by different brackets for each level: " + + "`[]`, `()`, and `<>`, in this order. Any other configurations specified will only be applied " + + "to the outmost keys.\n" + + "When `recursive` is `true`:\n" + + "`remove_brackets` cannot also be `true`;\n" + + "`skip_duplicate_values` will always be `true`;\n" + + "`whitespace` will always be `\"strict\"`.") @NotNull private boolean recursive = DEFAULT_RECURSIVE; @JsonProperty("tags_on_failure") + @JsonPropertyDescription("When a `kv` operation causes a runtime exception within the processor, " + + "the operation is safely stopped without crashing the processor, and the event is tagged " + + "with the provided tags.") private List tagsOnFailure; @JsonProperty("overwrite_if_destination_exists") + @JsonPropertyDescription("Specifies whether to overwrite existing fields if there are key conflicts " + + "when writing parsed fields to the event. Default is `true`.") private boolean overwriteIfDestinationExists = true; @JsonProperty("drop_keys_with_no_value") + @JsonPropertyDescription("Specifies whether keys should be dropped if they have a null value. Default is `false`. " + + "If `drop_keys_with_no_value` is set to `true`, " + + "then `{\"key1=value1&key2\"}` parses to `{\"key1\": \"value1\"}`.") private boolean dropKeysWithNoValue = false; @JsonProperty("key_value_when") + @JsonPropertyDescription("Allows you to specify a [conditional expression](https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/), " + + "such as `/some-key == \"test\"`, that will be evaluated to determine whether " + + "the processor should be applied to the event.") private String keyValueWhen; @JsonProperty("strict_grouping") + @JsonPropertyDescription("When enabled, groups with unmatched end characters yield errors. " + + "The event is ignored after the errors are logged. " + + "Specifies whether strict grouping should be enabled when the `value_grouping` " + + "or `string_literal_character` options are used. Default is `false`.") private boolean strictGrouping = false; @JsonProperty("string_literal_character") + @JsonPropertyDescription("When this option is used, any text contained within the specified quotation " + + "mark character will be ignored and excluded from key-value parsing. " + + "Can be set to either a single quotation mark (`'`) or a double quotation mark (`\"`). " + + "Default is `null`.") @Size(min = 0, max = 1, message = "string_literal_character may only have character") private String stringLiteralCharacter = null; @@ -124,7 +201,8 @@ boolean isValidValueGroupingAndFieldDelimiterRegex() { return (!valueGrouping || fieldDelimiterRegex == null); } - @AssertTrue(message = "Invalid Configuration. String literal character config is valid only when value_grouping is enabled, and only double quote (\") and single quote are (') are valid string literal characters.") + @AssertTrue(message = "Invalid Configuration. String literal character config is valid only when value_grouping is enabled, " + + "and only double quote (\") and single quote are (') are valid string literal characters.") boolean isValidStringLiteralConfig() { if (stringLiteralCharacter == null) return true; diff --git a/data-prepper-plugins/lambda/build.gradle b/data-prepper-plugins/lambda/build.gradle index d0c09c9c8b..8447c3abdf 100644 --- a/data-prepper-plugins/lambda/build.gradle +++ b/data-prepper-plugins/lambda/build.gradle @@ -27,6 +27,7 @@ dependencies { testImplementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' testImplementation project(':data-prepper-test-common') testImplementation project(':data-prepper-plugins:parse-json-processor') + testImplementation testLibs.slf4j.simple } test { diff --git a/data-prepper-plugins/lambda/src/test/java/org/opensearch/dataprepper/plugins/lambda/sink/LambdaSinkServiceTest.java b/data-prepper-plugins/lambda/src/test/java/org/opensearch/dataprepper/plugins/lambda/sink/LambdaSinkServiceTest.java index 4e678c191d..f8ca0f11ec 100644 --- a/data-prepper-plugins/lambda/src/test/java/org/opensearch/dataprepper/plugins/lambda/sink/LambdaSinkServiceTest.java +++ b/data-prepper-plugins/lambda/src/test/java/org/opensearch/dataprepper/plugins/lambda/sink/LambdaSinkServiceTest.java @@ -294,7 +294,7 @@ public void lambda_sink_test_batch_enabled() throws IOException { when(lambdaSinkConfig.getBatchOptions()).thenReturn(mock(BatchOptions.class)); when(lambdaSinkConfig.getBatchOptions().getBatchKey()).thenReturn(batchKey); when(lambdaSinkConfig.getBatchOptions().getThresholdOptions()).thenReturn(mock(ThresholdOptions.class)); - when(lambdaSinkConfig.getBatchOptions().getThresholdOptions().getEventCount()).thenReturn(maxEvents); + when(lambdaSinkConfig.getBatchOptions().getThresholdOptions().getEventCount()).thenReturn(1); when(lambdaSinkConfig.getBatchOptions().getThresholdOptions().getMaximumSize()).thenReturn(ByteCount.parse(maxSize)); when(lambdaSinkConfig.getBatchOptions().getThresholdOptions().getEventCollectTimeOut()).thenReturn(Duration.ofNanos(10L)); when(lambdaSinkConfig.getAwsAuthenticationOptions()).thenReturn(mock(AwsAuthenticationOptions.class)); diff --git a/data-prepper-plugins/lambda/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-plugins/lambda/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 23c33feb6d..0000000000 --- a/data-prepper-plugins/lambda/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1,3 +0,0 @@ -# To enable mocking of final classes with vanilla Mockito -# https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods -mock-maker-inline diff --git a/data-prepper-plugins/lambda/src/test/resources/simplelogger.properties b/data-prepper-plugins/lambda/src/test/resources/simplelogger.properties new file mode 100644 index 0000000000..f464558cf4 --- /dev/null +++ b/data-prepper-plugins/lambda/src/test/resources/simplelogger.properties @@ -0,0 +1,8 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# + +org.slf4j.simpleLogger.showDateTime=true +org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd' 'HH:mm:ss.SSS +org.slf4j.simpleLogger.log.org.opensearch.dataprepper.plugins.lambda.sink=trace diff --git a/data-prepper-plugins/mongodb/build.gradle b/data-prepper-plugins/mongodb/build.gradle index ae4a5a9d45..c5495880e6 100644 --- a/data-prepper-plugins/mongodb/build.gradle +++ b/data-prepper-plugins/mongodb/build.gradle @@ -16,7 +16,6 @@ dependencies { implementation project(path: ':data-prepper-plugins:common') - testImplementation testLibs.mockito.inline testImplementation testLibs.bundles.junit testImplementation testLibs.slf4j.simple testImplementation project(path: ':data-prepper-test-common') diff --git a/data-prepper-plugins/mutate-event-processors/build.gradle b/data-prepper-plugins/mutate-event-processors/build.gradle index 3fbbc37254..e4b0c63cea 100644 --- a/data-prepper-plugins/mutate-event-processors/build.gradle +++ b/data-prepper-plugins/mutate-event-processors/build.gradle @@ -22,4 +22,6 @@ dependencies { implementation project(':data-prepper-api') implementation project(':data-prepper-plugins:common') implementation 'com.fasterxml.jackson.core:jackson-databind' + testImplementation project(':data-prepper-test-event') + testImplementation testLibs.slf4j.simple } \ No newline at end of file diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessor.java index d7c902a32c..cfadf70d03 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessor.java @@ -10,6 +10,7 @@ import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventKey; import org.opensearch.dataprepper.model.processor.AbstractProcessor; import org.opensearch.dataprepper.model.processor.Processor; import org.opensearch.dataprepper.model.record.Record; @@ -17,6 +18,7 @@ import org.slf4j.LoggerFactory; import java.util.Collection; +import java.util.List; import java.util.Objects; import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; @@ -25,7 +27,7 @@ public class DeleteEntryProcessor extends AbstractProcessor, Record> { private static final Logger LOG = LoggerFactory.getLogger(DeleteEntryProcessor.class); - private final String[] entries; + private final List entries; private final String deleteWhen; private final ExpressionEvaluator expressionEvaluator; @@ -49,7 +51,7 @@ public Collection> doExecute(final Collection> recor } - for (String entry : entries) { + for (final EventKey entry : entries) { recordEvent.delete(entry); } } catch (final Exception e) { diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessorConfig.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessorConfig.java index 8470576a7b..b1df976770 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessorConfig.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessorConfig.java @@ -6,19 +6,29 @@ package org.opensearch.dataprepper.plugins.processor.mutateevent; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; +import org.opensearch.dataprepper.model.event.EventKey; +import org.opensearch.dataprepper.model.event.EventKeyConfiguration; +import org.opensearch.dataprepper.model.event.EventKeyFactory; + +import java.util.List; public class DeleteEntryProcessorConfig { @NotEmpty @NotNull @JsonProperty("with_keys") - private String[] withKeys; + @EventKeyConfiguration(EventKeyFactory.EventAction.DELETE) + @JsonPropertyDescription("An array of keys for the entries to be deleted.") + private List<@NotNull @NotEmpty EventKey> withKeys; @JsonProperty("delete_when") + @JsonPropertyDescription("Specifies under what condition the `delete_entries` processor should perform deletion. " + + "Default is no condition.") private String deleteWhen; - public String[] getWithKeys() { + public List getWithKeys() { return withKeys; } diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java index f1e723ad5a..d1ee0178a6 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java @@ -9,6 +9,9 @@ import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; +import org.opensearch.dataprepper.model.event.EventKey; +import org.opensearch.dataprepper.model.event.EventKeyConfiguration; +import org.opensearch.dataprepper.model.event.EventKeyFactory; import java.util.List; @@ -17,12 +20,14 @@ public static class Entry { @NotEmpty @NotNull @JsonProperty("from_key") - private String fromKey; + @EventKeyConfiguration({EventKeyFactory.EventAction.GET, EventKeyFactory.EventAction.DELETE}) + private EventKey fromKey; @NotEmpty @NotNull @JsonProperty("to_key") - private String toKey; + @EventKeyConfiguration(EventKeyFactory.EventAction.PUT) + private EventKey toKey; @JsonProperty("overwrite_if_to_key_exists") private boolean overwriteIfToKeyExists = false; @@ -30,11 +35,11 @@ public static class Entry { @JsonProperty("rename_when") private String renameWhen; - public String getFromKey() { + public EventKey getFromKey() { return fromKey; } - public String getToKey() { + public EventKey getToKey() { return toKey; } @@ -44,7 +49,7 @@ public boolean getOverwriteIfToKeyExists() { public String getRenameWhen() { return renameWhen; } - public Entry(final String fromKey, final String toKey, final boolean overwriteIfKeyExists, final String renameWhen) { + public Entry(final EventKey fromKey, final EventKey toKey, final boolean overwriteIfKeyExists, final String renameWhen) { this.fromKey = fromKey; this.toKey = toKey; this.overwriteIfToKeyExists = overwriteIfKeyExists; diff --git a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessorTests.java b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessorTests.java index 2394a5d958..bc0fb78870 100644 --- a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessorTests.java +++ b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessorTests.java @@ -5,15 +5,17 @@ package org.opensearch.dataprepper.plugins.processor.mutateevent; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.event.TestEventKeyFactory; import org.opensearch.dataprepper.expression.ExpressionEvaluator; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventKeyFactory; import org.opensearch.dataprepper.model.event.JacksonEvent; import org.opensearch.dataprepper.model.record.Record; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; import java.util.Collections; import java.util.HashMap; @@ -36,9 +38,11 @@ public class DeleteEntryProcessorTests { @Mock private ExpressionEvaluator expressionEvaluator; + private final EventKeyFactory eventKeyFactory = TestEventKeyFactory.getTestEventFactory(); + @Test public void testSingleDeleteProcessorTest() { - when(mockConfig.getWithKeys()).thenReturn(new String[] { "message" }); + when(mockConfig.getWithKeys()).thenReturn(List.of(eventKeyFactory.createEventKey("message", EventKeyFactory.EventAction.DELETE))); when(mockConfig.getDeleteWhen()).thenReturn(null); final DeleteEntryProcessor processor = createObjectUnderTest(); @@ -52,7 +56,7 @@ public void testSingleDeleteProcessorTest() { @Test public void testWithKeyDneDeleteProcessorTest() { - when(mockConfig.getWithKeys()).thenReturn(new String[] { "message2" }); + when(mockConfig.getWithKeys()).thenReturn(List.of(eventKeyFactory.createEventKey("message2", EventKeyFactory.EventAction.DELETE))); when(mockConfig.getDeleteWhen()).thenReturn(null); final DeleteEntryProcessor processor = createObjectUnderTest(); @@ -67,7 +71,9 @@ public void testWithKeyDneDeleteProcessorTest() { @Test public void testMultiDeleteProcessorTest() { - when(mockConfig.getWithKeys()).thenReturn(new String[] { "message", "message2" }); + when(mockConfig.getWithKeys()).thenReturn(List.of( + eventKeyFactory.createEventKey("message", EventKeyFactory.EventAction.DELETE), + eventKeyFactory.createEventKey("message2", EventKeyFactory.EventAction.DELETE))); when(mockConfig.getDeleteWhen()).thenReturn(null); final DeleteEntryProcessor processor = createObjectUnderTest(); @@ -83,7 +89,7 @@ public void testMultiDeleteProcessorTest() { @Test public void testKeyIsNotDeleted_when_deleteWhen_returns_false() { - when(mockConfig.getWithKeys()).thenReturn(new String[] { "message" }); + when(mockConfig.getWithKeys()).thenReturn(List.of(eventKeyFactory.createEventKey("message", EventKeyFactory.EventAction.DELETE))); final String deleteWhen = UUID.randomUUID().toString(); when(mockConfig.getDeleteWhen()).thenReturn(deleteWhen); @@ -98,8 +104,9 @@ public void testKeyIsNotDeleted_when_deleteWhen_returns_false() { assertThat(editedRecords.get(0).getData().containsKey("newMessage"), is(true)); } + @Test public void testNestedDeleteProcessorTest() { - when(mockConfig.getWithKeys()).thenReturn(new String[]{"nested/foo"}); + when(mockConfig.getWithKeys()).thenReturn(List.of(eventKeyFactory.createEventKey("nested/foo", EventKeyFactory.EventAction.DELETE))); Map nested = Map.of("foo", "bar", "fizz", 42); diff --git a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java index dfc5a7b595..6ae362bc46 100644 --- a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java +++ b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java @@ -5,9 +5,12 @@ package org.opensearch.dataprepper.plugins.processor.mutateevent; +import org.opensearch.dataprepper.event.TestEventKeyFactory; import org.opensearch.dataprepper.expression.ExpressionEvaluator; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventKey; +import org.opensearch.dataprepper.model.event.EventKeyFactory; import org.opensearch.dataprepper.model.event.JacksonEvent; import org.opensearch.dataprepper.model.record.Record; import org.junit.jupiter.api.Test; @@ -39,6 +42,8 @@ public class RenameKeyProcessorTests { @Mock private ExpressionEvaluator expressionEvaluator; + private final EventKeyFactory eventKeyFactory = TestEventKeyFactory.getTestEventFactory(); + @Test public void testSingleOverwriteRenameProcessorTests() { when(mockConfig.getEntries()).thenReturn(createListOfEntries(createEntry("message", "newMessage", true, null))); @@ -136,7 +141,9 @@ private RenameKeyProcessor createObjectUnderTest() { } private RenameKeyProcessorConfig.Entry createEntry(final String fromKey, final String toKey, final boolean overwriteIfToKeyExists, final String renameWhen) { - return new RenameKeyProcessorConfig.Entry(fromKey, toKey, overwriteIfToKeyExists, renameWhen); + final EventKey fromEventKey = eventKeyFactory.createEventKey(fromKey); + final EventKey toEventKey = eventKeyFactory.createEventKey(toKey); + return new RenameKeyProcessorConfig.Entry(fromEventKey, toEventKey, overwriteIfToKeyExists, renameWhen); } private List createListOfEntries(final RenameKeyProcessorConfig.Entry... entries) { diff --git a/data-prepper-plugins/mutate-string-processors/build.gradle b/data-prepper-plugins/mutate-string-processors/build.gradle index 3fbbc37254..0723e63c10 100644 --- a/data-prepper-plugins/mutate-string-processors/build.gradle +++ b/data-prepper-plugins/mutate-string-processors/build.gradle @@ -22,4 +22,5 @@ dependencies { implementation project(':data-prepper-api') implementation project(':data-prepper-plugins:common') implementation 'com.fasterxml.jackson.core:jackson-databind' + testImplementation project(':data-prepper-test-event') } \ No newline at end of file diff --git a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/AbstractStringProcessor.java b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/AbstractStringProcessor.java index 19d11daf62..ae7a242da3 100644 --- a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/AbstractStringProcessor.java +++ b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/AbstractStringProcessor.java @@ -8,6 +8,7 @@ import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventKey; import org.opensearch.dataprepper.model.processor.AbstractProcessor; import org.opensearch.dataprepper.model.record.Record; @@ -46,8 +47,8 @@ public Collection> doExecute(final Collection> recor private void performStringAction(final Event recordEvent) { try { - for(T entry : entries) { - final String key = getKey(entry); + for(final T entry : entries) { + final EventKey key = getKey(entry); if(recordEvent.containsKey(key)) { final Object value = recordEvent.get(key, Object.class); @@ -64,7 +65,7 @@ private void performStringAction(final Event recordEvent) protected abstract void performKeyAction(final Event recordEvent, final T entry, final String value); - protected abstract String getKey(final T entry); + protected abstract EventKey getKey(final T entry); @Override public void prepareForShutdown() { diff --git a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/LowercaseStringProcessor.java b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/LowercaseStringProcessor.java index b76e922c61..c2c2071e95 100644 --- a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/LowercaseStringProcessor.java +++ b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/LowercaseStringProcessor.java @@ -9,6 +9,7 @@ import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventKey; import org.opensearch.dataprepper.model.processor.Processor; import java.util.Locale; @@ -18,20 +19,20 @@ * no action is performed. */ @DataPrepperPlugin(name = "lowercase_string", pluginType = Processor.class, pluginConfigurationType = WithKeysConfig.class) -public class LowercaseStringProcessor extends AbstractStringProcessor { +public class LowercaseStringProcessor extends AbstractStringProcessor { @DataPrepperPluginConstructor public LowercaseStringProcessor(final PluginMetrics pluginMetrics, final WithKeysConfig config) { super(pluginMetrics, config); } @Override - protected void performKeyAction(final Event recordEvent, final String key, final String value) + protected void performKeyAction(final Event recordEvent, final EventKey key, final String value) { recordEvent.put(key, value.toLowerCase(Locale.ROOT)); } @Override - protected String getKey(final String entry) { + protected EventKey getKey(final EventKey entry) { return entry; } } diff --git a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessor.java b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessor.java index acac832095..6bc89178d8 100644 --- a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessor.java +++ b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessor.java @@ -10,6 +10,7 @@ import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventKey; import org.opensearch.dataprepper.model.processor.Processor; import java.util.HashMap; @@ -64,7 +65,7 @@ protected void performKeyAction(final Event recordEvent, final SplitStringProces } @Override - protected String getKey(final SplitStringProcessorConfig.Entry entry) { + protected EventKey getKey(final SplitStringProcessorConfig.Entry entry) { return entry.getSource(); } diff --git a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessorConfig.java b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessorConfig.java index 84e4228798..cb8edabfb6 100644 --- a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessorConfig.java +++ b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessorConfig.java @@ -7,10 +7,12 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.Size; +import org.opensearch.dataprepper.model.event.EventKey; import java.util.List; @@ -19,18 +21,26 @@ public static class Entry { @NotEmpty @NotNull - private String source; + @JsonPropertyDescription("The key to split.") + private EventKey source; @JsonProperty("delimiter_regex") + @JsonPropertyDescription("The regex string responsible for the split. Cannot be defined at the same time as `delimiter`. " + + "At least `delimiter` or `delimiter_regex` must be defined.") private String delimiterRegex; @Size(min = 1, max = 1) + @JsonPropertyDescription("The separator character responsible for the split. " + + "Cannot be defined at the same time as `delimiter_regex`. " + + "At least `delimiter` or `delimiter_regex` must be defined.") private String delimiter; @JsonProperty("split_when") + @JsonPropertyDescription("Specifies under what condition the `split_string` processor should perform splitting. " + + "Default is no condition.") private String splitWhen; - public String getSource() { + public EventKey getSource() { return source; } @@ -44,7 +54,7 @@ public String getDelimiter() { public String getSplitWhen() { return splitWhen; } - public Entry(final String source, final String delimiterRegex, final String delimiter, final String splitWhen) { + public Entry(final EventKey source, final String delimiterRegex, final String delimiter, final String splitWhen) { this.source = source; this.delimiterRegex = delimiterRegex; this.delimiter = delimiter; @@ -60,6 +70,7 @@ public List getIterativeConfig() { return entries; } + @JsonPropertyDescription("List of entries. Valid values are `source`, `delimiter`, and `delimiter_regex`.") private List<@Valid Entry> entries; public List getEntries() { diff --git a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SubstituteStringProcessor.java b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SubstituteStringProcessor.java index 7332ce836f..e6dceb62fc 100644 --- a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SubstituteStringProcessor.java +++ b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SubstituteStringProcessor.java @@ -10,6 +10,7 @@ import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventKey; import org.opensearch.dataprepper.model.processor.Processor; import java.util.HashMap; @@ -51,7 +52,7 @@ protected void performKeyAction(final Event recordEvent, final SubstituteStringP } @Override - protected String getKey(final SubstituteStringProcessorConfig.Entry entry) { + protected EventKey getKey(final SubstituteStringProcessorConfig.Entry entry) { return entry.getSource(); } } diff --git a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SubstituteStringProcessorConfig.java b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SubstituteStringProcessorConfig.java index 07789b083a..4a8f53f0fe 100644 --- a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SubstituteStringProcessorConfig.java +++ b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SubstituteStringProcessorConfig.java @@ -6,19 +6,27 @@ package org.opensearch.dataprepper.plugins.processor.mutatestring; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import org.opensearch.dataprepper.model.event.EventKey; import java.util.List; public class SubstituteStringProcessorConfig implements StringProcessorConfig { public static class Entry { - private String source; + @JsonPropertyDescription("The key to modify.") + private EventKey source; + @JsonPropertyDescription("The Regex String to be replaced. Special regex characters such as `[` and `]` must " + + "be escaped using `\\\\` when using double quotes and `\\ ` when using single quotes. " + + "See [Java Patterns](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/util/regex/Pattern.html) " + + "for more information.") private String from; + @JsonPropertyDescription("The String to be substituted for each match of `from`.") private String to; @JsonProperty("substitute_when") private String substituteWhen; - public String getSource() { + public EventKey getSource() { return source; } @@ -32,7 +40,7 @@ public String getTo() { public String getSubstituteWhen() { return substituteWhen; } - public Entry(final String source, final String from, final String to, final String substituteWhen) { + public Entry(final EventKey source, final String from, final String to, final String substituteWhen) { this.source = source; this.from = from; this.to = to; @@ -42,6 +50,7 @@ public Entry(final String source, final String from, final String to, final Stri public Entry() {} } + @JsonPropertyDescription("List of entries. Valid values are `source`, `from`, and `to`.") private List entries; public List getEntries() { diff --git a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/TrimStringProcessor.java b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/TrimStringProcessor.java index 2f0e5f0dc2..2a1213f30f 100644 --- a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/TrimStringProcessor.java +++ b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/TrimStringProcessor.java @@ -9,6 +9,7 @@ import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventKey; import org.opensearch.dataprepper.model.processor.Processor; /** @@ -16,20 +17,20 @@ * If the value is not a string, no action is performed. */ @DataPrepperPlugin(name = "trim_string", pluginType = Processor.class, pluginConfigurationType = WithKeysConfig.class) -public class TrimStringProcessor extends AbstractStringProcessor { +public class TrimStringProcessor extends AbstractStringProcessor { @DataPrepperPluginConstructor public TrimStringProcessor(final PluginMetrics pluginMetrics, final WithKeysConfig config) { super(pluginMetrics, config); } @Override - protected void performKeyAction(final Event recordEvent, final String key, final String value) + protected void performKeyAction(final Event recordEvent, final EventKey key, final String value) { recordEvent.put(key, value.trim()); } @Override - protected String getKey(final String entry) { + protected EventKey getKey(final EventKey entry) { return entry; } } diff --git a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/UppercaseStringProcessor.java b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/UppercaseStringProcessor.java index 9d3665fdd2..28e7aa9847 100644 --- a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/UppercaseStringProcessor.java +++ b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/UppercaseStringProcessor.java @@ -9,6 +9,7 @@ import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventKey; import org.opensearch.dataprepper.model.processor.Processor; import java.util.Locale; @@ -18,19 +19,19 @@ * no action is performed. */ @DataPrepperPlugin(name = "uppercase_string", pluginType = Processor.class, pluginConfigurationType = WithKeysConfig.class) -public class UppercaseStringProcessor extends AbstractStringProcessor { +public class UppercaseStringProcessor extends AbstractStringProcessor { @DataPrepperPluginConstructor public UppercaseStringProcessor(final PluginMetrics pluginMetrics, final WithKeysConfig config) { super(pluginMetrics, config); } @Override - protected String getKey(final String entry) { + protected EventKey getKey(final EventKey entry) { return entry; } @Override - protected void performKeyAction(final Event recordEvent, final String entry, final String value) + protected void performKeyAction(final Event recordEvent, final EventKey entry, final String value) { recordEvent.put(entry, value.toUpperCase(Locale.ROOT)); } diff --git a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/WithKeysConfig.java b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/WithKeysConfig.java index bfe10d02ca..3660b5d73d 100644 --- a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/WithKeysConfig.java +++ b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/WithKeysConfig.java @@ -6,24 +6,27 @@ package org.opensearch.dataprepper.plugins.processor.mutatestring; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; +import org.opensearch.dataprepper.model.event.EventKey; import java.util.List; -public class WithKeysConfig implements StringProcessorConfig { +public class WithKeysConfig implements StringProcessorConfig { @NotNull @NotEmpty @JsonProperty("with_keys") - private List withKeys; + @JsonPropertyDescription("A list of keys to trim the white space from.") + private List withKeys; @Override - public List getIterativeConfig() { + public List getIterativeConfig() { return withKeys; } - public List getWithKeys() { + public List getWithKeys() { return withKeys; } } diff --git a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/WithKeysProcessorConfig.java b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/WithKeysProcessorConfig.java deleted file mode 100644 index 814518c83d..0000000000 --- a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/WithKeysProcessorConfig.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.dataprepper.plugins.processor.mutatestring; - -import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; - -import java.util.List; - -public abstract class WithKeysProcessorConfig implements StringProcessorConfig { - @NotEmpty - @NotNull - @JsonProperty("with_keys") - private List withKeys; - - @Override - public List getIterativeConfig() { - return withKeys; - } - - public List getWithKeys() { - return withKeys; - } -} diff --git a/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/LowercaseStringProcessorTests.java b/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/LowercaseStringProcessorTests.java index 18bddf31a9..8185d8ef8c 100644 --- a/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/LowercaseStringProcessorTests.java +++ b/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/LowercaseStringProcessorTests.java @@ -5,21 +5,26 @@ package org.opensearch.dataprepper.plugins.processor.mutatestring; -import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.event.JacksonEvent; -import org.opensearch.dataprepper.model.record.Record; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.event.TestEventFactory; +import org.opensearch.dataprepper.event.TestEventKeyFactory; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventBuilder; +import org.opensearch.dataprepper.model.event.EventFactory; +import org.opensearch.dataprepper.model.event.EventKeyFactory; +import org.opensearch.dataprepper.model.record.Record; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; @@ -29,6 +34,9 @@ @ExtendWith(MockitoExtension.class) public class LowercaseStringProcessorTests { + private static final EventFactory TEST_EVENT_FACTORY = TestEventFactory.getTestEventFactory(); + private final EventKeyFactory eventKeyFactory = TestEventKeyFactory.getTestEventFactory(); + @Mock private PluginMetrics pluginMetrics; @@ -37,7 +45,7 @@ public class LowercaseStringProcessorTests { @BeforeEach public void setup() { - lenient().when(config.getIterativeConfig()).thenReturn(Collections.singletonList("message")); + lenient().when(config.getIterativeConfig()).thenReturn(Stream.of("message").map(eventKeyFactory::createEventKey).collect(Collectors.toList())); } @Test @@ -52,7 +60,7 @@ public void testHappyPathLowercaseStringProcessor() { @Test public void testHappyPathMultiLowercaseStringProcessor() { - when(config.getIterativeConfig()).thenReturn(Arrays.asList("message", "message2")); + when(config.getIterativeConfig()).thenReturn(Stream.of("message", "message2").map(eventKeyFactory::createEventKey).collect(Collectors.toList())); final LowercaseStringProcessor processor = createObjectUnderTest(); final Record record = getEvent("THISISAMESSAGE"); @@ -67,7 +75,7 @@ public void testHappyPathMultiLowercaseStringProcessor() { @Test public void testHappyPathMultiMixedLowercaseStringProcessor() { - lenient().when(config.getIterativeConfig()).thenReturn(Arrays.asList("message", "message2")); + lenient().when(config.getIterativeConfig()).thenReturn(Stream.of("message", "message2").map(eventKeyFactory::createEventKey).collect(Collectors.toList())); final LowercaseStringProcessor processor = createObjectUnderTest(); final Record record = getEvent("THISISAMESSAGE"); @@ -137,7 +145,7 @@ private Record getEvent(Object message) { } private static Record buildRecordWithEvent(final Map data) { - return new Record<>(JacksonEvent.builder() + return new Record<>(TEST_EVENT_FACTORY.eventBuilder(EventBuilder.class) .withData(data) .withEventType("event") .build()); diff --git a/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessorTests.java b/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessorTests.java index 1f2db4a672..7883dcfd05 100644 --- a/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessorTests.java +++ b/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SplitStringProcessorTests.java @@ -5,10 +5,15 @@ package org.opensearch.dataprepper.plugins.processor.mutatestring; +import org.opensearch.dataprepper.event.TestEventFactory; +import org.opensearch.dataprepper.event.TestEventKeyFactory; import org.opensearch.dataprepper.expression.ExpressionEvaluator; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.event.JacksonEvent; +import org.opensearch.dataprepper.model.event.EventBuilder; +import org.opensearch.dataprepper.model.event.EventFactory; +import org.opensearch.dataprepper.model.event.EventKey; +import org.opensearch.dataprepper.model.event.EventKeyFactory; import org.opensearch.dataprepper.model.record.Record; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -36,6 +41,8 @@ @ExtendWith(MockitoExtension.class) class SplitStringProcessorTests { + private final EventFactory testEventFactory = TestEventFactory.getTestEventFactory(); + private final EventKeyFactory eventKeyFactory = TestEventKeyFactory.getTestEventFactory(); @Mock private PluginMetrics pluginMetrics; @@ -115,13 +122,14 @@ void test_event_is_the_same_when_splitWhen_condition_returns_false() { private SplitStringProcessorConfig.Entry createEntry(final String source, final String delimiterRegex, final String delimiter, final String splitWhen) { - return new SplitStringProcessorConfig.Entry(source, delimiterRegex, delimiter, splitWhen); + final EventKey sourceKey = eventKeyFactory.createEventKey(source); + return new SplitStringProcessorConfig.Entry(sourceKey, delimiterRegex, delimiter, splitWhen); } private Record createEvent(final String message) { final Map eventData = new HashMap<>(); eventData.put("message", message); - return new Record<>(JacksonEvent.builder() + return new Record<>(testEventFactory.eventBuilder(EventBuilder.class) .withEventType("event") .withData(eventData) .build()); diff --git a/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SubstituteStringProcessorTests.java b/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SubstituteStringProcessorTests.java index 04175ee229..dd8d9b1dd8 100644 --- a/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SubstituteStringProcessorTests.java +++ b/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/SubstituteStringProcessorTests.java @@ -5,10 +5,15 @@ package org.opensearch.dataprepper.plugins.processor.mutatestring; +import org.opensearch.dataprepper.event.TestEventFactory; +import org.opensearch.dataprepper.event.TestEventKeyFactory; import org.opensearch.dataprepper.expression.ExpressionEvaluator; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.event.JacksonEvent; +import org.opensearch.dataprepper.model.event.EventBuilder; +import org.opensearch.dataprepper.model.event.EventFactory; +import org.opensearch.dataprepper.model.event.EventKey; +import org.opensearch.dataprepper.model.event.EventKeyFactory; import org.opensearch.dataprepper.model.record.Record; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -33,6 +38,8 @@ @ExtendWith(MockitoExtension.class) public class SubstituteStringProcessorTests { + private static final EventFactory TEST_EVENT_FACTORY = TestEventFactory.getTestEventFactory(); + private final EventKeyFactory eventKeyFactory = TestEventKeyFactory.getTestEventFactory(); @Mock private PluginMetrics pluginMetrics; @@ -42,6 +49,7 @@ public class SubstituteStringProcessorTests { @Mock private ExpressionEvaluator expressionEvaluator; + @BeforeEach public void setup() { lenient().when(config.getIterativeConfig()).thenReturn(Collections.singletonList(createEntry("message", "a", "b", null))); @@ -181,7 +189,8 @@ public boolean equals(Object other) { } private SubstituteStringProcessorConfig.Entry createEntry(final String source, final String from, final String to, final String substituteWhen) { - final SubstituteStringProcessorConfig.Entry entry = new SubstituteStringProcessorConfig.Entry(source, from, to, substituteWhen); + final EventKey sourceKey = eventKeyFactory.createEventKey(source); + final SubstituteStringProcessorConfig.Entry entry = new SubstituteStringProcessorConfig.Entry(sourceKey, from, to, substituteWhen); return entry; } @@ -197,7 +206,7 @@ private Record getEvent(Object message) { } private static Record buildRecordWithEvent(final Map data) { - return new Record<>(JacksonEvent.builder() + return new Record<>(TEST_EVENT_FACTORY.eventBuilder(EventBuilder.class) .withData(data) .withEventType("event") .build()); diff --git a/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/TrimStringProcessorTests.java b/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/TrimStringProcessorTests.java index 06efbbad96..921f6a6094 100644 --- a/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/TrimStringProcessorTests.java +++ b/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/TrimStringProcessorTests.java @@ -5,21 +5,26 @@ package org.opensearch.dataprepper.plugins.processor.mutatestring; -import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.event.JacksonEvent; -import org.opensearch.dataprepper.model.record.Record; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.event.TestEventFactory; +import org.opensearch.dataprepper.event.TestEventKeyFactory; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventBuilder; +import org.opensearch.dataprepper.model.event.EventFactory; +import org.opensearch.dataprepper.model.event.EventKeyFactory; +import org.opensearch.dataprepper.model.record.Record; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; @@ -29,6 +34,8 @@ @ExtendWith(MockitoExtension.class) public class TrimStringProcessorTests { + private static final EventFactory TEST_EVENT_FACTORY = TestEventFactory.getTestEventFactory(); + private final EventKeyFactory eventKeyFactory = TestEventKeyFactory.getTestEventFactory(); @Mock private PluginMetrics pluginMetrics; @@ -37,7 +44,7 @@ public class TrimStringProcessorTests { @BeforeEach public void setup() { - lenient().when(config.getIterativeConfig()).thenReturn(Collections.singletonList("message")); + lenient().when(config.getIterativeConfig()).thenReturn(Stream.of("message").map(eventKeyFactory::createEventKey).collect(Collectors.toList())); } @Test @@ -62,7 +69,7 @@ public void testSpaceInMiddleTrimStringProcessor() { @Test public void testHappyPathMultiTrimStringProcessor() { - when(config.getIterativeConfig()).thenReturn(Arrays.asList("message", "message2")); + when(config.getIterativeConfig()).thenReturn(Stream.of("message", "message2").map(eventKeyFactory::createEventKey).collect(Collectors.toList())); final TrimStringProcessor processor = createObjectUnderTest(); final Record record = getEvent("thisisamessage "); @@ -77,7 +84,7 @@ public void testHappyPathMultiTrimStringProcessor() { @Test public void testHappyPathMultiMixedTrimStringProcessor() { - lenient().when(config.getIterativeConfig()).thenReturn(Arrays.asList("message", "message2")); + lenient().when(config.getIterativeConfig()).thenReturn(Stream.of("message", "message2").map(eventKeyFactory::createEventKey).collect(Collectors.toList())); final TrimStringProcessor processor = createObjectUnderTest(); final Record record = getEvent("thisisamessage "); @@ -147,7 +154,7 @@ private Record getEvent(Object message) { } private static Record buildRecordWithEvent(final Map data) { - return new Record<>(JacksonEvent.builder() + return new Record<>(TEST_EVENT_FACTORY.eventBuilder(EventBuilder.class) .withData(data) .withEventType("event") .build()); diff --git a/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/UppercaseStringProcessorTests.java b/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/UppercaseStringProcessorTests.java index 14af79d202..c4db6a55e5 100644 --- a/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/UppercaseStringProcessorTests.java +++ b/data-prepper-plugins/mutate-string-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutatestring/UppercaseStringProcessorTests.java @@ -5,21 +5,26 @@ package org.opensearch.dataprepper.plugins.processor.mutatestring; -import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.event.JacksonEvent; -import org.opensearch.dataprepper.model.record.Record; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.event.TestEventFactory; +import org.opensearch.dataprepper.event.TestEventKeyFactory; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventBuilder; +import org.opensearch.dataprepper.model.event.EventFactory; +import org.opensearch.dataprepper.model.event.EventKeyFactory; +import org.opensearch.dataprepper.model.record.Record; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; @@ -29,6 +34,9 @@ @ExtendWith(MockitoExtension.class) public class UppercaseStringProcessorTests { + private static final EventFactory TEST_EVENT_FACTORY = TestEventFactory.getTestEventFactory(); + private final EventKeyFactory eventKeyFactory = TestEventKeyFactory.getTestEventFactory(); + @Mock private PluginMetrics pluginMetrics; @@ -37,7 +45,7 @@ public class UppercaseStringProcessorTests { @BeforeEach public void setup() { - lenient().when(config.getIterativeConfig()).thenReturn(Collections.singletonList("message")); + lenient().when(config.getIterativeConfig()).thenReturn(Stream.of("message").map(eventKeyFactory::createEventKey).collect(Collectors.toList())); } @Test @@ -52,7 +60,7 @@ public void testHappyPathUppercaseStringProcessor() { @Test public void testHappyPathMultiUppercaseStringProcessor() { - when(config.getIterativeConfig()).thenReturn(Arrays.asList("message", "message2")); + when(config.getIterativeConfig()).thenReturn(Stream.of("message", "message2").map(eventKeyFactory::createEventKey).collect(Collectors.toList())); final UppercaseStringProcessor processor = createObjectUnderTest(); final Record record = getEvent("thisisamessage"); @@ -67,7 +75,7 @@ public void testHappyPathMultiUppercaseStringProcessor() { @Test public void testHappyPathMultiMixedUppercaseStringProcessor() { - lenient().when(config.getIterativeConfig()).thenReturn(Arrays.asList("message", "message2")); + lenient().when(config.getIterativeConfig()).thenReturn(Stream.of("message", "message2").map(eventKeyFactory::createEventKey).collect(Collectors.toList())); final UppercaseStringProcessor processor = createObjectUnderTest(); final Record record = getEvent("thisisamessage"); @@ -137,7 +145,7 @@ private Record getEvent(Object message) { } private static Record buildRecordWithEvent(final Map data) { - return new Record<>(JacksonEvent.builder() + return new Record<>(TEST_EVENT_FACTORY.eventBuilder(EventBuilder.class) .withData(data) .withEventType("event") .build()); diff --git a/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java b/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java index b99753bc9f..e5893476e0 100644 --- a/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java +++ b/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java @@ -6,6 +6,7 @@ package org.opensearch.dataprepper.plugins.processor.obfuscation; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import org.opensearch.dataprepper.expression.ExpressionEvaluator; @@ -17,6 +18,7 @@ public class ObfuscationProcessorConfig { @JsonProperty("source") + @JsonPropertyDescription("The source field to obfuscate.") @NotEmpty @NotNull private String source; @@ -25,18 +27,29 @@ public class ObfuscationProcessorConfig { private List patterns; @JsonProperty("target") + @JsonPropertyDescription("The new field in which to store the obfuscated value. " + + "This leaves the original source field unchanged. " + + "When no `target` is provided, the source field updates with the obfuscated value.") private String target; @JsonProperty("action") + @JsonPropertyDescription("The obfuscation action. As of Data Prepper 2.3, only the `mask` action is supported.") private PluginModel action; @JsonProperty("obfuscate_when") + @JsonPropertyDescription("Specifies under what condition the Obfuscate processor should perform matching. " + + "Default is no condition.") private String obfuscateWhen; @JsonProperty("tags_on_match_failure") + @JsonPropertyDescription("The tag to add to an event if the obfuscate processor fails to match the pattern.") private List tagsOnMatchFailure; @JsonProperty("single_word_only") + @JsonPropertyDescription("When set to `true`, a word boundary `\b` is added to the pattern, " + + "which causes obfuscation to be applied only to words that are standalone in the input text. " + + "By default, it is false, meaning obfuscation patterns are applied to all occurrences. " + + "Can be used for Data Prepper 2.8 or greater.") private boolean singleWordOnly = false; public ObfuscationProcessorConfig() { diff --git a/data-prepper-plugins/opensearch/build.gradle b/data-prepper-plugins/opensearch/build.gradle index bece32eaae..5e7879d8d1 100644 --- a/data-prepper-plugins/opensearch/build.gradle +++ b/data-prepper-plugins/opensearch/build.gradle @@ -44,7 +44,6 @@ dependencies { testImplementation 'net.bytebuddy:byte-buddy:1.14.17' testImplementation 'net.bytebuddy:byte-buddy-agent:1.14.17' testImplementation testLibs.slf4j.simple - testImplementation testLibs.mockito.inline } sourceSets { diff --git a/data-prepper-plugins/opensearch/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-plugins/opensearch/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 78ccc25012..0000000000 --- a/data-prepper-plugins/opensearch/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1,3 +0,0 @@ -# To enable mocking of final classes with vanilla Mockito -# https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods -mock-maker-inline \ No newline at end of file diff --git a/data-prepper-plugins/otel-logs-source/build.gradle b/data-prepper-plugins/otel-logs-source/build.gradle index 97901da8c3..822e945ba9 100644 --- a/data-prepper-plugins/otel-logs-source/build.gradle +++ b/data-prepper-plugins/otel-logs-source/build.gradle @@ -31,7 +31,6 @@ dependencies { implementation libs.bouncycastle.bcprov implementation libs.bouncycastle.bcpkix testImplementation 'org.assertj:assertj-core:3.25.3' - testImplementation testLibs.mockito.inline testImplementation libs.commons.io } diff --git a/data-prepper-plugins/otel-logs-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-plugins/otel-logs-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 78ccc25012..0000000000 --- a/data-prepper-plugins/otel-logs-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1,3 +0,0 @@ -# To enable mocking of final classes with vanilla Mockito -# https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods -mock-maker-inline \ No newline at end of file diff --git a/data-prepper-plugins/otel-metrics-raw-processor/build.gradle b/data-prepper-plugins/otel-metrics-raw-processor/build.gradle index af20b2e74b..a4316fca16 100644 --- a/data-prepper-plugins/otel-metrics-raw-processor/build.gradle +++ b/data-prepper-plugins/otel-metrics-raw-processor/build.gradle @@ -22,7 +22,6 @@ dependencies { implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' implementation libs.guava.core testImplementation 'org.assertj:assertj-core:3.25.3' - testImplementation testLibs.mockito.inline } jacocoTestCoverageVerification { diff --git a/data-prepper-plugins/otel-metrics-raw-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/OtelMetricsRawProcessorConfig.java b/data-prepper-plugins/otel-metrics-raw-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/OtelMetricsRawProcessorConfig.java index 9935cc9218..b71a0d1800 100644 --- a/data-prepper-plugins/otel-metrics-raw-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/OtelMetricsRawProcessorConfig.java +++ b/data-prepper-plugins/otel-metrics-raw-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/OtelMetricsRawProcessorConfig.java @@ -6,17 +6,23 @@ package org.opensearch.dataprepper.plugins.processor.otelmetrics; import static org.opensearch.dataprepper.plugins.otel.codec.OTelProtoCodec.DEFAULT_EXPONENTIAL_HISTOGRAM_MAX_ALLOWED_SCALE; + import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; public class OtelMetricsRawProcessorConfig { @JsonProperty("flatten_attributes") + @JsonPropertyDescription("Whether or not to flatten the `attributes` field in the JSON data.") boolean flattenAttributesFlag = true; + @JsonPropertyDescription("Whether or not to calculate histogram buckets.") private Boolean calculateHistogramBuckets = true; + @JsonPropertyDescription("Whether or not to calculate exponential histogram buckets.") private Boolean calculateExponentialHistogramBuckets = true; + @JsonPropertyDescription("Maximum allowed scale in exponential histogram calculation.") private Integer exponentialHistogramMaxAllowedScale = DEFAULT_EXPONENTIAL_HISTOGRAM_MAX_ALLOWED_SCALE; public Boolean getCalculateExponentialHistogramBuckets() { diff --git a/data-prepper-plugins/otel-metrics-source/build.gradle b/data-prepper-plugins/otel-metrics-source/build.gradle index 25ea578566..96d250d67d 100644 --- a/data-prepper-plugins/otel-metrics-source/build.gradle +++ b/data-prepper-plugins/otel-metrics-source/build.gradle @@ -31,7 +31,6 @@ dependencies { implementation libs.bouncycastle.bcprov implementation libs.bouncycastle.bcpkix testImplementation 'org.assertj:assertj-core:3.25.3' - testImplementation testLibs.mockito.inline testImplementation libs.commons.io } diff --git a/data-prepper-plugins/otel-metrics-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-plugins/otel-metrics-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 78ccc25012..0000000000 --- a/data-prepper-plugins/otel-metrics-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1,3 +0,0 @@ -# To enable mocking of final classes with vanilla Mockito -# https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods -mock-maker-inline \ No newline at end of file diff --git a/data-prepper-plugins/otel-trace-group-processor/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-plugins/otel-trace-group-processor/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 78ccc25012..0000000000 --- a/data-prepper-plugins/otel-trace-group-processor/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1,3 +0,0 @@ -# To enable mocking of final classes with vanilla Mockito -# https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods -mock-maker-inline \ No newline at end of file diff --git a/data-prepper-plugins/otel-trace-raw-processor/build.gradle b/data-prepper-plugins/otel-trace-raw-processor/build.gradle index ff2bfc4a60..2df90630d8 100644 --- a/data-prepper-plugins/otel-trace-raw-processor/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-processor/build.gradle @@ -20,7 +20,6 @@ dependencies { implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' implementation libs.caffeine testImplementation 'org.assertj:assertj-core:3.25.3' - testImplementation testLibs.mockito.inline } jacocoTestCoverageVerification { diff --git a/data-prepper-plugins/otel-trace-raw-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/oteltrace/OtelTraceRawProcessorConfig.java b/data-prepper-plugins/otel-trace-raw-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/oteltrace/OtelTraceRawProcessorConfig.java index 553e1ed2d1..6b850f7354 100644 --- a/data-prepper-plugins/otel-trace-raw-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/oteltrace/OtelTraceRawProcessorConfig.java +++ b/data-prepper-plugins/otel-trace-raw-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/oteltrace/OtelTraceRawProcessorConfig.java @@ -6,6 +6,7 @@ package org.opensearch.dataprepper.plugins.processor.oteltrace; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import java.time.Duration; @@ -14,12 +15,17 @@ public class OtelTraceRawProcessorConfig { static final Duration DEFAULT_TRACE_ID_TTL = Duration.ofSeconds(15L); static final long MAX_TRACE_ID_CACHE_SIZE = 1_000_000L; @JsonProperty("trace_flush_interval") + @JsonPropertyDescription("Represents the time interval in seconds to flush all the descendant spans without any " + + "root span. Default is 180.") private long traceFlushInterval = DEFAULT_TG_FLUSH_INTERVAL_SEC; @JsonProperty("trace_group_cache_ttl") + @JsonPropertyDescription("Represents the time-to-live to cache a trace group details. Default is 15 seconds.") private Duration traceGroupCacheTimeToLive = DEFAULT_TRACE_ID_TTL; @JsonProperty("trace_group_cache_max_size") + @JsonPropertyDescription("Represents the maximum size of the cache to store the trace group details from root spans. " + + "Default is 1000000.") private long traceGroupCacheMaxSize = MAX_TRACE_ID_CACHE_SIZE; public long getTraceFlushIntervalSeconds() { diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 39c0869851..d1dcdfa12a 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -29,7 +29,6 @@ dependencies { implementation libs.bouncycastle.bcprov implementation libs.bouncycastle.bcpkix testImplementation 'org.assertj:assertj-core:3.25.3' - testImplementation testLibs.mockito.inline testImplementation testLibs.slf4j.simple testImplementation libs.commons.io } diff --git a/data-prepper-plugins/otel-trace-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-plugins/otel-trace-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 78ccc25012..0000000000 --- a/data-prepper-plugins/otel-trace-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1,3 +0,0 @@ -# To enable mocking of final classes with vanilla Mockito -# https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods -mock-maker-inline \ No newline at end of file diff --git a/data-prepper-plugins/parquet-codecs/build.gradle b/data-prepper-plugins/parquet-codecs/build.gradle index fbc8f4a209..c402fb6741 100644 --- a/data-prepper-plugins/parquet-codecs/build.gradle +++ b/data-prepper-plugins/parquet-codecs/build.gradle @@ -15,15 +15,19 @@ dependencies { runtimeOnly(libs.hadoop.common) { exclude group: 'org.eclipse.jetty' exclude group: 'org.apache.hadoop', module: 'hadoop-auth' + exclude group: 'org.apache.zookeeper', module: 'zookeeper' } runtimeOnly(libs.hadoop.mapreduce) { + exclude group: 'org.eclipse.jetty' exclude group: 'org.apache.hadoop', module: 'hadoop-hdfs-client' + exclude group: 'org.apache.zookeeper', module: 'zookeeper' } testImplementation project(':data-prepper-test-common') testImplementation project(':data-prepper-test-event') testImplementation(libs.hadoop.common) { exclude group: 'org.eclipse.jetty' exclude group: 'org.apache.hadoop', module: 'hadoop-auth' + exclude group: 'org.apache.zookeeper', module: 'zookeeper' } constraints { diff --git a/data-prepper-plugins/prometheus-sink/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-plugins/prometheus-sink/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 23c33feb6d..0000000000 --- a/data-prepper-plugins/prometheus-sink/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1,3 +0,0 @@ -# To enable mocking of final classes with vanilla Mockito -# https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods -mock-maker-inline diff --git a/data-prepper-plugins/rds-source/build.gradle b/data-prepper-plugins/rds-source/build.gradle index 8372276564..f83b1332eb 100644 --- a/data-prepper-plugins/rds-source/build.gradle +++ b/data-prepper-plugins/rds-source/build.gradle @@ -8,6 +8,7 @@ dependencies { implementation project(path: ':data-prepper-plugins:buffer-common') implementation project(path: ':data-prepper-plugins:http-common') implementation project(path: ':data-prepper-plugins:common') + implementation project(path: ':data-prepper-plugins:parquet-codecs') implementation 'io.micrometer:micrometer-core' @@ -20,7 +21,7 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-core' implementation 'com.fasterxml.jackson.core:jackson-databind' - testImplementation testLibs.mockito.inline testImplementation project(path: ':data-prepper-test-common') testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' + testImplementation project(path: ':data-prepper-test-event') } diff --git a/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/ClientFactory.java b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/ClientFactory.java index 9cdb2bfa50..7831754f0f 100644 --- a/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/ClientFactory.java +++ b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/ClientFactory.java @@ -10,6 +10,7 @@ import org.opensearch.dataprepper.plugins.source.rds.configuration.AwsAuthenticationConfig; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.services.rds.RdsClient; +import software.amazon.awssdk.services.s3.S3Client; public class ClientFactory { private final AwsCredentialsProvider awsCredentialsProvider; @@ -32,4 +33,11 @@ public RdsClient buildRdsClient() { .credentialsProvider(awsCredentialsProvider) .build(); } + + public S3Client buildS3Client() { + return S3Client.builder() + .region(awsAuthenticationConfig.getAwsRegion()) + .credentialsProvider(awsCredentialsProvider) + .build(); + } } diff --git a/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/RdsService.java b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/RdsService.java index f059dd52bf..77956e6b0e 100644 --- a/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/RdsService.java +++ b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/RdsService.java @@ -8,13 +8,16 @@ import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator; +import org.opensearch.dataprepper.plugins.source.rds.export.DataFileScheduler; import org.opensearch.dataprepper.plugins.source.rds.export.ExportScheduler; import org.opensearch.dataprepper.plugins.source.rds.leader.LeaderScheduler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.rds.RdsClient; +import software.amazon.awssdk.services.s3.S3Client; import java.util.ArrayList; import java.util.List; @@ -24,23 +27,34 @@ public class RdsService { private static final Logger LOG = LoggerFactory.getLogger(RdsService.class); + /** + * Maximum concurrent data loader per node + */ + public static final int DATA_LOADER_MAX_JOB_COUNT = 1; + private final RdsClient rdsClient; + private final S3Client s3Client; private final EnhancedSourceCoordinator sourceCoordinator; + private final EventFactory eventFactory; private final PluginMetrics pluginMetrics; private final RdsSourceConfig sourceConfig; private ExecutorService executor; private LeaderScheduler leaderScheduler; private ExportScheduler exportScheduler; + private DataFileScheduler dataFileScheduler; public RdsService(final EnhancedSourceCoordinator sourceCoordinator, final RdsSourceConfig sourceConfig, + final EventFactory eventFactory, final ClientFactory clientFactory, final PluginMetrics pluginMetrics) { this.sourceCoordinator = sourceCoordinator; + this.eventFactory = eventFactory; this.pluginMetrics = pluginMetrics; this.sourceConfig = sourceConfig; rdsClient = clientFactory.buildRdsClient(); + s3Client = clientFactory.buildS3Client(); } /** @@ -54,9 +68,15 @@ public void start(Buffer> buffer) { LOG.info("Start running RDS service"); final List runnableList = new ArrayList<>(); leaderScheduler = new LeaderScheduler(sourceCoordinator, sourceConfig); - exportScheduler = new ExportScheduler(sourceCoordinator, rdsClient, pluginMetrics); runnableList.add(leaderScheduler); - runnableList.add(exportScheduler); + + if (sourceConfig.isExportEnabled()) { + exportScheduler = new ExportScheduler(sourceCoordinator, rdsClient, s3Client, pluginMetrics); + dataFileScheduler = new DataFileScheduler( + sourceCoordinator, sourceConfig, s3Client, eventFactory, buffer); + runnableList.add(exportScheduler); + runnableList.add(dataFileScheduler); + } executor = Executors.newFixedThreadPool(runnableList.size()); runnableList.forEach(executor::submit); @@ -69,7 +89,10 @@ public void start(Buffer> buffer) { public void shutdown() { if (executor != null) { LOG.info("shutdown RDS schedulers"); - exportScheduler.shutdown(); + if (sourceConfig.isExportEnabled()) { + exportScheduler.shutdown(); + dataFileScheduler.shutdown(); + } leaderScheduler.shutdown(); executor.shutdownNow(); } diff --git a/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/RdsSource.java b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/RdsSource.java index a9fe983572..43806c0475 100644 --- a/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/RdsSource.java +++ b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/RdsSource.java @@ -11,6 +11,7 @@ import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.source.Source; import org.opensearch.dataprepper.model.source.coordinator.SourcePartitionStoreItem; @@ -33,15 +34,18 @@ public class RdsSource implements Source>, UsesEnhancedSourceCoord private final ClientFactory clientFactory; private final PluginMetrics pluginMetrics; private final RdsSourceConfig sourceConfig; + private final EventFactory eventFactory; private EnhancedSourceCoordinator sourceCoordinator; private RdsService rdsService; @DataPrepperPluginConstructor public RdsSource(final PluginMetrics pluginMetrics, final RdsSourceConfig sourceConfig, + final EventFactory eventFactory, final AwsCredentialsSupplier awsCredentialsSupplier) { this.pluginMetrics = pluginMetrics; this.sourceConfig = sourceConfig; + this.eventFactory = eventFactory; clientFactory = new ClientFactory(awsCredentialsSupplier, sourceConfig.getAwsAuthenticationConfig()); } @@ -51,7 +55,7 @@ public void start(Buffer> buffer) { Objects.requireNonNull(sourceCoordinator); sourceCoordinator.createPartition(new LeaderPartition()); - rdsService = new RdsService(sourceCoordinator, sourceConfig, clientFactory, pluginMetrics); + rdsService = new RdsService(sourceCoordinator, sourceConfig, eventFactory, clientFactory, pluginMetrics); LOG.info("Start RDS service"); rdsService.start(buffer); diff --git a/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/converter/ExportRecordConverter.java b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/converter/ExportRecordConverter.java new file mode 100644 index 0000000000..11932cd512 --- /dev/null +++ b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/converter/ExportRecordConverter.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.rds.converter; + +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventMetadata; +import org.opensearch.dataprepper.model.record.Record; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.opensearch.dataprepper.plugins.source.rds.converter.MetadataKeyAttributes.EVENT_TABLE_NAME_METADATA_ATTRIBUTE; +import static org.opensearch.dataprepper.plugins.source.rds.converter.MetadataKeyAttributes.INGESTION_EVENT_TYPE_ATTRIBUTE; +import static org.opensearch.dataprepper.plugins.source.rds.converter.MetadataKeyAttributes.PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE; + +public class ExportRecordConverter { + + private static final Logger LOG = LoggerFactory.getLogger(ExportRecordConverter.class); + + static final String EXPORT_EVENT_TYPE = "EXPORT"; + + public Event convert(Record record, String tableName, String primaryKeyName) { + Event event = record.getData(); + + EventMetadata eventMetadata = event.getMetadata(); + eventMetadata.setAttribute(EVENT_TABLE_NAME_METADATA_ATTRIBUTE, tableName); + eventMetadata.setAttribute(INGESTION_EVENT_TYPE_ATTRIBUTE, EXPORT_EVENT_TYPE); + + final Object primaryKeyValue = record.getData().get(primaryKeyName, Object.class); + eventMetadata.setAttribute(PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE, primaryKeyValue); + + return event; + } +} diff --git a/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/converter/MetadataKeyAttributes.java b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/converter/MetadataKeyAttributes.java new file mode 100644 index 0000000000..91eecdf07b --- /dev/null +++ b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/converter/MetadataKeyAttributes.java @@ -0,0 +1,20 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.rds.converter; + +public class MetadataKeyAttributes { + static final String PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE = "primary_key"; + + static final String EVENT_VERSION_FROM_TIMESTAMP = "document_version"; + + static final String EVENT_TIMESTAMP_METADATA_ATTRIBUTE = "event_timestamp"; + + static final String EVENT_NAME_BULK_ACTION_METADATA_ATTRIBUTE = "opensearch_action"; + + static final String EVENT_TABLE_NAME_METADATA_ATTRIBUTE = "table_name"; + + static final String INGESTION_EVENT_TYPE_ATTRIBUTE = "ingestion_type"; +} diff --git a/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/coordination/PartitionFactory.java b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/coordination/PartitionFactory.java index db35f5076b..6213263b09 100644 --- a/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/coordination/PartitionFactory.java +++ b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/coordination/PartitionFactory.java @@ -7,6 +7,7 @@ import org.opensearch.dataprepper.model.source.coordinator.SourcePartitionStoreItem; import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourcePartition; +import org.opensearch.dataprepper.plugins.source.rds.coordination.partition.DataFilePartition; import org.opensearch.dataprepper.plugins.source.rds.coordination.partition.ExportPartition; import org.opensearch.dataprepper.plugins.source.rds.coordination.partition.GlobalState; import org.opensearch.dataprepper.plugins.source.rds.coordination.partition.LeaderPartition; @@ -25,8 +26,10 @@ public EnhancedSourcePartition apply(SourcePartitionStoreItem partitionStoreItem if (LeaderPartition.PARTITION_TYPE.equals(partitionType)) { return new LeaderPartition(partitionStoreItem); - } if (ExportPartition.PARTITION_TYPE.equals(partitionType)) { + } else if (ExportPartition.PARTITION_TYPE.equals(partitionType)) { return new ExportPartition(partitionStoreItem); + } else if (DataFilePartition.PARTITION_TYPE.equals(partitionType)) { + return new DataFilePartition(partitionStoreItem); } else { // Unable to acquire other partitions. return new GlobalState(partitionStoreItem); diff --git a/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/coordination/partition/DataFilePartition.java b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/coordination/partition/DataFilePartition.java new file mode 100644 index 0000000000..985f48b652 --- /dev/null +++ b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/coordination/partition/DataFilePartition.java @@ -0,0 +1,77 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.rds.coordination.partition; + +import org.opensearch.dataprepper.model.source.coordinator.SourcePartitionStoreItem; +import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourcePartition; +import org.opensearch.dataprepper.plugins.source.rds.coordination.state.DataFileProgressState; + +import java.util.Optional; + +/** + * An DataFilePartition represents an export data file needs to be loaded. + * The source identifier contains keyword 'DATAFILE' + */ +public class DataFilePartition extends EnhancedSourcePartition { + + public static final String PARTITION_TYPE = "DATAFILE"; + + private final String exportTaskId; + private final String bucket; + private final String key; + private final DataFileProgressState state; + + public DataFilePartition(final SourcePartitionStoreItem sourcePartitionStoreItem) { + + setSourcePartitionStoreItem(sourcePartitionStoreItem); + String[] keySplits = sourcePartitionStoreItem.getSourcePartitionKey().split("\\|"); + exportTaskId = keySplits[0]; + bucket = keySplits[1]; + key = keySplits[2]; + state = convertStringToPartitionProgressState(DataFileProgressState.class, sourcePartitionStoreItem.getPartitionProgressState()); + + } + + public DataFilePartition(final String exportTaskId, + final String bucket, + final String key, + final Optional state) { + this.exportTaskId = exportTaskId; + this.bucket = bucket; + this.key = key; + this.state = state.orElse(null); + } + + @Override + public String getPartitionType() { + return PARTITION_TYPE; + } + + @Override + public String getPartitionKey() { + return exportTaskId + "|" + bucket + "|" + key; + } + + @Override + public Optional getProgressState() { + if (state != null) { + return Optional.of(state); + } + return Optional.empty(); + } + + public String getExportTaskId() { + return exportTaskId; + } + + public String getBucket() { + return bucket; + } + + public String getKey() { + return key; + } +} diff --git a/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/coordination/state/DataFileProgressState.java b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/coordination/state/DataFileProgressState.java new file mode 100644 index 0000000000..c65c0bbe01 --- /dev/null +++ b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/coordination/state/DataFileProgressState.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.rds.coordination.state; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class DataFileProgressState { + + @JsonProperty("isLoaded") + private boolean isLoaded = false; + + @JsonProperty("totalRecords") + private int totalRecords; + + @JsonProperty("sourceTable") + private String sourceTable; + + public int getTotalRecords() { + return totalRecords; + } + + public void setTotalRecords(int totalRecords) { + this.totalRecords = totalRecords; + } + + public boolean getLoaded() { + return isLoaded; + } + + public void setLoaded(boolean loaded) { + this.isLoaded = loaded; + } + + public String getSourceTable() { + return sourceTable; + } + + public void setSourceTable(String sourceTable) { + this.sourceTable = sourceTable; + } +} diff --git a/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/export/DataFileLoader.java b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/export/DataFileLoader.java new file mode 100644 index 0000000000..e76a04e99d --- /dev/null +++ b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/export/DataFileLoader.java @@ -0,0 +1,83 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.rds.export; + +import org.opensearch.dataprepper.buffer.common.BufferAccumulator; +import org.opensearch.dataprepper.model.codec.InputCodec; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.source.rds.converter.ExportRecordConverter; +import org.opensearch.dataprepper.plugins.source.rds.coordination.partition.DataFilePartition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.InputStream; + +public class DataFileLoader implements Runnable { + + private static final Logger LOG = LoggerFactory.getLogger(DataFileLoader.class); + + private final DataFilePartition dataFilePartition; + private final String bucket; + private final String objectKey; + private final S3ObjectReader objectReader; + private final InputCodec codec; + private final BufferAccumulator> bufferAccumulator; + private final ExportRecordConverter recordConverter; + + private DataFileLoader(final DataFilePartition dataFilePartition, + final InputCodec codec, + final BufferAccumulator> bufferAccumulator, + final S3ObjectReader objectReader, + final ExportRecordConverter recordConverter) { + this.dataFilePartition = dataFilePartition; + bucket = dataFilePartition.getBucket(); + objectKey = dataFilePartition.getKey(); + this.objectReader = objectReader; + this.codec = codec; + this.bufferAccumulator = bufferAccumulator; + this.recordConverter = recordConverter; + } + + public static DataFileLoader create(final DataFilePartition dataFilePartition, + final InputCodec codec, + final BufferAccumulator> bufferAccumulator, + final S3ObjectReader objectReader, + final ExportRecordConverter recordConverter) { + return new DataFileLoader(dataFilePartition, codec, bufferAccumulator, objectReader, recordConverter); + } + + @Override + public void run() { + LOG.info("Start loading s3://{}/{}", bucket, objectKey); + + try (InputStream inputStream = objectReader.readFile(bucket, objectKey)) { + + codec.parse(inputStream, record -> { + try { + final String tableName = dataFilePartition.getProgressState().get().getSourceTable(); + // TODO: primary key to be obtained by querying database schema + final String primaryKeyName = "id"; + Record transformedRecord = new Record<>(recordConverter.convert(record, tableName, primaryKeyName)); + bufferAccumulator.add(transformedRecord); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + LOG.info("Completed loading object s3://{}/{} to buffer", bucket, objectKey); + } catch (Exception e) { + LOG.error("Failed to load object s3://{}/{} to buffer", bucket, objectKey, e); + throw new RuntimeException(e); + } + + try { + bufferAccumulator.flush(); + } catch (Exception e) { + LOG.error("Failed to write events to buffer", e); + } + } +} diff --git a/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/export/DataFileScheduler.java b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/export/DataFileScheduler.java new file mode 100644 index 0000000000..d465d55076 --- /dev/null +++ b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/export/DataFileScheduler.java @@ -0,0 +1,163 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.rds.export; + +import org.opensearch.dataprepper.buffer.common.BufferAccumulator; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.codec.InputCodec; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator; +import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourcePartition; +import org.opensearch.dataprepper.plugins.codec.parquet.ParquetInputCodec; +import org.opensearch.dataprepper.plugins.source.rds.RdsSourceConfig; +import org.opensearch.dataprepper.plugins.source.rds.converter.ExportRecordConverter; +import org.opensearch.dataprepper.plugins.source.rds.coordination.partition.DataFilePartition; +import org.opensearch.dataprepper.plugins.source.rds.coordination.partition.GlobalState; +import org.opensearch.dataprepper.plugins.source.rds.model.LoadStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.s3.S3Client; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.opensearch.dataprepper.plugins.source.rds.RdsService.DATA_LOADER_MAX_JOB_COUNT; + +public class DataFileScheduler implements Runnable { + + private static final Logger LOG = LoggerFactory.getLogger(DataFileScheduler.class); + + private final AtomicInteger numOfWorkers = new AtomicInteger(0); + + /** + * Default interval to acquire a lease from coordination store + */ + private static final int DEFAULT_LEASE_INTERVAL_MILLIS = 2_000; + + private static final Duration DEFAULT_UPDATE_LOAD_STATUS_TIMEOUT = Duration.ofMinutes(30); + + static final Duration BUFFER_TIMEOUT = Duration.ofSeconds(60); + static final int DEFAULT_BUFFER_BATCH_SIZE = 1_000; + + + private final EnhancedSourceCoordinator sourceCoordinator; + private final ExecutorService executor; + private final RdsSourceConfig sourceConfig; + private final S3ObjectReader objectReader; + private final InputCodec codec; + private final BufferAccumulator> bufferAccumulator; + private final ExportRecordConverter recordConverter; + + private volatile boolean shutdownRequested = false; + + public DataFileScheduler(final EnhancedSourceCoordinator sourceCoordinator, + final RdsSourceConfig sourceConfig, + final S3Client s3Client, + final EventFactory eventFactory, + final Buffer> buffer) { + this.sourceCoordinator = sourceCoordinator; + this.sourceConfig = sourceConfig; + codec = new ParquetInputCodec(eventFactory); + bufferAccumulator = BufferAccumulator.create(buffer, DEFAULT_BUFFER_BATCH_SIZE, BUFFER_TIMEOUT); + objectReader = new S3ObjectReader(s3Client); + recordConverter = new ExportRecordConverter(); + executor = Executors.newFixedThreadPool(DATA_LOADER_MAX_JOB_COUNT); + } + + @Override + public void run() { + LOG.debug("Starting Data File Scheduler to process S3 data files for export"); + + while (!shutdownRequested && !Thread.currentThread().isInterrupted()) { + try { + if (numOfWorkers.get() < DATA_LOADER_MAX_JOB_COUNT) { + final Optional sourcePartition = sourceCoordinator.acquireAvailablePartition(DataFilePartition.PARTITION_TYPE); + + if (sourcePartition.isPresent()) { + LOG.debug("Acquired data file partition"); + DataFilePartition dataFilePartition = (DataFilePartition) sourcePartition.get(); + LOG.debug("Start processing data file partition"); + processDataFilePartition(dataFilePartition); + } + } + try { + Thread.sleep(DEFAULT_LEASE_INTERVAL_MILLIS); + } catch (final InterruptedException e) { + LOG.info("The DataFileScheduler was interrupted while waiting to retry, stopping processing"); + break; + } + } catch (final Exception e) { + LOG.error("Received an exception while processing an S3 data file, backing off and retrying", e); + try { + Thread.sleep(DEFAULT_LEASE_INTERVAL_MILLIS); + } catch (final InterruptedException ex) { + LOG.info("The DataFileScheduler was interrupted while waiting to retry, stopping processing"); + break; + } + } + } + LOG.warn("Data file scheduler is interrupted, stopping all data file loaders..."); + + executor.shutdown(); + } + + public void shutdown() { + shutdownRequested = true; + } + + private void processDataFilePartition(DataFilePartition dataFilePartition) { + Runnable loader = DataFileLoader.create(dataFilePartition, codec, bufferAccumulator, objectReader, recordConverter); + CompletableFuture runLoader = CompletableFuture.runAsync(loader, executor); + + runLoader.whenComplete((v, ex) -> { + if (ex == null) { + // Update global state so we know if all s3 files have been loaded + updateLoadStatus(dataFilePartition.getExportTaskId(), DEFAULT_UPDATE_LOAD_STATUS_TIMEOUT); + sourceCoordinator.completePartition(dataFilePartition); + } else { + LOG.error("There was an exception while processing an S3 data file", (Throwable) ex); + sourceCoordinator.giveUpPartition(dataFilePartition); + } + numOfWorkers.decrementAndGet(); + }); + numOfWorkers.incrementAndGet(); + } + + private void updateLoadStatus(String exportTaskId, Duration timeout) { + + Instant endTime = Instant.now().plus(timeout); + // Keep retrying in case update fails due to conflicts until timed out + while (Instant.now().isBefore(endTime)) { + Optional globalStatePartition = sourceCoordinator.getPartition(exportTaskId); + if (globalStatePartition.isEmpty()) { + LOG.error("Failed to get data file load status for {}", exportTaskId); + return; + } + + GlobalState globalState = (GlobalState) globalStatePartition.get(); + LoadStatus loadStatus = LoadStatus.fromMap(globalState.getProgressState().get()); + loadStatus.setLoadedFiles(loadStatus.getLoadedFiles() + 1); + LOG.info("Current data file load status: total {} loaded {}", loadStatus.getTotalFiles(), loadStatus.getLoadedFiles()); + + globalState.setProgressState(loadStatus.toMap()); + + try { + sourceCoordinator.saveProgressStateForPartition(globalState, null); + // TODO: Stream is enabled and loadStatus.getLoadedFiles() == loadStatus.getTotalFiles(), create global state to indicate that stream can start + break; + } catch (Exception e) { + LOG.error("Failed to update the global status, looks like the status was out of date, will retry.."); + } + } + } +} diff --git a/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/export/ExportScheduler.java b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/export/ExportScheduler.java index 51db82248b..abcbd2c1f4 100644 --- a/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/export/ExportScheduler.java +++ b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/export/ExportScheduler.java @@ -8,22 +8,36 @@ import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator; import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourcePartition; +import org.opensearch.dataprepper.plugins.source.rds.coordination.partition.DataFilePartition; import org.opensearch.dataprepper.plugins.source.rds.coordination.partition.ExportPartition; +import org.opensearch.dataprepper.plugins.source.rds.coordination.partition.GlobalState; +import org.opensearch.dataprepper.plugins.source.rds.coordination.state.DataFileProgressState; import org.opensearch.dataprepper.plugins.source.rds.coordination.state.ExportProgressState; +import org.opensearch.dataprepper.plugins.source.rds.model.ExportObjectKey; import org.opensearch.dataprepper.plugins.source.rds.model.ExportStatus; +import org.opensearch.dataprepper.plugins.source.rds.model.LoadStatus; import org.opensearch.dataprepper.plugins.source.rds.model.SnapshotInfo; import org.opensearch.dataprepper.plugins.source.rds.model.SnapshotStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.rds.RdsClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.S3Object; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; +import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; +import java.util.stream.Collectors; public class ExportScheduler implements Runnable { private static final Logger LOG = LoggerFactory.getLogger(ExportScheduler.class); @@ -34,8 +48,10 @@ public class ExportScheduler implements Runnable { private static final int DEFAULT_CHECKPOINT_INTERVAL_MILLS = 5 * 60_000; private static final int DEFAULT_CHECK_STATUS_INTERVAL_MILLS = 30 * 1000; private static final Duration DEFAULT_SNAPSHOT_STATUS_CHECK_TIMEOUT = Duration.ofMinutes(60); + static final String PARQUET_SUFFIX = ".parquet"; private final RdsClient rdsClient; + private final S3Client s3Client; private final PluginMetrics pluginMetrics; private final EnhancedSourceCoordinator sourceCoordinator; private final ExecutorService executor; @@ -46,10 +62,12 @@ public class ExportScheduler implements Runnable { public ExportScheduler(final EnhancedSourceCoordinator sourceCoordinator, final RdsClient rdsClient, + final S3Client s3Client, final PluginMetrics pluginMetrics) { this.pluginMetrics = pluginMetrics; this.sourceCoordinator = sourceCoordinator; this.rdsClient = rdsClient; + this.s3Client = s3Client; this.executor = Executors.newCachedThreadPool(); this.exportTaskManager = new ExportTaskManager(rdsClient); this.snapshotManager = new SnapshotManager(rdsClient); @@ -72,7 +90,8 @@ public void run() { LOG.error("The export to S3 failed, it will be retried"); closeExportPartitionWithError(exportPartition); } else { - CompletableFuture checkStatus = CompletableFuture.supplyAsync(() -> checkExportStatus(exportPartition), executor); + CheckExportStatusRunner checkExportStatusRunner = new CheckExportStatusRunner(sourceCoordinator, exportTaskManager, exportPartition); + CompletableFuture checkStatus = CompletableFuture.supplyAsync(checkExportStatusRunner::call, executor); checkStatus.whenComplete(completeExport(exportPartition)); } } @@ -179,29 +198,46 @@ private String checkSnapshotStatus(String snapshotId, Duration timeout) { throw new RuntimeException("Snapshot status check timed out."); } - private String checkExportStatus(ExportPartition exportPartition) { - long lastCheckpointTime = System.currentTimeMillis(); - String exportTaskId = exportPartition.getProgressState().get().getExportTaskId(); + static class CheckExportStatusRunner implements Callable { + private final EnhancedSourceCoordinator sourceCoordinator; + private final ExportTaskManager exportTaskManager; + private final ExportPartition exportPartition; - LOG.debug("Start checking the status of export {}", exportTaskId); - while (true) { - if (System.currentTimeMillis() - lastCheckpointTime > DEFAULT_CHECKPOINT_INTERVAL_MILLS) { - sourceCoordinator.saveProgressStateForPartition(exportPartition, null); - lastCheckpointTime = System.currentTimeMillis(); - } + CheckExportStatusRunner(EnhancedSourceCoordinator sourceCoordinator, ExportTaskManager exportTaskManager, ExportPartition exportPartition) { + this.sourceCoordinator = sourceCoordinator; + this.exportTaskManager = exportTaskManager; + this.exportPartition = exportPartition; + } - // Valid statuses are: CANCELED, CANCELING, COMPLETE, FAILED, IN_PROGRESS, STARTING - String status = exportTaskManager.checkExportStatus(exportTaskId); - LOG.debug("Current export status is {}.", status); - if (ExportStatus.isTerminal(status)) { - LOG.info("Export {} is completed with final status {}", exportTaskId, status); - return status; - } - LOG.debug("Export {} is still running in progress. Wait and check later", exportTaskId); - try { - Thread.sleep(DEFAULT_CHECK_STATUS_INTERVAL_MILLS); - } catch (InterruptedException e) { - throw new RuntimeException(e); + @Override + public String call() { + return checkExportStatus(exportPartition); + } + + private String checkExportStatus(ExportPartition exportPartition) { + long lastCheckpointTime = System.currentTimeMillis(); + String exportTaskId = exportPartition.getProgressState().get().getExportTaskId(); + + LOG.debug("Start checking the status of export {}", exportTaskId); + while (true) { + if (System.currentTimeMillis() - lastCheckpointTime > DEFAULT_CHECKPOINT_INTERVAL_MILLS) { + sourceCoordinator.saveProgressStateForPartition(exportPartition, null); + lastCheckpointTime = System.currentTimeMillis(); + } + + // Valid statuses are: CANCELED, CANCELING, COMPLETE, FAILED, IN_PROGRESS, STARTING + String status = exportTaskManager.checkExportStatus(exportTaskId); + LOG.debug("Current export status is {}.", status); + if (ExportStatus.isTerminal(status)) { + LOG.info("Export {} is completed with final status {}", exportTaskId, status); + return status; + } + LOG.debug("Export {} is still running in progress. Wait and check later", exportTaskId); + try { + Thread.sleep(DEFAULT_CHECK_STATUS_INTERVAL_MILLS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } } } @@ -219,11 +255,61 @@ private BiConsumer completeExport(ExportPartition exportParti } LOG.info("Export for {} completed successfully", exportPartition.getPartitionKey()); + ExportProgressState state = exportPartition.getProgressState().get(); + String bucket = state.getBucket(); + String prefix = state.getPrefix(); + String exportTaskId = state.getExportTaskId(); + + // Create data file partitions for processing S3 files + List dataFileObjectKeys = getDataFileObjectKeys(bucket, prefix, exportTaskId); + createDataFilePartitions(bucket, exportTaskId, dataFileObjectKeys); + completeExportPartition(exportPartition); } }; } + private List getDataFileObjectKeys(String bucket, String prefix, String exportTaskId) { + LOG.debug("Fetching object keys for export data files."); + ListObjectsV2Request.Builder requestBuilder = ListObjectsV2Request.builder() + .bucket(bucket) + .prefix(prefix + "/" + exportTaskId); + + List objectKeys = new ArrayList<>(); + ListObjectsV2Response response = null; + do { + String nextToken = response == null ? null : response.nextContinuationToken(); + response = s3Client.listObjectsV2(requestBuilder + .continuationToken(nextToken) + .build()); + objectKeys.addAll(response.contents().stream() + .map(S3Object::key) + .filter(key -> key.endsWith(PARQUET_SUFFIX)) + .collect(Collectors.toList())); + + } while (response.isTruncated()); + return objectKeys; + } + + private void createDataFilePartitions(String bucket, String exportTaskId, List dataFileObjectKeys) { + LOG.info("Total of {} data files generated for export {}", dataFileObjectKeys.size(), exportTaskId); + AtomicInteger totalFiles = new AtomicInteger(); + for (final String objectKey : dataFileObjectKeys) { + DataFileProgressState progressState = new DataFileProgressState(); + ExportObjectKey exportObjectKey = ExportObjectKey.fromString(objectKey); + String table = exportObjectKey.getTableName(); + progressState.setSourceTable(table); + + DataFilePartition dataFilePartition = new DataFilePartition(exportTaskId, bucket, objectKey, Optional.of(progressState)); + sourceCoordinator.createPartition(dataFilePartition); + totalFiles.getAndIncrement(); + } + + // Create a global state to track overall progress for data file processing + LoadStatus loadStatus = new LoadStatus(totalFiles.get(), 0); + sourceCoordinator.createPartition(new GlobalState(exportTaskId, loadStatus.toMap())); + } + private void completeExportPartition(ExportPartition exportPartition) { ExportProgressState progressState = exportPartition.getProgressState().get(); progressState.setStatus("Completed"); diff --git a/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/export/S3ObjectReader.java b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/export/S3ObjectReader.java new file mode 100644 index 0000000000..39c0079198 --- /dev/null +++ b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/export/S3ObjectReader.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.rds.export; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; + +import java.io.InputStream; + +public class S3ObjectReader { + + private static final Logger LOG = LoggerFactory.getLogger(S3ObjectReader.class); + + private final S3Client s3Client; + + public S3ObjectReader(S3Client s3Client) { + this.s3Client = s3Client; + } + + public InputStream readFile(String bucketName, String s3Key) { + LOG.debug("Read file from s3://{}/{}", bucketName, s3Key); + + GetObjectRequest objectRequest = GetObjectRequest.builder() + .bucket(bucketName) + .key(s3Key) + .build(); + + return s3Client.getObject(objectRequest); + } + +} diff --git a/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/model/ExportObjectKey.java b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/model/ExportObjectKey.java new file mode 100644 index 0000000000..c69dcc7651 --- /dev/null +++ b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/model/ExportObjectKey.java @@ -0,0 +1,68 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.rds.model; + +/** + * Represents the object key for an object exported to S3 by RDS. + * The object key has this structure: "{prefix}/{export task ID}/{database name}/{table name}/{numbered folder}/{file name}" + */ +public class ExportObjectKey { + + private final String prefix; + private final String exportTaskId; + private final String databaseName; + private final String tableName; + private final String numberedFolder; + private final String fileName; + + ExportObjectKey(final String prefix, final String exportTaskId, final String databaseName, final String tableName, final String numberedFolder, final String fileName) { + this.prefix = prefix; + this.exportTaskId = exportTaskId; + this.databaseName = databaseName; + this.tableName = tableName; + this.numberedFolder = numberedFolder; + this.fileName = fileName; + } + + public static ExportObjectKey fromString(final String objectKeyString) { + + final String[] parts = objectKeyString.split("/"); + if (parts.length != 6) { + throw new IllegalArgumentException("Export object key is not valid: " + objectKeyString); + } + final String prefix = parts[0]; + final String exportTaskId = parts[1]; + final String databaseName = parts[2]; + final String tableName = parts[3]; + final String numberedFolder = parts[4]; + final String fileName = parts[5]; + return new ExportObjectKey(prefix, exportTaskId, databaseName, tableName, numberedFolder, fileName); + } + + public String getPrefix() { + return prefix; + } + + public String getExportTaskId() { + return exportTaskId; + } + + public String getDatabaseName() { + return databaseName; + } + + public String getTableName() { + return tableName; + } + + public String getNumberedFolder() { + return numberedFolder; + } + + public String getFileName() { + return fileName; + } +} diff --git a/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/model/LoadStatus.java b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/model/LoadStatus.java new file mode 100644 index 0000000000..a2762c1b38 --- /dev/null +++ b/data-prepper-plugins/rds-source/src/main/java/org/opensearch/dataprepper/plugins/source/rds/model/LoadStatus.java @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.rds.model; + +import java.util.Map; + +public class LoadStatus { + + private static final String TOTAL_FILES = "totalFiles"; + private static final String LOADED_FILES = "loadedFiles"; + + private int totalFiles; + + private int loadedFiles; + + public LoadStatus(int totalFiles, int loadedFiles) { + this.totalFiles = totalFiles; + this.loadedFiles = loadedFiles; + } + + public int getTotalFiles() { + return totalFiles; + } + + public void setTotalFiles(int totalFiles) { + this.totalFiles = totalFiles; + } + + public int getLoadedFiles() { + return loadedFiles; + } + + public void setLoadedFiles(int loadedFiles) { + this.loadedFiles = loadedFiles; + } + + public Map toMap() { + return Map.of( + TOTAL_FILES, totalFiles, + LOADED_FILES, loadedFiles + ); + } + + public static LoadStatus fromMap(Map map) { + return new LoadStatus( + ((Number) map.get(TOTAL_FILES)).intValue(), + ((Number) map.get(LOADED_FILES)).intValue() + ); + } +} diff --git a/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/RdsServiceTest.java b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/RdsServiceTest.java index 6aaa0b0bd5..7a18dd6159 100644 --- a/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/RdsServiceTest.java +++ b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/RdsServiceTest.java @@ -14,8 +14,10 @@ import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator; +import org.opensearch.dataprepper.plugins.source.rds.export.DataFileScheduler; import org.opensearch.dataprepper.plugins.source.rds.export.ExportScheduler; import org.opensearch.dataprepper.plugins.source.rds.leader.LeaderScheduler; import software.amazon.awssdk.services.rds.RdsClient; @@ -47,6 +49,9 @@ class RdsServiceTest { @Mock private ExecutorService executor; + @Mock + private EventFactory eventFactory; + @Mock private ClientFactory clientFactory; @@ -59,8 +64,9 @@ void setUp() { } @Test - void test_normal_service_start() { + void test_normal_service_start_when_export_is_enabled() { RdsService rdsService = createObjectUnderTest(); + when(sourceConfig.isExportEnabled()).thenReturn(true); try (final MockedStatic executorsMockedStatic = mockStatic(Executors.class)) { executorsMockedStatic.when(() -> Executors.newFixedThreadPool(anyInt())).thenReturn(executor); rdsService.start(buffer); @@ -68,6 +74,7 @@ void test_normal_service_start() { verify(executor).submit(any(LeaderScheduler.class)); verify(executor).submit(any(ExportScheduler.class)); + verify(executor).submit(any(DataFileScheduler.class)); } @Test @@ -83,6 +90,6 @@ void test_service_shutdown_calls_executor_shutdownNow() { } private RdsService createObjectUnderTest() { - return new RdsService(sourceCoordinator, sourceConfig, clientFactory, pluginMetrics); + return new RdsService(sourceCoordinator, sourceConfig, eventFactory, clientFactory, pluginMetrics); } } \ No newline at end of file diff --git a/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/RdsSourceTest.java b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/RdsSourceTest.java index edd409e5e4..682f16ed51 100644 --- a/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/RdsSourceTest.java +++ b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/RdsSourceTest.java @@ -12,6 +12,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.event.EventFactory; import org.opensearch.dataprepper.plugins.source.rds.configuration.AwsAuthenticationConfig; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -27,6 +28,9 @@ class RdsSourceTest { @Mock private RdsSourceConfig sourceConfig; + @Mock + private EventFactory eventFactory; + @Mock AwsCredentialsSupplier awsCredentialsSupplier; @@ -45,6 +49,6 @@ void test_when_buffer_is_null_then_start_throws_exception() { } private RdsSource createObjectUnderTest() { - return new RdsSource(pluginMetrics, sourceConfig, awsCredentialsSupplier); + return new RdsSource(pluginMetrics, sourceConfig, eventFactory, awsCredentialsSupplier); } } \ No newline at end of file diff --git a/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/converter/ExportRecordConverterTest.java b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/converter/ExportRecordConverterTest.java new file mode 100644 index 0000000000..79c5597c3b --- /dev/null +++ b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/converter/ExportRecordConverterTest.java @@ -0,0 +1,51 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.rds.converter; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.event.TestEventFactory; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventBuilder; +import org.opensearch.dataprepper.model.record.Record; + +import java.util.Map; +import java.util.UUID; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.sameInstance; +import static org.opensearch.dataprepper.plugins.source.rds.converter.ExportRecordConverter.EXPORT_EVENT_TYPE; +import static org.opensearch.dataprepper.plugins.source.rds.converter.MetadataKeyAttributes.EVENT_TABLE_NAME_METADATA_ATTRIBUTE; +import static org.opensearch.dataprepper.plugins.source.rds.converter.MetadataKeyAttributes.INGESTION_EVENT_TYPE_ATTRIBUTE; +import static org.opensearch.dataprepper.plugins.source.rds.converter.MetadataKeyAttributes.PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE; + +@ExtendWith(MockitoExtension.class) +class ExportRecordConverterTest { + + @Test + void test_convert() { + final String tableName = UUID.randomUUID().toString(); + final String primaryKeyName = UUID.randomUUID().toString(); + final String primaryKeyValue = UUID.randomUUID().toString(); + final Event testEvent = TestEventFactory.getTestEventFactory().eventBuilder(EventBuilder.class) + .withEventType("EVENT") + .withData(Map.of(primaryKeyName, primaryKeyValue)) + .build(); + + Record testRecord = new Record<>(testEvent); + + ExportRecordConverter exportRecordConverter = new ExportRecordConverter(); + Event actualEvent = exportRecordConverter.convert(testRecord, tableName, primaryKeyName); + + // Assert + assertThat(actualEvent.getMetadata().getAttribute(EVENT_TABLE_NAME_METADATA_ATTRIBUTE), equalTo(tableName)); + assertThat(actualEvent.getMetadata().getAttribute(PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE), equalTo(primaryKeyValue)); + assertThat(actualEvent.getMetadata().getAttribute(INGESTION_EVENT_TYPE_ATTRIBUTE), equalTo(EXPORT_EVENT_TYPE)); + assertThat(actualEvent, sameInstance(testRecord.getData())); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/DataFileLoaderTest.java b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/DataFileLoaderTest.java new file mode 100644 index 0000000000..1ed91bc031 --- /dev/null +++ b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/DataFileLoaderTest.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.rds.export; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.buffer.common.BufferAccumulator; +import org.opensearch.dataprepper.model.codec.InputCodec; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.source.rds.converter.ExportRecordConverter; +import org.opensearch.dataprepper.plugins.source.rds.coordination.partition.DataFilePartition; + +import java.io.InputStream; +import java.util.UUID; +import java.util.function.Consumer; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class DataFileLoaderTest { + + @Mock + private DataFilePartition dataFilePartition; + + @Mock + private BufferAccumulator> bufferAccumulator; + + @Mock + private InputCodec codec; + + @Mock + private S3ObjectReader s3ObjectReader; + + @Mock + private ExportRecordConverter recordConverter; + + @Test + void test_run() throws Exception { + final String bucket = UUID.randomUUID().toString(); + final String key = UUID.randomUUID().toString(); + when(dataFilePartition.getBucket()).thenReturn(bucket); + when(dataFilePartition.getKey()).thenReturn(key); + + InputStream inputStream = mock(InputStream.class); + when(s3ObjectReader.readFile(bucket, key)).thenReturn(inputStream); + + DataFileLoader objectUnderTest = createObjectUnderTest(); + objectUnderTest.run(); + + verify(codec).parse(eq(inputStream), any(Consumer.class)); + verify(bufferAccumulator).flush(); + } + + private DataFileLoader createObjectUnderTest() { + return DataFileLoader.create(dataFilePartition, codec, bufferAccumulator, s3ObjectReader, recordConverter); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/DataFileSchedulerTest.java b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/DataFileSchedulerTest.java new file mode 100644 index 0000000000..ee0d0e2852 --- /dev/null +++ b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/DataFileSchedulerTest.java @@ -0,0 +1,137 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.rds.export; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.buffer.common.BufferAccumulator; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.codec.InputCodec; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator; +import org.opensearch.dataprepper.plugins.source.rds.RdsSourceConfig; +import org.opensearch.dataprepper.plugins.source.rds.converter.ExportRecordConverter; +import org.opensearch.dataprepper.plugins.source.rds.coordination.partition.DataFilePartition; +import org.opensearch.dataprepper.plugins.source.rds.coordination.partition.GlobalState; +import org.opensearch.dataprepper.plugins.source.rds.model.LoadStatus; +import software.amazon.awssdk.services.s3.S3Client; + +import java.time.Duration; +import java.util.Map; +import java.util.Optional; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class DataFileSchedulerTest { + + @Mock + private EnhancedSourceCoordinator sourceCoordinator; + + @Mock + private RdsSourceConfig sourceConfig; + + @Mock + private S3Client s3Client; + + @Mock + private EventFactory eventFactory; + + @Mock + private Buffer> buffer; + + @Mock + private DataFilePartition dataFilePartition; + + private Random random; + + @BeforeEach + void setUp() { + random = new Random(); + } + + @Test + void test_given_no_datafile_partition_then_no_export() throws InterruptedException { + when(sourceCoordinator.acquireAvailablePartition(DataFilePartition.PARTITION_TYPE)).thenReturn(Optional.empty()); + + final DataFileScheduler objectUnderTest = createObjectUnderTest(); + final ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.submit(objectUnderTest); + await().atMost(Duration.ofSeconds(1)) + .untilAsserted(() -> verify(sourceCoordinator).acquireAvailablePartition(DataFilePartition.PARTITION_TYPE)); + Thread.sleep(100); + executorService.shutdownNow(); + + verifyNoInteractions(s3Client, buffer); + } + + @Test + void test_given_available_datafile_partition_then_load_datafile() { + DataFileScheduler objectUnderTest = createObjectUnderTest(); + final String exportTaskId = UUID.randomUUID().toString(); + when(dataFilePartition.getExportTaskId()).thenReturn(exportTaskId); + + when(sourceCoordinator.acquireAvailablePartition(DataFilePartition.PARTITION_TYPE)).thenReturn(Optional.of(dataFilePartition)); + final GlobalState globalStatePartition = mock(GlobalState.class); + final int totalFiles = random.nextInt() + 1; + final Map loadStatusMap = new LoadStatus(totalFiles, totalFiles - 1).toMap(); + when(globalStatePartition.getProgressState()).thenReturn(Optional.of(loadStatusMap)); + when(sourceCoordinator.getPartition(exportTaskId)).thenReturn(Optional.of(globalStatePartition)); + + final ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.submit(() -> { + // MockedStatic needs to be created on the same thread it's used + try (MockedStatic dataFileLoaderMockedStatic = mockStatic(DataFileLoader.class)) { + DataFileLoader dataFileLoader = mock(DataFileLoader.class); + dataFileLoaderMockedStatic.when(() -> DataFileLoader.create( + eq(dataFilePartition), any(InputCodec.class), any(BufferAccumulator.class), any(S3ObjectReader.class), any(ExportRecordConverter.class))) + .thenReturn(dataFileLoader); + doNothing().when(dataFileLoader).run(); + objectUnderTest.run(); + } + }); + await().atMost(Duration.ofSeconds(1)) + .untilAsserted(() -> verify(sourceCoordinator).completePartition(dataFilePartition)); + executorService.shutdownNow(); + + verify(sourceCoordinator).completePartition(dataFilePartition); + } + + @Test + void test_shutdown() { + DataFileScheduler objectUnderTest = createObjectUnderTest(); + final ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.submit(objectUnderTest); + + objectUnderTest.shutdown(); + + verifyNoMoreInteractions(sourceCoordinator); + executorService.shutdownNow(); + } + + private DataFileScheduler createObjectUnderTest() { + return new DataFileScheduler(sourceCoordinator, sourceConfig, s3Client, eventFactory, buffer); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/ExportSchedulerTest.java b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/ExportSchedulerTest.java index d0560ab30d..32aff02a57 100644 --- a/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/ExportSchedulerTest.java +++ b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/ExportSchedulerTest.java @@ -15,6 +15,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator; +import org.opensearch.dataprepper.plugins.source.rds.coordination.partition.DataFilePartition; import org.opensearch.dataprepper.plugins.source.rds.coordination.partition.ExportPartition; import org.opensearch.dataprepper.plugins.source.rds.coordination.state.ExportProgressState; import software.amazon.awssdk.services.rds.RdsClient; @@ -27,9 +28,14 @@ import software.amazon.awssdk.services.rds.model.DescribeExportTasksResponse; import software.amazon.awssdk.services.rds.model.StartExportTaskRequest; import software.amazon.awssdk.services.rds.model.StartExportTaskResponse; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.S3Object; import java.time.Duration; import java.time.Instant; +import java.util.List; import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutorService; @@ -44,6 +50,7 @@ import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.rds.export.ExportScheduler.PARQUET_SUFFIX; @ExtendWith(MockitoExtension.class) @@ -55,6 +62,9 @@ class ExportSchedulerTest { @Mock private RdsClient rdsClient; + @Mock + private S3Client s3Client; + @Mock private PluginMetrics pluginMetrics; @@ -96,6 +106,18 @@ void test_given_export_partition_and_task_id_then_complete_export() throws Inter when(describeExportTasksResponse.exportTasks().get(0).status()).thenReturn("COMPLETE"); when(rdsClient.describeExportTasks(any(DescribeExportTasksRequest.class))).thenReturn(describeExportTasksResponse); + // Mock list s3 objects response + ListObjectsV2Response listObjectsV2Response = mock(ListObjectsV2Response.class); + String exportTaskId = UUID.randomUUID().toString(); + String tableName = UUID.randomUUID().toString(); + // objectKey needs to have this structure: "{prefix}/{export task ID}/{database name}/{table name}/{numbered folder}/{file name}" + S3Object s3Object = S3Object.builder() + .key("prefix/" + exportTaskId + "/my_db/" + tableName + "/1/file1" + PARQUET_SUFFIX) + .build(); + when(listObjectsV2Response.contents()).thenReturn(List.of(s3Object)); + when(listObjectsV2Response.isTruncated()).thenReturn(false); + when(s3Client.listObjectsV2(any(ListObjectsV2Request.class))).thenReturn(listObjectsV2Response); + final ExecutorService executorService = Executors.newSingleThreadExecutor(); executorService.submit(exportScheduler); await().atMost(Duration.ofSeconds(1)) @@ -103,6 +125,7 @@ void test_given_export_partition_and_task_id_then_complete_export() throws Inter Thread.sleep(100); executorService.shutdownNow(); + verify(sourceCoordinator).createPartition(any(DataFilePartition.class)); verify(sourceCoordinator).completePartition(exportPartition); verify(rdsClient, never()).startExportTask(any(StartExportTaskRequest.class)); verify(rdsClient, never()).createDBSnapshot(any(CreateDbSnapshotRequest.class)); @@ -110,7 +133,7 @@ void test_given_export_partition_and_task_id_then_complete_export() throws Inter @Test - void test_given_export_partition_and_no_task_id_then_start_and_complete_export() throws InterruptedException { + void test_given_export_partition_without_task_id_then_start_and_complete_export() throws InterruptedException { when(sourceCoordinator.acquireAvailablePartition(ExportPartition.PARTITION_TYPE)).thenReturn(Optional.of(exportPartition)); when(exportPartition.getPartitionKey()).thenReturn(UUID.randomUUID().toString()); when(exportProgressState.getExportTaskId()).thenReturn(null).thenReturn(UUID.randomUUID().toString()); @@ -142,6 +165,18 @@ void test_given_export_partition_and_no_task_id_then_start_and_complete_export() when(describeExportTasksResponse.exportTasks().get(0).status()).thenReturn("COMPLETE"); when(rdsClient.describeExportTasks(any(DescribeExportTasksRequest.class))).thenReturn(describeExportTasksResponse); + // Mock list s3 objects response + ListObjectsV2Response listObjectsV2Response = mock(ListObjectsV2Response.class); + String exportTaskId = UUID.randomUUID().toString(); + String tableName = UUID.randomUUID().toString(); + // objectKey needs to have this structure: "{prefix}/{export task ID}/{database name}/{table name}/{numbered folder}/{file name}" + S3Object s3Object = S3Object.builder() + .key("prefix/" + exportTaskId + "/my_db/" + tableName + "/1/file1" + PARQUET_SUFFIX) + .build(); + when(listObjectsV2Response.contents()).thenReturn(List.of(s3Object)); + when(listObjectsV2Response.isTruncated()).thenReturn(false); + when(s3Client.listObjectsV2(any(ListObjectsV2Request.class))).thenReturn(listObjectsV2Response); + final ExecutorService executorService = Executors.newSingleThreadExecutor(); executorService.submit(exportScheduler); await().atMost(Duration.ofSeconds(1)) @@ -151,6 +186,7 @@ void test_given_export_partition_and_no_task_id_then_start_and_complete_export() verify(rdsClient).createDBSnapshot(any(CreateDbSnapshotRequest.class)); verify(rdsClient).startExportTask(any(StartExportTaskRequest.class)); + verify(sourceCoordinator).createPartition(any(DataFilePartition.class)); verify(sourceCoordinator).completePartition(exportPartition); } @@ -166,6 +202,6 @@ void test_shutDown() { } private ExportScheduler createObjectUnderTest() { - return new ExportScheduler(sourceCoordinator, rdsClient, pluginMetrics); + return new ExportScheduler(sourceCoordinator, rdsClient, s3Client, pluginMetrics); } } \ No newline at end of file diff --git a/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/S3ObjectReaderTest.java b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/S3ObjectReaderTest.java new file mode 100644 index 0000000000..44aa22f6ad --- /dev/null +++ b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/S3ObjectReaderTest.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.rds.export; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; + +import java.util.UUID; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.verify; + +@ExtendWith(MockitoExtension.class) +class S3ObjectReaderTest { + + @Mock + private S3Client s3Client; + + private S3ObjectReader s3ObjectReader; + + + @BeforeEach + void setUp() { + s3ObjectReader = createObjectUnderTest(); + } + + @Test + void test_readFile() { + final String bucketName = UUID.randomUUID().toString(); + final String key = UUID.randomUUID().toString(); + + + s3ObjectReader.readFile(bucketName, key); + + ArgumentCaptor getObjectRequestArgumentCaptor = ArgumentCaptor.forClass(GetObjectRequest.class); + verify(s3Client).getObject(getObjectRequestArgumentCaptor.capture()); + + GetObjectRequest request = getObjectRequestArgumentCaptor.getValue(); + assertThat(request.bucket(), equalTo(bucketName)); + assertThat(request.key(), equalTo(key)); + } + + private S3ObjectReader createObjectUnderTest() { + return new S3ObjectReader(s3Client); + } +} diff --git a/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/model/ExportObjectKeyTest.java b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/model/ExportObjectKeyTest.java new file mode 100644 index 0000000000..7056114572 --- /dev/null +++ b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/model/ExportObjectKeyTest.java @@ -0,0 +1,37 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.rds.model; + +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class ExportObjectKeyTest { + + @Test + void test_fromString_with_valid_input_string() { + final String objectKeyString = "prefix/export-task-id/db-name/table-name/1/file-name.parquet"; + final ExportObjectKey exportObjectKey = ExportObjectKey.fromString(objectKeyString); + + assertThat(exportObjectKey.getPrefix(), equalTo("prefix")); + assertThat(exportObjectKey.getExportTaskId(), equalTo("export-task-id")); + assertThat(exportObjectKey.getDatabaseName(), equalTo("db-name")); + assertThat(exportObjectKey.getTableName(), equalTo("table-name")); + assertThat(exportObjectKey.getNumberedFolder(), equalTo("1")); + assertThat(exportObjectKey.getFileName(), equalTo("file-name.parquet")); + } + + @Test + void test_fromString_with_invalid_input_string() { + final String objectKeyString = "prefix/export-task-id/db-name/table-name/1/"; + + Throwable exception = assertThrows(IllegalArgumentException.class, () -> ExportObjectKey.fromString(objectKeyString)); + assertThat(exception.getMessage(), containsString("Export object key is not valid: " + objectKeyString)); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/s3-sink/build.gradle b/data-prepper-plugins/s3-sink/build.gradle index d8ca855b13..4ea0a364fd 100644 --- a/data-prepper-plugins/s3-sink/build.gradle +++ b/data-prepper-plugins/s3-sink/build.gradle @@ -22,6 +22,7 @@ dependencies { implementation(libs.hadoop.common) { exclude group: 'org.eclipse.jetty' exclude group: 'org.apache.hadoop', module: 'hadoop-auth' + exclude group: 'org.apache.zookeeper', module: 'zookeeper' } implementation libs.parquet.avro implementation 'software.amazon.awssdk:apache-client' diff --git a/data-prepper-plugins/s3-sink/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-plugins/s3-sink/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 23c33feb6d..0000000000 --- a/data-prepper-plugins/s3-sink/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1,3 +0,0 @@ -# To enable mocking of final classes with vanilla Mockito -# https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods -mock-maker-inline diff --git a/data-prepper-plugins/s3-source/build.gradle b/data-prepper-plugins/s3-source/build.gradle index b0209a5d08..06818d8eaa 100644 --- a/data-prepper-plugins/s3-source/build.gradle +++ b/data-prepper-plugins/s3-source/build.gradle @@ -45,7 +45,11 @@ dependencies { testImplementation project(':data-prepper-plugins:parquet-codecs') testImplementation project(':data-prepper-test-event') testImplementation libs.avro.core - testImplementation libs.hadoop.common + testImplementation(libs.hadoop.common) { + exclude group: 'org.eclipse.jetty' + exclude group: 'org.apache.hadoop', module: 'hadoop-auth' + exclude group: 'org.apache.zookeeper', module: 'zookeeper' + } testImplementation libs.parquet.avro testImplementation libs.parquet.column testImplementation libs.parquet.hadoop diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/SqsService.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/SqsService.java index b05d2806d4..c674be5f68 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/SqsService.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/SqsService.java @@ -17,9 +17,12 @@ import software.amazon.awssdk.services.sqs.SqsClient; import java.time.Duration; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.Executors; import java.util.concurrent.ExecutorService; +import java.util.stream.Collectors; +import java.util.stream.IntStream; public class SqsService { private static final Logger LOG = LoggerFactory.getLogger(SqsService.class); @@ -34,6 +37,7 @@ public class SqsService { private final PluginMetrics pluginMetrics; private final AcknowledgementSetManager acknowledgementSetManager; private final ExecutorService executorService; + private final List sqsWorkers; public SqsService(final AcknowledgementSetManager acknowledgementSetManager, final S3SourceConfig s3SourceConfig, @@ -46,18 +50,20 @@ public SqsService(final AcknowledgementSetManager acknowledgementSetManager, this.acknowledgementSetManager = acknowledgementSetManager; this.sqsClient = createSqsClient(credentialsProvider); executorService = Executors.newFixedThreadPool(s3SourceConfig.getNumWorkers(), BackgroundThreadFactory.defaultExecutorThreadFactory("s3-source-sqs")); - } - public void start() { final Backoff backoff = Backoff.exponential(INITIAL_DELAY, MAXIMUM_DELAY).withJitter(JITTER_RATE) .withMaxAttempts(Integer.MAX_VALUE); - for (int i = 0; i < s3SourceConfig.getNumWorkers(); i++) { - executorService.submit(new SqsWorker(acknowledgementSetManager, sqsClient, s3Accessor, s3SourceConfig, pluginMetrics, backoff)); - } + sqsWorkers = IntStream.range(0, s3SourceConfig.getNumWorkers()) + .mapToObj(i -> new SqsWorker(acknowledgementSetManager, sqsClient, s3Accessor, s3SourceConfig, pluginMetrics, backoff)) + .collect(Collectors.toList()); + } + + public void start() { + sqsWorkers.forEach(executorService::submit); } SqsClient createSqsClient(final AwsCredentialsProvider credentialsProvider) { - LOG.info("Creating SQS client"); + LOG.debug("Creating SQS client"); return SqsClient.builder() .region(s3SourceConfig.getAwsAuthenticationOptions().getAwsRegion()) .credentialsProvider(credentialsProvider) @@ -68,8 +74,8 @@ SqsClient createSqsClient(final AwsCredentialsProvider credentialsProvider) { } public void stop() { - sqsClient.close(); executorService.shutdown(); + sqsWorkers.forEach(SqsWorker::stop); try { if (!executorService.awaitTermination(SHUTDOWN_TIMEOUT, TimeUnit.SECONDS)) { LOG.warn("Failed to terminate SqsWorkers"); @@ -82,5 +88,7 @@ public void stop() { Thread.currentThread().interrupt(); } } + + sqsClient.close(); } } diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorker.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorker.java index b3404cebf6..3c5fba0701 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorker.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorker.java @@ -5,7 +5,6 @@ package org.opensearch.dataprepper.plugins.source.s3; -import com.fasterxml.jackson.databind.ObjectMapper; import com.linecorp.armeria.client.retry.Backoff; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Timer; @@ -20,8 +19,7 @@ import org.opensearch.dataprepper.plugins.source.s3.filter.S3EventFilter; import org.opensearch.dataprepper.plugins.source.s3.filter.S3ObjectCreatedFilter; import org.opensearch.dataprepper.plugins.source.s3.parser.ParsedMessage; -import org.opensearch.dataprepper.plugins.source.s3.parser.S3EventBridgeNotificationParser; -import org.opensearch.dataprepper.plugins.source.s3.parser.S3EventNotificationParser; +import org.opensearch.dataprepper.plugins.source.s3.parser.SqsMessageParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.core.exception.SdkException; @@ -75,11 +73,10 @@ public class SqsWorker implements Runnable { private final Counter sqsVisibilityTimeoutChangeFailedCount; private final Timer sqsMessageDelayTimer; private final Backoff standardBackoff; + private final SqsMessageParser sqsMessageParser; private int failedAttemptCount; private final boolean endToEndAcknowledgementsEnabled; private final AcknowledgementSetManager acknowledgementSetManager; - - private final ObjectMapper objectMapper = new ObjectMapper(); private volatile boolean isStopped = false; private Map parsedMessageVisibilityTimesMap; @@ -98,6 +95,7 @@ public SqsWorker(final AcknowledgementSetManager acknowledgementSetManager, sqsOptions = s3SourceConfig.getSqsOptions(); objectCreatedFilter = new S3ObjectCreatedFilter(); evenBridgeObjectCreatedFilter = new EventBridgeObjectCreatedFilter(); + sqsMessageParser = new SqsMessageParser(s3SourceConfig); failedAttemptCount = 0; parsedMessageVisibilityTimesMap = new HashMap<>(); @@ -139,7 +137,7 @@ int processSqsMessages() { if (!sqsMessages.isEmpty()) { sqsMessagesReceivedCounter.increment(sqsMessages.size()); - final Collection s3MessageEventNotificationRecords = getS3MessageEventNotificationRecords(sqsMessages); + final Collection s3MessageEventNotificationRecords = sqsMessageParser.parseSqsMessages(sqsMessages); // build s3ObjectReference from S3EventNotificationRecord if event name starts with ObjectCreated final List deleteMessageBatchRequestEntries = processS3EventNotificationRecords(s3MessageEventNotificationRecords); @@ -191,22 +189,6 @@ private ReceiveMessageRequest createReceiveMessageRequest() { .build(); } - private Collection getS3MessageEventNotificationRecords(final List sqsMessages) { - return sqsMessages.stream() - .map(this::convertS3EventMessages) - .collect(Collectors.toList()); - } - - private ParsedMessage convertS3EventMessages(final Message message) { - if (s3SourceConfig.getNotificationSource().equals(NotificationSourceOption.S3)) { - return new S3EventNotificationParser().parseMessage(message, objectMapper); - } - else if (s3SourceConfig.getNotificationSource().equals(NotificationSourceOption.EVENTBRIDGE)) { - return new S3EventBridgeNotificationParser().parseMessage(message, objectMapper); - } - return new ParsedMessage(message, true); - } - private List processS3EventNotificationRecords(final Collection s3EventNotificationRecords) { final List deleteMessageBatchRequestEntryCollection = new ArrayList<>(); final List parsedMessagesToRead = new ArrayList<>(); @@ -276,21 +258,7 @@ && isEventBridgeEventTypeCreated(parsedMessage)) { return; } parsedMessageVisibilityTimesMap.put(parsedMessage, newValue); - final ChangeMessageVisibilityRequest changeMessageVisibilityRequest = ChangeMessageVisibilityRequest.builder() - .visibilityTimeout(newVisibilityTimeoutSeconds) - .queueUrl(sqsOptions.getSqsUrl()) - .receiptHandle(parsedMessage.getMessage().receiptHandle()) - .build(); - - try { - sqsClient.changeMessageVisibility(changeMessageVisibilityRequest); - sqsVisibilityTimeoutChangedCount.increment(); - LOG.debug("Set visibility timeout for message {} to {}", parsedMessage.getMessage().messageId(), newVisibilityTimeoutSeconds); - } catch (Exception e) { - LOG.error("Failed to set visibility timeout for message {} to {}", parsedMessage.getMessage().messageId(), newVisibilityTimeoutSeconds, e); - sqsVisibilityTimeoutChangeFailedCount.increment(); - } - + increaseVisibilityTimeout(parsedMessage, newVisibilityTimeoutSeconds); }, Duration.ofSeconds(progressCheckInterval)); } @@ -308,6 +276,27 @@ && isEventBridgeEventTypeCreated(parsedMessage)) { return deleteMessageBatchRequestEntryCollection; } + private void increaseVisibilityTimeout(final ParsedMessage parsedMessage, final int newVisibilityTimeoutSeconds) { + if(isStopped) { + LOG.info("Some messages are pending completion of acknowledgments. Data Prepper will not increase the visibility timeout because it is shutting down. {}", parsedMessage); + return; + } + final ChangeMessageVisibilityRequest changeMessageVisibilityRequest = ChangeMessageVisibilityRequest.builder() + .visibilityTimeout(newVisibilityTimeoutSeconds) + .queueUrl(sqsOptions.getSqsUrl()) + .receiptHandle(parsedMessage.getMessage().receiptHandle()) + .build(); + + try { + sqsClient.changeMessageVisibility(changeMessageVisibilityRequest); + sqsVisibilityTimeoutChangedCount.increment(); + LOG.debug("Set visibility timeout for message {} to {}", parsedMessage.getMessage().messageId(), newVisibilityTimeoutSeconds); + } catch (Exception e) { + LOG.error("Failed to set visibility timeout for message {} to {}", parsedMessage.getMessage().messageId(), newVisibilityTimeoutSeconds, e); + sqsVisibilityTimeoutChangeFailedCount.increment(); + } + } + private Optional processS3Object( final ParsedMessage parsedMessage, final S3ObjectReference s3ObjectReference, @@ -328,6 +317,8 @@ private Optional processS3Object( } private void deleteSqsMessages(final List deleteMessageBatchRequestEntryCollection) { + if(isStopped) + return; if (deleteMessageBatchRequestEntryCollection.size() == 0) { return; } @@ -396,6 +387,5 @@ private S3ObjectReference populateS3Reference(final String bucketName, final Str void stop() { isStopped = true; - Thread.currentThread().interrupt(); } } diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/parser/ParsedMessage.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/parser/ParsedMessage.java index 18bbc58499..ed68dff063 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/parser/ParsedMessage.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/parser/ParsedMessage.java @@ -11,6 +11,7 @@ import software.amazon.awssdk.services.sqs.model.Message; import java.util.List; +import java.util.Objects; public class ParsedMessage { private final Message message; @@ -24,14 +25,14 @@ public class ParsedMessage { private String detailType; public ParsedMessage(final Message message, final boolean failedParsing) { - this.message = message; + this.message = Objects.requireNonNull(message); this.failedParsing = failedParsing; this.emptyNotification = true; } - // S3EventNotification contains only one S3EventNotificationRecord ParsedMessage(final Message message, final List notificationRecords) { - this.message = message; + this.message = Objects.requireNonNull(message); + // S3EventNotification contains only one S3EventNotificationRecord this.bucketName = notificationRecords.get(0).getS3().getBucket().getName(); this.objectKey = notificationRecords.get(0).getS3().getObject().getUrlDecodedKey(); this.objectSize = notificationRecords.get(0).getS3().getObject().getSizeAsLong(); @@ -42,7 +43,7 @@ public ParsedMessage(final Message message, final boolean failedParsing) { } ParsedMessage(final Message message, final S3EventBridgeNotification eventBridgeNotification) { - this.message = message; + this.message = Objects.requireNonNull(message); this.bucketName = eventBridgeNotification.getDetail().getBucket().getName(); this.objectKey = eventBridgeNotification.getDetail().getObject().getUrlDecodedKey(); this.objectSize = eventBridgeNotification.getDetail().getObject().getSize(); @@ -85,4 +86,12 @@ public boolean isEmptyNotification() { public String getDetailType() { return detailType; } + + @Override + public String toString() { + return "Message{" + + "messageId=" + message.messageId() + + ", objectKey=" + objectKey + + '}'; + } } diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/parser/SqsMessageParser.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/parser/SqsMessageParser.java new file mode 100644 index 0000000000..ea40e3f041 --- /dev/null +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/parser/SqsMessageParser.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.s3.parser; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.opensearch.dataprepper.plugins.source.s3.S3SourceConfig; +import software.amazon.awssdk.services.sqs.model.Message; + +import java.util.Collection; +import java.util.stream.Collectors; + +public class SqsMessageParser { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private final S3SourceConfig s3SourceConfig; + private final S3NotificationParser s3NotificationParser; + + public SqsMessageParser(final S3SourceConfig s3SourceConfig) { + this.s3SourceConfig = s3SourceConfig; + s3NotificationParser = createNotificationParser(s3SourceConfig); + } + + public Collection parseSqsMessages(final Collection sqsMessages) { + return sqsMessages.stream() + .map(this::convertS3EventMessages) + .collect(Collectors.toList()); + } + + private ParsedMessage convertS3EventMessages(final Message message) { + return s3NotificationParser.parseMessage(message, OBJECT_MAPPER); + } + + private static S3NotificationParser createNotificationParser(final S3SourceConfig s3SourceConfig) { + switch (s3SourceConfig.getNotificationSource()) { + case EVENTBRIDGE: + return new S3EventBridgeNotificationParser(); + case S3: + default: + return new S3EventNotificationParser(); + } + } +} diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorkerTest.java b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorkerTest.java index 50ed879f4a..ada789cea6 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorkerTest.java +++ b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorkerTest.java @@ -12,6 +12,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -19,19 +20,21 @@ import org.junit.jupiter.params.provider.ArgumentsSource; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; -import org.opensearch.dataprepper.plugins.source.s3.configuration.AwsAuthenticationOptions; +import org.opensearch.dataprepper.model.acknowledgements.ProgressCheck; import org.opensearch.dataprepper.plugins.source.s3.configuration.NotificationSourceOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.OnErrorOption; import org.opensearch.dataprepper.plugins.source.s3.configuration.SqsOptions; import org.opensearch.dataprepper.plugins.source.s3.exception.SqsRetriesExhaustedException; import org.opensearch.dataprepper.plugins.source.s3.filter.S3EventFilter; import org.opensearch.dataprepper.plugins.source.s3.filter.S3ObjectCreatedFilter; -import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.sqs.SqsClient; import software.amazon.awssdk.services.sqs.model.BatchResultErrorEntry; +import software.amazon.awssdk.services.sqs.model.ChangeMessageVisibilityRequest; import software.amazon.awssdk.services.sqs.model.DeleteMessageBatchRequest; import software.amazon.awssdk.services.sqs.model.DeleteMessageBatchResponse; import software.amazon.awssdk.services.sqs.model.DeleteMessageBatchResultEntry; @@ -50,6 +53,7 @@ import java.util.Collections; import java.util.List; import java.util.UUID; +import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -65,20 +69,23 @@ import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.s3.SqsWorker.ACKNOWLEDGEMENT_SET_CALLACK_METRIC_NAME; +import static org.opensearch.dataprepper.plugins.source.s3.SqsWorker.S3_OBJECTS_EMPTY_METRIC_NAME; import static org.opensearch.dataprepper.plugins.source.s3.SqsWorker.SQS_MESSAGES_DELETED_METRIC_NAME; import static org.opensearch.dataprepper.plugins.source.s3.SqsWorker.SQS_MESSAGES_DELETE_FAILED_METRIC_NAME; import static org.opensearch.dataprepper.plugins.source.s3.SqsWorker.SQS_MESSAGES_FAILED_METRIC_NAME; import static org.opensearch.dataprepper.plugins.source.s3.SqsWorker.SQS_MESSAGES_RECEIVED_METRIC_NAME; import static org.opensearch.dataprepper.plugins.source.s3.SqsWorker.SQS_MESSAGE_DELAY_METRIC_NAME; -import static org.opensearch.dataprepper.plugins.source.s3.SqsWorker.S3_OBJECTS_EMPTY_METRIC_NAME; +import static org.opensearch.dataprepper.plugins.source.s3.SqsWorker.SQS_VISIBILITY_TIMEOUT_CHANGED_COUNT_METRIC_NAME; +@ExtendWith(MockitoExtension.class) class SqsWorkerTest { - private SqsWorker sqsWorker; private SqsClient sqsClient; private S3Service s3Service; private S3SourceConfig s3SourceConfig; @@ -90,10 +97,13 @@ class SqsWorkerTest { private Counter sqsMessagesFailedCounter; private Counter sqsMessagesDeleteFailedCounter; private Counter s3ObjectsEmptyCounter; + @Mock + private Counter sqsVisibilityTimeoutChangedCount; private Timer sqsMessageDelayTimer; private AcknowledgementSetManager acknowledgementSetManager; private AcknowledgementSet acknowledgementSet; private SqsOptions sqsOptions; + private String queueUrl; @BeforeEach void setUp() { @@ -105,15 +115,11 @@ void setUp() { objectCreatedFilter = new S3ObjectCreatedFilter(); backoff = mock(Backoff.class); - AwsAuthenticationOptions awsAuthenticationOptions = mock(AwsAuthenticationOptions.class); - when(awsAuthenticationOptions.getAwsRegion()).thenReturn(Region.US_EAST_1); - sqsOptions = mock(SqsOptions.class); - when(sqsOptions.getSqsUrl()).thenReturn("https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue"); + queueUrl = "https://sqs.us-east-2.amazonaws.com/123456789012/" + UUID.randomUUID(); + when(sqsOptions.getSqsUrl()).thenReturn(queueUrl); - when(s3SourceConfig.getAwsAuthenticationOptions()).thenReturn(awsAuthenticationOptions); when(s3SourceConfig.getSqsOptions()).thenReturn(sqsOptions); - when(s3SourceConfig.getOnErrorOption()).thenReturn(OnErrorOption.RETAIN_MESSAGES); when(s3SourceConfig.getAcknowledgements()).thenReturn(false); when(s3SourceConfig.getNotificationSource()).thenReturn(NotificationSourceOption.S3); @@ -130,8 +136,12 @@ void setUp() { when(pluginMetrics.counter(SQS_MESSAGES_DELETE_FAILED_METRIC_NAME)).thenReturn(sqsMessagesDeleteFailedCounter); when(pluginMetrics.counter(S3_OBJECTS_EMPTY_METRIC_NAME)).thenReturn(s3ObjectsEmptyCounter); when(pluginMetrics.timer(SQS_MESSAGE_DELAY_METRIC_NAME)).thenReturn(sqsMessageDelayTimer); + when(pluginMetrics.counter(ACKNOWLEDGEMENT_SET_CALLACK_METRIC_NAME)).thenReturn(mock(Counter.class)); + when(pluginMetrics.counter(SQS_VISIBILITY_TIMEOUT_CHANGED_COUNT_METRIC_NAME)).thenReturn(sqsVisibilityTimeoutChangedCount); + } - sqsWorker = new SqsWorker(acknowledgementSetManager, sqsClient, s3Service, s3SourceConfig, pluginMetrics, backoff); + private SqsWorker createObjectUnderTest() { + return new SqsWorker(acknowledgementSetManager, sqsClient, s3Service, s3SourceConfig, pluginMetrics, backoff); } @AfterEach @@ -167,7 +177,7 @@ void processSqsMessages_should_return_number_of_messages_processed(final String when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(receiveMessageResponse); when(receiveMessageResponse.messages()).thenReturn(Collections.singletonList(message)); - final int messagesProcessed = sqsWorker.processSqsMessages(); + final int messagesProcessed = createObjectUnderTest().processSqsMessages(); final ArgumentCaptor deleteMessageBatchRequestArgumentCaptor = ArgumentCaptor.forClass(DeleteMessageBatchRequest.class); verify(sqsClient).deleteMessageBatch(deleteMessageBatchRequestArgumentCaptor.capture()); final DeleteMessageBatchRequest actualDeleteMessageBatchRequest = deleteMessageBatchRequestArgumentCaptor.getValue(); @@ -190,93 +200,6 @@ void processSqsMessages_should_return_number_of_messages_processed(final String assertThat(actualDelay, greaterThanOrEqualTo(Duration.ofHours(1).minus(Duration.ofSeconds(5)))); } - @ParameterizedTest - @ValueSource(strings = {"ObjectCreated:Put", "ObjectCreated:Post", "ObjectCreated:Copy", "ObjectCreated:CompleteMultipartUpload"}) - void processSqsMessages_should_return_number_of_messages_processed_with_acknowledgements(final String eventName) throws IOException { - when(acknowledgementSetManager.create(any(), any(Duration.class))).thenReturn(acknowledgementSet); - when(s3SourceConfig.getAcknowledgements()).thenReturn(true); - sqsWorker = new SqsWorker(acknowledgementSetManager, sqsClient, s3Service, s3SourceConfig, pluginMetrics, backoff); - Instant startTime = Instant.now().minus(1, ChronoUnit.HOURS); - final Message message = mock(Message.class); - when(message.body()).thenReturn(createEventNotification(eventName, startTime)); - final String testReceiptHandle = UUID.randomUUID().toString(); - when(message.messageId()).thenReturn(testReceiptHandle); - when(message.receiptHandle()).thenReturn(testReceiptHandle); - - final ReceiveMessageResponse receiveMessageResponse = mock(ReceiveMessageResponse.class); - when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(receiveMessageResponse); - when(receiveMessageResponse.messages()).thenReturn(Collections.singletonList(message)); - - final int messagesProcessed = sqsWorker.processSqsMessages(); - final ArgumentCaptor deleteMessageBatchRequestArgumentCaptor = ArgumentCaptor.forClass(DeleteMessageBatchRequest.class); - - final ArgumentCaptor durationArgumentCaptor = ArgumentCaptor.forClass(Duration.class); - verify(sqsMessageDelayTimer).record(durationArgumentCaptor.capture()); - Duration actualDelay = durationArgumentCaptor.getValue(); - - assertThat(messagesProcessed, equalTo(1)); - verify(s3Service).addS3Object(any(S3ObjectReference.class), any()); - verify(acknowledgementSetManager).create(any(), any(Duration.class)); - verify(sqsMessagesReceivedCounter).increment(1); - verifyNoInteractions(sqsMessagesDeletedCounter); - assertThat(actualDelay, lessThanOrEqualTo(Duration.ofHours(1).plus(Duration.ofSeconds(5)))); - assertThat(actualDelay, greaterThanOrEqualTo(Duration.ofHours(1).minus(Duration.ofSeconds(5)))); - } - - @ParameterizedTest - @ValueSource(strings = {"ObjectCreated:Put", "ObjectCreated:Post", "ObjectCreated:Copy", "ObjectCreated:CompleteMultipartUpload"}) - void processSqsMessages_should_return_number_of_messages_processed_with_acknowledgements_and_progress_check(final String eventName) throws IOException { - when(sqsOptions.getVisibilityDuplicateProtection()).thenReturn(true); - when(sqsOptions.getVisibilityTimeout()).thenReturn(Duration.ofSeconds(6)); - when(acknowledgementSetManager.create(any(), any(Duration.class))).thenReturn(acknowledgementSet); - when(s3SourceConfig.getAcknowledgements()).thenReturn(true); - sqsWorker = new SqsWorker(acknowledgementSetManager, sqsClient, s3Service, s3SourceConfig, pluginMetrics, backoff); - Instant startTime = Instant.now().minus(1, ChronoUnit.HOURS); - final Message message = mock(Message.class); - when(message.body()).thenReturn(createEventNotification(eventName, startTime)); - final String testReceiptHandle = UUID.randomUUID().toString(); - when(message.messageId()).thenReturn(testReceiptHandle); - when(message.receiptHandle()).thenReturn(testReceiptHandle); - - final ReceiveMessageResponse receiveMessageResponse = mock(ReceiveMessageResponse.class); - when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(receiveMessageResponse); - when(receiveMessageResponse.messages()).thenReturn(Collections.singletonList(message)); - - final int messagesProcessed = sqsWorker.processSqsMessages(); - final ArgumentCaptor deleteMessageBatchRequestArgumentCaptor = ArgumentCaptor.forClass(DeleteMessageBatchRequest.class); - - final ArgumentCaptor durationArgumentCaptor = ArgumentCaptor.forClass(Duration.class); - verify(sqsMessageDelayTimer).record(durationArgumentCaptor.capture()); - Duration actualDelay = durationArgumentCaptor.getValue(); - - assertThat(messagesProcessed, equalTo(1)); - verify(s3Service).addS3Object(any(S3ObjectReference.class), any()); - verify(acknowledgementSetManager).create(any(), any(Duration.class)); - verify(acknowledgementSet).addProgressCheck(any(), any(Duration.class)); - verify(sqsMessagesReceivedCounter).increment(1); - verifyNoInteractions(sqsMessagesDeletedCounter); - assertThat(actualDelay, lessThanOrEqualTo(Duration.ofHours(1).plus(Duration.ofSeconds(5)))); - assertThat(actualDelay, greaterThanOrEqualTo(Duration.ofHours(1).minus(Duration.ofSeconds(5)))); - } - - @ParameterizedTest - @ValueSource(strings = {"", "{\"foo\": \"bar\""}) - void processSqsMessages_should_not_interact_with_S3Service_if_input_is_not_valid_JSON(String inputString) { - final Message message = mock(Message.class); - when(message.body()).thenReturn(inputString); - - final ReceiveMessageResponse receiveMessageResponse = mock(ReceiveMessageResponse.class); - when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(receiveMessageResponse); - when(receiveMessageResponse.messages()).thenReturn(Collections.singletonList(message)); - - final int messagesProcessed = sqsWorker.processSqsMessages(); - assertThat(messagesProcessed, equalTo(1)); - verifyNoInteractions(s3Service); - verify(sqsClient, never()).deleteMessageBatch(any(DeleteMessageBatchRequest.class)); - verify(sqsMessagesReceivedCounter).increment(1); - verify(sqsMessagesFailedCounter).increment(); - } - @Test void processSqsMessages_should_not_interact_with_S3Service_and_delete_message_if_TestEvent() { final String messageId = UUID.randomUUID().toString(); @@ -291,7 +214,7 @@ void processSqsMessages_should_not_interact_with_S3Service_and_delete_message_if when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(receiveMessageResponse); when(receiveMessageResponse.messages()).thenReturn(Collections.singletonList(message)); - final int messagesProcessed = sqsWorker.processSqsMessages(); + final int messagesProcessed = createObjectUnderTest().processSqsMessages(); assertThat(messagesProcessed, equalTo(1)); verifyNoInteractions(s3Service); @@ -324,7 +247,7 @@ void processSqsMessages_should_not_interact_with_S3Service_and_delete_message_if when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(receiveMessageResponse); when(receiveMessageResponse.messages()).thenReturn(Collections.singletonList(message)); - final int messagesProcessed = sqsWorker.processSqsMessages(); + final int messagesProcessed = createObjectUnderTest().processSqsMessages(); assertThat(messagesProcessed, equalTo(1)); verifyNoInteractions(s3Service); @@ -354,7 +277,7 @@ void processSqsMessages_with_irrelevant_eventName_should_return_number_of_messag when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(receiveMessageResponse); when(receiveMessageResponse.messages()).thenReturn(Collections.singletonList(message)); - final int messagesProcessed = sqsWorker.processSqsMessages(); + final int messagesProcessed = createObjectUnderTest().processSqsMessages(); assertThat(messagesProcessed, equalTo(1)); verifyNoInteractions(s3Service); @@ -378,7 +301,7 @@ void processSqsMessages_should_invoke_delete_if_input_is_not_valid_JSON_and_dele when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(receiveMessageResponse); when(receiveMessageResponse.messages()).thenReturn(Collections.singletonList(message)); - final int messagesProcessed = sqsWorker.processSqsMessages(); + final int messagesProcessed = createObjectUnderTest().processSqsMessages(); final ArgumentCaptor deleteMessageBatchRequestArgumentCaptor = ArgumentCaptor.forClass(DeleteMessageBatchRequest.class); verify(sqsClient).deleteMessageBatch(deleteMessageBatchRequestArgumentCaptor.capture()); final DeleteMessageBatchRequest actualDeleteMessageBatchRequest = deleteMessageBatchRequestArgumentCaptor.getValue(); @@ -410,7 +333,7 @@ void processSqsMessages_should_return_number_of_messages_processed_when_using_Ev when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(receiveMessageResponse); when(receiveMessageResponse.messages()).thenReturn(Collections.singletonList(message)); - final int messagesProcessed = sqsWorker.processSqsMessages(); + final int messagesProcessed = createObjectUnderTest().processSqsMessages(); final ArgumentCaptor deleteMessageBatchRequestArgumentCaptor = ArgumentCaptor.forClass(DeleteMessageBatchRequest.class); verify(sqsClient).deleteMessageBatch(deleteMessageBatchRequestArgumentCaptor.capture()); final DeleteMessageBatchRequest actualDeleteMessageBatchRequest = deleteMessageBatchRequestArgumentCaptor.getValue(); @@ -447,7 +370,7 @@ void processSqsMessages_should_return_number_of_messages_processed_when_using_Se when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(receiveMessageResponse); when(receiveMessageResponse.messages()).thenReturn(Collections.singletonList(message)); - final int messagesProcessed = sqsWorker.processSqsMessages(); + final int messagesProcessed = createObjectUnderTest().processSqsMessages(); final ArgumentCaptor deleteMessageBatchRequestArgumentCaptor = ArgumentCaptor.forClass(DeleteMessageBatchRequest.class); verify(sqsClient).deleteMessageBatch(deleteMessageBatchRequestArgumentCaptor.capture()); final DeleteMessageBatchRequest actualDeleteMessageBatchRequest = deleteMessageBatchRequestArgumentCaptor.getValue(); @@ -502,7 +425,7 @@ void processSqsMessages_should_report_correct_metrics_for_DeleteMessages_when_so when(deleteMessageBatchResponse.failed()).thenReturn(failedDeletes); when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))).thenReturn(deleteMessageBatchResponse); - final int messagesProcessed = sqsWorker.processSqsMessages(); + final int messagesProcessed = createObjectUnderTest().processSqsMessages(); final ArgumentCaptor deleteMessageBatchRequestArgumentCaptor = ArgumentCaptor.forClass(DeleteMessageBatchRequest.class); verify(sqsClient).deleteMessageBatch(deleteMessageBatchRequestArgumentCaptor.capture()); @@ -542,7 +465,7 @@ void processSqsMessages_should_report_correct_metrics_for_DeleteMessages_when_re when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))).thenThrow(exClass); - final int messagesProcessed = sqsWorker.processSqsMessages(); + final int messagesProcessed = createObjectUnderTest().processSqsMessages(); final ArgumentCaptor deleteMessageBatchRequestArgumentCaptor = ArgumentCaptor.forClass(DeleteMessageBatchRequest.class); verify(sqsClient).deleteMessageBatch(deleteMessageBatchRequestArgumentCaptor.capture()); @@ -565,7 +488,7 @@ void processSqsMessages_should_report_correct_metrics_for_DeleteMessages_when_re @Test void processSqsMessages_should_return_zero_messages_when_a_SqsException_is_thrown() { when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenThrow(SqsException.class); - final int messagesProcessed = sqsWorker.processSqsMessages(); + final int messagesProcessed = createObjectUnderTest().processSqsMessages(); assertThat(messagesProcessed, equalTo(0)); verify(sqsClient, never()).deleteMessageBatch(any(DeleteMessageBatchRequest.class)); } @@ -573,7 +496,7 @@ void processSqsMessages_should_return_zero_messages_when_a_SqsException_is_throw @Test void processSqsMessages_should_return_zero_messages_with_backoff_when_a_SqsException_is_thrown() { when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenThrow(SqsException.class); - final int messagesProcessed = sqsWorker.processSqsMessages(); + final int messagesProcessed = createObjectUnderTest().processSqsMessages(); verify(backoff).nextDelayMillis(1); assertThat(messagesProcessed, equalTo(0)); } @@ -582,7 +505,8 @@ void processSqsMessages_should_return_zero_messages_with_backoff_when_a_SqsExcep void processSqsMessages_should_throw_when_a_SqsException_is_thrown_with_max_retries() { when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenThrow(SqsException.class); when(backoff.nextDelayMillis(anyInt())).thenReturn((long) -1); - assertThrows(SqsRetriesExhaustedException.class, () -> sqsWorker.processSqsMessages()); + SqsWorker objectUnderTest = createObjectUnderTest(); + assertThrows(SqsRetriesExhaustedException.class, () -> objectUnderTest.processSqsMessages()); } @ParameterizedTest @@ -591,11 +515,13 @@ void processSqsMessages_should_return_zero_messages_when_messages_are_not_S3Even final Message message = mock(Message.class); when(message.body()).thenReturn(inputString); + when(s3SourceConfig.getOnErrorOption()).thenReturn(OnErrorOption.RETAIN_MESSAGES); + final ReceiveMessageResponse receiveMessageResponse = mock(ReceiveMessageResponse.class); when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(receiveMessageResponse); when(receiveMessageResponse.messages()).thenReturn(Collections.singletonList(message)); - final int messagesProcessed = sqsWorker.processSqsMessages(); + final int messagesProcessed = createObjectUnderTest().processSqsMessages(); assertThat(messagesProcessed, equalTo(1)); verifyNoInteractions(s3Service); verify(sqsClient, never()).deleteMessageBatch(any(DeleteMessageBatchRequest.class)); @@ -605,6 +531,7 @@ void processSqsMessages_should_return_zero_messages_when_messages_are_not_S3Even @Test void populateS3Reference_should_interact_with_getUrlDecodedKey() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + reset(sqsOptions); // Using reflection to unit test a private method as part of bug fix. Class params[] = new Class[2]; params[0] = String.class; @@ -617,21 +544,176 @@ void populateS3Reference_should_interact_with_getUrlDecodedKey() throws NoSuchMe final S3EventNotification.S3ObjectEntity s3ObjectEntity = mock(S3EventNotification.S3ObjectEntity.class); final S3EventNotification.S3BucketEntity s3BucketEntity = mock(S3EventNotification.S3BucketEntity.class); - when(s3EventNotificationRecord.getS3()).thenReturn(s3Entity); - when(s3Entity.getBucket()).thenReturn(s3BucketEntity); - when(s3Entity.getObject()).thenReturn(s3ObjectEntity); - when(s3BucketEntity.getName()).thenReturn("test-bucket-name"); - when(s3ObjectEntity.getUrlDecodedKey()).thenReturn("test-key"); - - final S3ObjectReference s3ObjectReference = (S3ObjectReference) method.invoke(sqsWorker, "test-bucket-name", "test-key"); + final S3ObjectReference s3ObjectReference = (S3ObjectReference) method.invoke(createObjectUnderTest(), "test-bucket-name", "test-key"); assertThat(s3ObjectReference, notNullValue()); assertThat(s3ObjectReference.getBucketName(), equalTo("test-bucket-name")); assertThat(s3ObjectReference.getKey(), equalTo("test-key")); -// verify(s3ObjectEntity).getUrlDecodedKey(); verifyNoMoreInteractions(s3ObjectEntity); } + + @ParameterizedTest + @ValueSource(strings = {"ObjectCreated:Put", "ObjectCreated:Post", "ObjectCreated:Copy", "ObjectCreated:CompleteMultipartUpload"}) + void processSqsMessages_should_return_number_of_messages_processed_with_acknowledgements(final String eventName) throws IOException { + when(acknowledgementSetManager.create(any(), any(Duration.class))).thenReturn(acknowledgementSet); + when(s3SourceConfig.getAcknowledgements()).thenReturn(true); + Instant startTime = Instant.now().minus(1, ChronoUnit.HOURS); + final Message message = mock(Message.class); + when(message.body()).thenReturn(createEventNotification(eventName, startTime)); + final String testReceiptHandle = UUID.randomUUID().toString(); + when(message.messageId()).thenReturn(testReceiptHandle); + when(message.receiptHandle()).thenReturn(testReceiptHandle); + + final ReceiveMessageResponse receiveMessageResponse = mock(ReceiveMessageResponse.class); + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(receiveMessageResponse); + when(receiveMessageResponse.messages()).thenReturn(Collections.singletonList(message)); + + final int messagesProcessed = createObjectUnderTest().processSqsMessages(); + final ArgumentCaptor deleteMessageBatchRequestArgumentCaptor = ArgumentCaptor.forClass(DeleteMessageBatchRequest.class); + + final ArgumentCaptor durationArgumentCaptor = ArgumentCaptor.forClass(Duration.class); + verify(sqsMessageDelayTimer).record(durationArgumentCaptor.capture()); + Duration actualDelay = durationArgumentCaptor.getValue(); + + assertThat(messagesProcessed, equalTo(1)); + verify(s3Service).addS3Object(any(S3ObjectReference.class), any()); + verify(acknowledgementSetManager).create(any(), any(Duration.class)); + verify(sqsMessagesReceivedCounter).increment(1); + verifyNoInteractions(sqsMessagesDeletedCounter); + assertThat(actualDelay, lessThanOrEqualTo(Duration.ofHours(1).plus(Duration.ofSeconds(5)))); + assertThat(actualDelay, greaterThanOrEqualTo(Duration.ofHours(1).minus(Duration.ofSeconds(5)))); + } + + @ParameterizedTest + @ValueSource(strings = {"ObjectCreated:Put", "ObjectCreated:Post", "ObjectCreated:Copy", "ObjectCreated:CompleteMultipartUpload"}) + void processSqsMessages_should_return_number_of_messages_processed_with_acknowledgements_and_progress_check(final String eventName) throws IOException { + when(sqsOptions.getVisibilityDuplicateProtection()).thenReturn(true); + when(sqsOptions.getVisibilityTimeout()).thenReturn(Duration.ofSeconds(6)); + when(acknowledgementSetManager.create(any(), any(Duration.class))).thenReturn(acknowledgementSet); + when(s3SourceConfig.getAcknowledgements()).thenReturn(true); + Instant startTime = Instant.now().minus(1, ChronoUnit.HOURS); + final Message message = mock(Message.class); + when(message.body()).thenReturn(createEventNotification(eventName, startTime)); + final String testReceiptHandle = UUID.randomUUID().toString(); + when(message.messageId()).thenReturn(testReceiptHandle); + when(message.receiptHandle()).thenReturn(testReceiptHandle); + + final ReceiveMessageResponse receiveMessageResponse = mock(ReceiveMessageResponse.class); + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(receiveMessageResponse); + when(receiveMessageResponse.messages()).thenReturn(Collections.singletonList(message)); + + final int messagesProcessed = createObjectUnderTest().processSqsMessages(); + final ArgumentCaptor deleteMessageBatchRequestArgumentCaptor = ArgumentCaptor.forClass(DeleteMessageBatchRequest.class); + + final ArgumentCaptor durationArgumentCaptor = ArgumentCaptor.forClass(Duration.class); + verify(sqsMessageDelayTimer).record(durationArgumentCaptor.capture()); + Duration actualDelay = durationArgumentCaptor.getValue(); + + assertThat(messagesProcessed, equalTo(1)); + verify(s3Service).addS3Object(any(S3ObjectReference.class), any()); + verify(acknowledgementSetManager).create(any(), any(Duration.class)); + verify(acknowledgementSet).addProgressCheck(any(), any(Duration.class)); + verify(sqsMessagesReceivedCounter).increment(1); + verifyNoInteractions(sqsMessagesDeletedCounter); + assertThat(actualDelay, lessThanOrEqualTo(Duration.ofHours(1).plus(Duration.ofSeconds(5)))); + assertThat(actualDelay, greaterThanOrEqualTo(Duration.ofHours(1).minus(Duration.ofSeconds(5)))); + } + + @ParameterizedTest + @ValueSource(strings = {"", "{\"foo\": \"bar\""}) + void processSqsMessages_should_not_interact_with_S3Service_if_input_is_not_valid_JSON(String inputString) { + final Message message = mock(Message.class); + when(message.body()).thenReturn(inputString); + + when(s3SourceConfig.getOnErrorOption()).thenReturn(OnErrorOption.RETAIN_MESSAGES); + + final ReceiveMessageResponse receiveMessageResponse = mock(ReceiveMessageResponse.class); + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(receiveMessageResponse); + when(receiveMessageResponse.messages()).thenReturn(Collections.singletonList(message)); + + final int messagesProcessed = createObjectUnderTest().processSqsMessages(); + assertThat(messagesProcessed, equalTo(1)); + verifyNoInteractions(s3Service); + verify(sqsClient, never()).deleteMessageBatch(any(DeleteMessageBatchRequest.class)); + verify(sqsMessagesReceivedCounter).increment(1); + verify(sqsMessagesFailedCounter).increment(); + } + + @Test + void processSqsMessages_should_update_visibility_timeout_when_progress_changes() throws IOException { + when(sqsOptions.getVisibilityDuplicateProtection()).thenReturn(true); + when(sqsOptions.getVisibilityTimeout()).thenReturn(Duration.ofMillis(1)); + when(acknowledgementSetManager.create(any(), any(Duration.class))).thenReturn(acknowledgementSet); + when(s3SourceConfig.getAcknowledgements()).thenReturn(true); + Instant startTime = Instant.now().minus(1, ChronoUnit.HOURS); + final Message message = mock(Message.class); + when(message.body()).thenReturn(createEventNotification("ObjectCreated:Put", startTime)); + final String testReceiptHandle = UUID.randomUUID().toString(); + when(message.messageId()).thenReturn(testReceiptHandle); + when(message.receiptHandle()).thenReturn(testReceiptHandle); + + final ReceiveMessageResponse receiveMessageResponse = mock(ReceiveMessageResponse.class); + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(receiveMessageResponse); + when(receiveMessageResponse.messages()).thenReturn(Collections.singletonList(message)); + + final int messagesProcessed = createObjectUnderTest().processSqsMessages(); + + assertThat(messagesProcessed, equalTo(1)); + verify(s3Service).addS3Object(any(S3ObjectReference.class), any()); + verify(acknowledgementSetManager).create(any(), any(Duration.class)); + + ArgumentCaptor> progressConsumerArgumentCaptor = ArgumentCaptor.forClass(Consumer.class); + verify(acknowledgementSet).addProgressCheck(progressConsumerArgumentCaptor.capture(), any(Duration.class)); + final Consumer actualConsumer = progressConsumerArgumentCaptor.getValue(); + final ProgressCheck progressCheck = mock(ProgressCheck.class); + actualConsumer.accept(progressCheck); + + ArgumentCaptor changeMessageVisibilityRequestArgumentCaptor = ArgumentCaptor.forClass(ChangeMessageVisibilityRequest.class); + verify(sqsClient).changeMessageVisibility(changeMessageVisibilityRequestArgumentCaptor.capture()); + ChangeMessageVisibilityRequest actualChangeVisibilityRequest = changeMessageVisibilityRequestArgumentCaptor.getValue(); + assertThat(actualChangeVisibilityRequest.queueUrl(), equalTo(queueUrl)); + assertThat(actualChangeVisibilityRequest.receiptHandle(), equalTo(testReceiptHandle)); + verify(sqsMessagesReceivedCounter).increment(1); + verify(sqsMessageDelayTimer).record(any(Duration.class)); + } + + @Test + void processSqsMessages_should_stop_updating_visibility_timeout_after_stop() throws IOException { + when(sqsOptions.getVisibilityDuplicateProtection()).thenReturn(true); + when(sqsOptions.getVisibilityTimeout()).thenReturn(Duration.ofMillis(1)); + when(acknowledgementSetManager.create(any(), any(Duration.class))).thenReturn(acknowledgementSet); + when(s3SourceConfig.getAcknowledgements()).thenReturn(true); + Instant startTime = Instant.now().minus(1, ChronoUnit.HOURS); + final Message message = mock(Message.class); + when(message.body()).thenReturn(createEventNotification("ObjectCreated:Put", startTime)); + final String testReceiptHandle = UUID.randomUUID().toString(); + when(message.messageId()).thenReturn(testReceiptHandle); + when(message.receiptHandle()).thenReturn(testReceiptHandle); + + final ReceiveMessageResponse receiveMessageResponse = mock(ReceiveMessageResponse.class); + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(receiveMessageResponse); + when(receiveMessageResponse.messages()).thenReturn(Collections.singletonList(message)); + + SqsWorker objectUnderTest = createObjectUnderTest(); + final int messagesProcessed = objectUnderTest.processSqsMessages(); + objectUnderTest.stop(); + + assertThat(messagesProcessed, equalTo(1)); + verify(s3Service).addS3Object(any(S3ObjectReference.class), any()); + verify(acknowledgementSetManager).create(any(), any(Duration.class)); + + ArgumentCaptor> progressConsumerArgumentCaptor = ArgumentCaptor.forClass(Consumer.class); + verify(acknowledgementSet).addProgressCheck(progressConsumerArgumentCaptor.capture(), any(Duration.class)); + final Consumer actualConsumer = progressConsumerArgumentCaptor.getValue(); + final ProgressCheck progressCheck = mock(ProgressCheck.class); + actualConsumer.accept(progressCheck); + + verify(sqsClient, never()).changeMessageVisibility(any(ChangeMessageVisibilityRequest.class)); + verify(sqsMessagesReceivedCounter).increment(1); + verify(sqsMessageDelayTimer).record(any(Duration.class)); + } + private static String createPutNotification(final Instant startTime) { return createEventNotification("ObjectCreated:Put", startTime); } diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/parser/ParsedMessageTest.java b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/parser/ParsedMessageTest.java index 3acec973e1..51f3abad06 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/parser/ParsedMessageTest.java +++ b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/parser/ParsedMessageTest.java @@ -2,6 +2,7 @@ import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.opensearch.dataprepper.plugins.source.s3.S3EventBridgeNotification; import org.opensearch.dataprepper.plugins.source.s3.S3EventNotification; @@ -12,33 +13,31 @@ import java.util.UUID; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; class ParsedMessageTest { private static final Random RANDOM = new Random(); private Message message; - private S3EventNotification.S3Entity s3Entity; - private S3EventNotification.S3BucketEntity s3BucketEntity; - private S3EventNotification.S3ObjectEntity s3ObjectEntity; - private S3EventNotification.S3EventNotificationRecord s3EventNotificationRecord; - private S3EventBridgeNotification s3EventBridgeNotification; - private S3EventBridgeNotification.Detail detail; - private S3EventBridgeNotification.Bucket bucket; - private S3EventBridgeNotification.Object object; + private String testBucketName; + private String testDecodedObjectKey; + private long testSize; @BeforeEach void setUp() { message = mock(Message.class); - s3Entity = mock(S3EventNotification.S3Entity.class); - s3BucketEntity = mock(S3EventNotification.S3BucketEntity.class); - s3ObjectEntity = mock(S3EventNotification.S3ObjectEntity.class); - s3EventNotificationRecord = mock(S3EventNotification.S3EventNotificationRecord.class); - s3EventBridgeNotification = mock(S3EventBridgeNotification.class); - detail = mock(S3EventBridgeNotification.Detail.class); - bucket = mock(S3EventBridgeNotification.Bucket.class); - object = mock(S3EventBridgeNotification.Object.class); + testBucketName = UUID.randomUUID().toString(); + testDecodedObjectKey = UUID.randomUUID().toString(); + testSize = RANDOM.nextInt(1_000_000_000) + 1; + } + + @Test + void constructor_with_failed_parsing_throws_if_Message_is_null() { + assertThrows(NullPointerException.class, () -> new ParsedMessage(null, true)); } @Test @@ -50,61 +49,156 @@ void test_parsed_message_with_failed_parsing() { } @Test - void test_parsed_message_with_S3EventNotificationRecord() { - final String testBucketName = UUID.randomUUID().toString(); - final String testDecodedObjectKey = UUID.randomUUID().toString(); - final String testEventName = UUID.randomUUID().toString(); - final DateTime testEventTime = DateTime.now(); - final long testSize = RANDOM.nextLong(); - - when(s3EventNotificationRecord.getS3()).thenReturn(s3Entity); - when(s3Entity.getBucket()).thenReturn(s3BucketEntity); - when(s3Entity.getObject()).thenReturn(s3ObjectEntity); - when(s3ObjectEntity.getSizeAsLong()).thenReturn(testSize); - when(s3BucketEntity.getName()).thenReturn(testBucketName); - when(s3ObjectEntity.getUrlDecodedKey()).thenReturn(testDecodedObjectKey); - when(s3EventNotificationRecord.getEventName()).thenReturn(testEventName); - when(s3EventNotificationRecord.getEventTime()).thenReturn(testEventTime); - - final ParsedMessage parsedMessage = new ParsedMessage(message, List.of(s3EventNotificationRecord)); + void toString_with_failed_parsing_and_messageId() { + final String messageId = UUID.randomUUID().toString(); + when(message.messageId()).thenReturn(messageId); - assertThat(parsedMessage.getMessage(), equalTo(message)); - assertThat(parsedMessage.getBucketName(), equalTo(testBucketName)); - assertThat(parsedMessage.getObjectKey(), equalTo(testDecodedObjectKey)); - assertThat(parsedMessage.getObjectSize(), equalTo(testSize)); - assertThat(parsedMessage.getEventName(), equalTo(testEventName)); - assertThat(parsedMessage.getEventTime(), equalTo(testEventTime)); - assertThat(parsedMessage.isFailedParsing(), equalTo(false)); - assertThat(parsedMessage.isEmptyNotification(), equalTo(false)); + final ParsedMessage parsedMessage = new ParsedMessage(message, true); + final String actualString = parsedMessage.toString(); + assertThat(actualString, notNullValue()); + assertThat(actualString, containsString(messageId)); } @Test - void test_parsed_message_with_S3EventBridgeNotification() { - final String testBucketName = UUID.randomUUID().toString(); - final String testDecodedObjectKey = UUID.randomUUID().toString(); - final String testDetailType = UUID.randomUUID().toString(); - final DateTime testEventTime = DateTime.now(); - final int testSize = RANDOM.nextInt(); + void toString_with_failed_parsing_and_no_messageId() { + final ParsedMessage parsedMessage = new ParsedMessage(message, true); + final String actualString = parsedMessage.toString(); + assertThat(actualString, notNullValue()); + } - when(s3EventBridgeNotification.getDetail()).thenReturn(detail); - when(s3EventBridgeNotification.getDetail().getBucket()).thenReturn(bucket); - when(s3EventBridgeNotification.getDetail().getObject()).thenReturn(object); + @Nested + class WithS3EventNotificationRecord { + private S3EventNotification.S3Entity s3Entity; + private S3EventNotification.S3BucketEntity s3BucketEntity; + private S3EventNotification.S3ObjectEntity s3ObjectEntity; + private S3EventNotification.S3EventNotificationRecord s3EventNotificationRecord; + private List s3EventNotificationRecords; + private String testEventName; + private DateTime testEventTime; - when(bucket.getName()).thenReturn(testBucketName); - when(object.getUrlDecodedKey()).thenReturn(testDecodedObjectKey); - when(object.getSize()).thenReturn(testSize); - when(s3EventBridgeNotification.getDetailType()).thenReturn(testDetailType); - when(s3EventBridgeNotification.getTime()).thenReturn(testEventTime); + @BeforeEach + void setUp() { + testEventName = UUID.randomUUID().toString(); + testEventTime = DateTime.now(); - final ParsedMessage parsedMessage = new ParsedMessage(message, s3EventBridgeNotification); + s3Entity = mock(S3EventNotification.S3Entity.class); + s3BucketEntity = mock(S3EventNotification.S3BucketEntity.class); + s3ObjectEntity = mock(S3EventNotification.S3ObjectEntity.class); + s3EventNotificationRecord = mock(S3EventNotification.S3EventNotificationRecord.class); - assertThat(parsedMessage.getMessage(), equalTo(message)); - assertThat(parsedMessage.getBucketName(), equalTo(testBucketName)); - assertThat(parsedMessage.getObjectKey(), equalTo(testDecodedObjectKey)); - assertThat(parsedMessage.getObjectSize(), equalTo((long) testSize)); - assertThat(parsedMessage.getDetailType(), equalTo(testDetailType)); - assertThat(parsedMessage.getEventTime(), equalTo(testEventTime)); - assertThat(parsedMessage.isFailedParsing(), equalTo(false)); - assertThat(parsedMessage.isEmptyNotification(), equalTo(false)); + when(s3EventNotificationRecord.getS3()).thenReturn(s3Entity); + when(s3Entity.getBucket()).thenReturn(s3BucketEntity); + when(s3Entity.getObject()).thenReturn(s3ObjectEntity); + when(s3ObjectEntity.getSizeAsLong()).thenReturn(testSize); + when(s3BucketEntity.getName()).thenReturn(testBucketName); + when(s3ObjectEntity.getUrlDecodedKey()).thenReturn(testDecodedObjectKey); + when(s3EventNotificationRecord.getEventName()).thenReturn(testEventName); + when(s3EventNotificationRecord.getEventTime()).thenReturn(testEventTime); + + s3EventNotificationRecords = List.of(s3EventNotificationRecord); + } + + private ParsedMessage createObjectUnderTest() { + return new ParsedMessage(message, s3EventNotificationRecords); + } + + @Test + void constructor_with_S3EventNotificationRecord_throws_if_Message_is_null() { + message = null; + assertThrows(NullPointerException.class, this::createObjectUnderTest); + } + + @Test + void test_parsed_message_with_S3EventNotificationRecord() { + final ParsedMessage parsedMessage = createObjectUnderTest(); + + assertThat(parsedMessage.getMessage(), equalTo(message)); + assertThat(parsedMessage.getBucketName(), equalTo(testBucketName)); + assertThat(parsedMessage.getObjectKey(), equalTo(testDecodedObjectKey)); + assertThat(parsedMessage.getObjectSize(), equalTo(testSize)); + assertThat(parsedMessage.getEventName(), equalTo(testEventName)); + assertThat(parsedMessage.getEventTime(), equalTo(testEventTime)); + assertThat(parsedMessage.isFailedParsing(), equalTo(false)); + assertThat(parsedMessage.isEmptyNotification(), equalTo(false)); + } + + @Test + void toString_with_messageId() { + final String messageId = UUID.randomUUID().toString(); + when(message.messageId()).thenReturn(messageId); + + final ParsedMessage parsedMessage = createObjectUnderTest(); + final String actualString = parsedMessage.toString(); + assertThat(actualString, notNullValue()); + assertThat(actualString, containsString(messageId)); + assertThat(actualString, containsString(testDecodedObjectKey)); + } + } + + @Nested + class WithS3EventBridgeNotification { + private String testDetailType; + private DateTime testEventTime; + private S3EventBridgeNotification s3EventBridgeNotification; + private S3EventBridgeNotification.Detail detail; + private S3EventBridgeNotification.Bucket bucket; + private S3EventBridgeNotification.Object object; + + @BeforeEach + void setUp() { + s3EventBridgeNotification = mock(S3EventBridgeNotification.class); + detail = mock(S3EventBridgeNotification.Detail.class); + bucket = mock(S3EventBridgeNotification.Bucket.class); + object = mock(S3EventBridgeNotification.Object.class); + + testDetailType = UUID.randomUUID().toString(); + testEventTime = DateTime.now(); + + when(s3EventBridgeNotification.getDetail()).thenReturn(detail); + when(s3EventBridgeNotification.getDetail().getBucket()).thenReturn(bucket); + when(s3EventBridgeNotification.getDetail().getObject()).thenReturn(object); + + when(bucket.getName()).thenReturn(testBucketName); + when(object.getUrlDecodedKey()).thenReturn(testDecodedObjectKey); + when(object.getSize()).thenReturn((int) testSize); + when(s3EventBridgeNotification.getDetailType()).thenReturn(testDetailType); + when(s3EventBridgeNotification.getTime()).thenReturn(testEventTime); + } + + private ParsedMessage createObjectUnderTest() { + return new ParsedMessage(message, s3EventBridgeNotification); + } + + @Test + void constructor_with_S3EventBridgeNotification_throws_if_Message_is_null() { + message = null; + assertThrows(NullPointerException.class, () -> createObjectUnderTest()); + } + + @Test + void test_parsed_message_with_S3EventBridgeNotification() { + final ParsedMessage parsedMessage = createObjectUnderTest(); + + assertThat(parsedMessage.getMessage(), equalTo(message)); + assertThat(parsedMessage.getBucketName(), equalTo(testBucketName)); + assertThat(parsedMessage.getObjectKey(), equalTo(testDecodedObjectKey)); + assertThat(parsedMessage.getObjectSize(), equalTo(testSize)); + assertThat(parsedMessage.getDetailType(), equalTo(testDetailType)); + assertThat(parsedMessage.getEventTime(), equalTo(testEventTime)); + assertThat(parsedMessage.isFailedParsing(), equalTo(false)); + assertThat(parsedMessage.isEmptyNotification(), equalTo(false)); + } + + @Test + void toString_with_messageId() { + final String messageId = UUID.randomUUID().toString(); + when(message.messageId()).thenReturn(messageId); + + final ParsedMessage parsedMessage = createObjectUnderTest(); + final String actualString = parsedMessage.toString(); + assertThat(actualString, notNullValue()); + assertThat(actualString, containsString(messageId)); + assertThat(actualString, containsString(testDecodedObjectKey)); + } } } diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/parser/S3EventBridgeNotificationParserTest.java b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/parser/S3EventBridgeNotificationParserTest.java index c779ec561f..db361d70e1 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/parser/S3EventBridgeNotificationParserTest.java +++ b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/parser/S3EventBridgeNotificationParserTest.java @@ -19,7 +19,7 @@ class S3EventBridgeNotificationParserTest { private final ObjectMapper objectMapper = new ObjectMapper(); - private final String EVENTBRIDGE_MESSAGE = "{\"version\":\"0\",\"id\":\"17793124-05d4-b198-2fde-7ededc63b103\",\"detail-type\":\"Object Created\"," + + static final String EVENTBRIDGE_MESSAGE = "{\"version\":\"0\",\"id\":\"17793124-05d4-b198-2fde-7ededc63b103\",\"detail-type\":\"Object Created\"," + "\"source\":\"aws.s3\",\"account\":\"111122223333\",\"time\":\"2021-11-12T00:00:00Z\"," + "\"region\":\"ca-central-1\",\"resources\":[\"arn:aws:s3:::DOC-EXAMPLE-BUCKET1\"]," + "\"detail\":{\"version\":\"0\",\"bucket\":{\"name\":\"DOC-EXAMPLE-BUCKET1\"}," + diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/parser/S3EventNotificationParserTest.java b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/parser/S3EventNotificationParserTest.java index a3d2c91679..c9e3a39da8 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/parser/S3EventNotificationParserTest.java +++ b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/parser/S3EventNotificationParserTest.java @@ -16,8 +16,8 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -class S3EventNotificationParserTest { - private static final String DIRECT_SQS_MESSAGE = +public class S3EventNotificationParserTest { + static final String DIRECT_SQS_MESSAGE = "{\"Records\":[{\"eventVersion\":\"2.1\",\"eventSource\":\"aws:s3\",\"awsRegion\":\"us-east-1\",\"eventTime\":\"2023-04-28T16:00:11.324Z\"," + "\"eventName\":\"ObjectCreated:Put\",\"userIdentity\":{\"principalId\":\"AWS:xyz\"},\"requestParameters\":{\"sourceIPAddress\":\"127.0.0.1\"}," + "\"responseElements\":{\"x-amz-request-id\":\"xyz\",\"x-amz-id-2\":\"xyz\"},\"s3\":{\"s3SchemaVersion\":\"1.0\"," + @@ -25,7 +25,7 @@ class S3EventNotificationParserTest { "\"arn\":\"arn:aws:s3:::my-bucket\"},\"object\":{\"key\":\"path/to/myfile.log.gz\",\"size\":3159112,\"eTag\":\"abcd123\"," + "\"sequencer\":\"000\"}}}]}"; - private static final String SNS_BASED_MESSAGE = "{\n" + + public static final String SNS_BASED_MESSAGE = "{\n" + " \"Type\" : \"Notification\",\n" + " \"MessageId\" : \"4e01e115-5b91-5096-8a74-bee95ed1e123\",\n" + " \"TopicArn\" : \"arn:aws:sns:us-east-1:123456789012:notifications\",\n" + diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/parser/SqsMessageParserTest.java b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/parser/SqsMessageParserTest.java new file mode 100644 index 0000000000..d0dd711f7e --- /dev/null +++ b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/parser/SqsMessageParserTest.java @@ -0,0 +1,96 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.s3.parser; + +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.s3.S3SourceConfig; +import org.opensearch.dataprepper.plugins.source.s3.configuration.NotificationSourceOption; +import software.amazon.awssdk.services.sqs.model.Message; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.empty; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class SqsMessageParserTest { + @Mock + private S3SourceConfig s3SourceConfig; + + private SqsMessageParser createObjectUnderTest() { + return new SqsMessageParser(s3SourceConfig); + } + + @ParameterizedTest + @ArgumentsSource(SourceArgumentsProvider.class) + void parseSqsMessages_returns_empty_for_empty_messages(final NotificationSourceOption sourceOption) { + when(s3SourceConfig.getNotificationSource()).thenReturn(sourceOption); + final Collection parsedMessages = createObjectUnderTest().parseSqsMessages(Collections.emptyList()); + + assertThat(parsedMessages, notNullValue()); + assertThat(parsedMessages, empty()); + } + + @ParameterizedTest + @ArgumentsSource(SourceArgumentsProvider.class) + void parseSqsMessages_parsed_messages(final NotificationSourceOption sourceOption, + final String messageBody, + final String replacementString) { + when(s3SourceConfig.getNotificationSource()).thenReturn(sourceOption); + final int numberOfMessages = 10; + List messages = IntStream.range(0, numberOfMessages) + .mapToObj(i -> messageBody.replaceAll(replacementString, replacementString + i)) + .map(SqsMessageParserTest::createMockMessage) + .collect(Collectors.toList()); + final Collection parsedMessages = createObjectUnderTest().parseSqsMessages(messages); + + assertThat(parsedMessages, notNullValue()); + assertThat(parsedMessages.size(), equalTo(numberOfMessages)); + + final Set bucketNames = parsedMessages.stream().map(ParsedMessage::getBucketName).collect(Collectors.toSet()); + assertThat("The bucket names are unique, so the bucketNames should match the numberOfMessages.", + bucketNames.size(), equalTo(numberOfMessages)); + } + + static class SourceArgumentsProvider implements ArgumentsProvider { + @Override + public Stream provideArguments(final ExtensionContext extensionContext) { + return Stream.of( + Arguments.arguments( + NotificationSourceOption.S3, + S3EventNotificationParserTest.DIRECT_SQS_MESSAGE, + "my-bucket"), + Arguments.arguments( + NotificationSourceOption.EVENTBRIDGE, + S3EventBridgeNotificationParserTest.EVENTBRIDGE_MESSAGE, + "DOC-EXAMPLE-BUCKET1") + ); + } + } + + private static Message createMockMessage(final String body) { + final Message message = mock(Message.class); + when(message.body()).thenReturn(body); + return message; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/s3-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-plugins/s3-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 23c33feb6d..0000000000 --- a/data-prepper-plugins/s3-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1,3 +0,0 @@ -# To enable mocking of final classes with vanilla Mockito -# https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods -mock-maker-inline diff --git a/data-prepper-plugins/service-map-stateful/build.gradle b/data-prepper-plugins/service-map-stateful/build.gradle index 60b9512ed9..ab2300f020 100644 --- a/data-prepper-plugins/service-map-stateful/build.gradle +++ b/data-prepper-plugins/service-map-stateful/build.gradle @@ -19,7 +19,7 @@ dependencies { exclude group: 'com.google.protobuf', module: 'protobuf-java' } implementation libs.protobuf.core - testImplementation testLibs.mockito.inline + testImplementation project(':data-prepper-test-common') } jacocoTestCoverageVerification { diff --git a/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/ServiceMapProcessorConfig.java b/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/ServiceMapProcessorConfig.java index 8c337b2737..7f72fb5286 100644 --- a/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/ServiceMapProcessorConfig.java +++ b/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/ServiceMapProcessorConfig.java @@ -5,8 +5,20 @@ package org.opensearch.dataprepper.plugins.processor; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; + public class ServiceMapProcessorConfig { - static final String WINDOW_DURATION = "window_duration"; + private static final String WINDOW_DURATION = "window_duration"; static final int DEFAULT_WINDOW_DURATION = 180; static final String DEFAULT_DB_PATH = "data/service-map/"; + + @JsonProperty(WINDOW_DURATION) + @JsonPropertyDescription("Represents the fixed time window, in seconds, " + + "during which service map relationships are evaluated. Default value is 180.") + private int windowDuration = DEFAULT_WINDOW_DURATION; + + public int getWindowDuration() { + return windowDuration; + } } diff --git a/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/ServiceMapStatefulProcessor.java b/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/ServiceMapStatefulProcessor.java index c02ccb17d6..75041a09b4 100644 --- a/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/ServiceMapStatefulProcessor.java +++ b/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/ServiceMapStatefulProcessor.java @@ -6,9 +6,11 @@ package org.opensearch.dataprepper.plugins.processor; import org.apache.commons.codec.DecoderException; +import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.annotations.SingleThread; -import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.JacksonEvent; import org.opensearch.dataprepper.model.peerforwarder.RequiresPeerForwarding; @@ -40,7 +42,8 @@ import java.util.concurrent.atomic.AtomicInteger; @SingleThread -@DataPrepperPlugin(name = "service_map", deprecatedName = "service_map_stateful", pluginType = Processor.class) +@DataPrepperPlugin(name = "service_map", deprecatedName = "service_map_stateful", pluginType = Processor.class, + pluginConfigurationType = ServiceMapProcessorConfig.class) public class ServiceMapStatefulProcessor extends AbstractProcessor, Record> implements RequiresPeerForwarding { static final String SPANS_DB_SIZE = "spansDbSize"; @@ -75,20 +78,24 @@ public class ServiceMapStatefulProcessor extends AbstractProcessor private final int thisProcessorId; - public ServiceMapStatefulProcessor(final PluginSetting pluginSetting) { - this(pluginSetting.getIntegerOrDefault(ServiceMapProcessorConfig.WINDOW_DURATION, ServiceMapProcessorConfig.DEFAULT_WINDOW_DURATION) * TO_MILLIS, + @DataPrepperPluginConstructor + public ServiceMapStatefulProcessor( + final ServiceMapProcessorConfig serviceMapProcessorConfig, + final PluginMetrics pluginMetrics, + final PipelineDescription pipelineDescription) { + this((long) serviceMapProcessorConfig.getWindowDuration() * TO_MILLIS, new File(ServiceMapProcessorConfig.DEFAULT_DB_PATH), Clock.systemUTC(), - pluginSetting.getNumberOfProcessWorkers(), - pluginSetting); + pipelineDescription.getNumberOfProcessWorkers(), + pluginMetrics); } - public ServiceMapStatefulProcessor(final long windowDurationMillis, + ServiceMapStatefulProcessor(final long windowDurationMillis, final File databasePath, final Clock clock, final int processWorkers, - final PluginSetting pluginSetting) { - super(pluginSetting); + final PluginMetrics pluginMetrics) { + super(pluginMetrics); ServiceMapStatefulProcessor.clock = clock; this.thisProcessorId = processorsCreated.getAndIncrement(); diff --git a/data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/processor/ServiceMapProcessorConfigTest.java b/data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/processor/ServiceMapProcessorConfigTest.java new file mode 100644 index 0000000000..35ef3b0c07 --- /dev/null +++ b/data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/processor/ServiceMapProcessorConfigTest.java @@ -0,0 +1,38 @@ +package org.opensearch.dataprepper.plugins.processor; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.test.helper.ReflectivelySetField; + +import java.util.Random; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.opensearch.dataprepper.plugins.processor.ServiceMapProcessorConfig.DEFAULT_WINDOW_DURATION; + +class ServiceMapProcessorConfigTest { + private ServiceMapProcessorConfig serviceMapProcessorConfig; + Random random; + + @BeforeEach + void setUp() { + serviceMapProcessorConfig = new ServiceMapProcessorConfig(); + random = new Random(); + } + + @Test + void testDefaultConfig() { + assertThat(serviceMapProcessorConfig.getWindowDuration(), equalTo(DEFAULT_WINDOW_DURATION)); + } + + @Test + void testGetter() throws NoSuchFieldException, IllegalAccessException { + final int windowDuration = 1 + random.nextInt(300); + ReflectivelySetField.setField( + ServiceMapProcessorConfig.class, + serviceMapProcessorConfig, + "windowDuration", + windowDuration); + assertThat(serviceMapProcessorConfig.getWindowDuration(), equalTo(windowDuration)); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/processor/ServiceMapStatefulProcessorTest.java b/data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/processor/ServiceMapStatefulProcessorTest.java index 28789615aa..b565642e19 100644 --- a/data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/processor/ServiceMapStatefulProcessorTest.java +++ b/data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/processor/ServiceMapStatefulProcessorTest.java @@ -14,6 +14,8 @@ import org.mockito.Mockito; import org.opensearch.dataprepper.metrics.MetricNames; import org.opensearch.dataprepper.metrics.MetricsTestUtil; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.trace.Span; @@ -43,6 +45,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.processor.ServiceMapProcessorConfig.DEFAULT_WINDOW_DURATION; public class ServiceMapStatefulProcessorTest { @@ -54,12 +57,20 @@ public class ServiceMapStatefulProcessorTest { private static final String PAYMENT_SERVICE = "PAY"; private static final String CART_SERVICE = "CART"; private PluginSetting pluginSetting; + private PluginMetrics pluginMetrics; + private PipelineDescription pipelineDescription; + private ServiceMapProcessorConfig serviceMapProcessorConfig; @BeforeEach public void setup() throws NoSuchFieldException, IllegalAccessException { resetServiceMapStatefulProcessorStatic(); MetricsTestUtil.initMetrics(); pluginSetting = mock(PluginSetting.class); + pipelineDescription = mock(PipelineDescription.class); + serviceMapProcessorConfig = mock(ServiceMapProcessorConfig.class); + when(serviceMapProcessorConfig.getWindowDuration()).thenReturn(DEFAULT_WINDOW_DURATION); + pluginMetrics = PluginMetrics.fromNames( + "testServiceMapProcessor", "testPipelineName"); when(pluginSetting.getName()).thenReturn("testServiceMapProcessor"); when(pluginSetting.getPipelineName()).thenReturn("testPipelineName"); } @@ -116,13 +127,11 @@ private Set evaluateEdges(Set serv } @Test - public void testPluginSettingConstructor() { - - final PluginSetting pluginSetting = new PluginSetting("testPluginSetting", Collections.emptyMap()); - pluginSetting.setProcessWorkers(4); - pluginSetting.setPipelineName("TestPipeline"); + public void testDataPrepperConstructor() { + when(pipelineDescription.getNumberOfProcessWorkers()).thenReturn(4); //Nothing is accessible to validate, so just verify that no exception is thrown. - final ServiceMapStatefulProcessor serviceMapStatefulProcessor = new ServiceMapStatefulProcessor(pluginSetting); + final ServiceMapStatefulProcessor serviceMapStatefulProcessor = new ServiceMapStatefulProcessor( + serviceMapProcessorConfig, pluginMetrics, pipelineDescription); } @Test @@ -132,8 +141,8 @@ public void testTraceGroupsWithEventRecordData() throws Exception { Mockito.when(clock.instant()).thenReturn(Instant.now()); ExecutorService threadpool = Executors.newCachedThreadPool(); final File path = new File(ServiceMapProcessorConfig.DEFAULT_DB_PATH); - final ServiceMapStatefulProcessor serviceMapStateful1 = new ServiceMapStatefulProcessor(100, path, clock, 2, pluginSetting); - final ServiceMapStatefulProcessor serviceMapStateful2 = new ServiceMapStatefulProcessor(100, path, clock, 2, pluginSetting); + final ServiceMapStatefulProcessor serviceMapStateful1 = new ServiceMapStatefulProcessor(100, path, clock, 2, pluginMetrics); + final ServiceMapStatefulProcessor serviceMapStateful2 = new ServiceMapStatefulProcessor(100, path, clock, 2, pluginMetrics); final byte[] rootSpanId1Bytes = ServiceMapTestUtils.getRandomBytes(8); final byte[] rootSpanId2Bytes = ServiceMapTestUtils.getRandomBytes(8); @@ -327,8 +336,8 @@ public void testTraceGroupsWithIsolatedServiceEventRecordData() throws Exception Mockito.when(clock.instant()).thenReturn(Instant.now()); ExecutorService threadpool = Executors.newCachedThreadPool(); final File path = new File(ServiceMapProcessorConfig.DEFAULT_DB_PATH); - final ServiceMapStatefulProcessor serviceMapStateful1 = new ServiceMapStatefulProcessor(100, path, clock, 2, pluginSetting); - final ServiceMapStatefulProcessor serviceMapStateful2 = new ServiceMapStatefulProcessor(100, path, clock, 2, pluginSetting); + final ServiceMapStatefulProcessor serviceMapStateful1 = new ServiceMapStatefulProcessor(100, path, clock, 2, pluginMetrics); + final ServiceMapStatefulProcessor serviceMapStateful2 = new ServiceMapStatefulProcessor(100, path, clock, 2, pluginMetrics); final byte[] rootSpanIdBytes = ServiceMapTestUtils.getRandomBytes(8); final byte[] traceIdBytes = ServiceMapTestUtils.getRandomBytes(16); @@ -383,7 +392,7 @@ public void testTraceGroupsWithIsolatedServiceEventRecordData() throws Exception @Test public void testPrepareForShutdownWithEventRecordData() { final File path = new File(ServiceMapProcessorConfig.DEFAULT_DB_PATH); - final ServiceMapStatefulProcessor serviceMapStateful = new ServiceMapStatefulProcessor(100, path, Clock.systemUTC(), 1, pluginSetting); + final ServiceMapStatefulProcessor serviceMapStateful = new ServiceMapStatefulProcessor(100, path, Clock.systemUTC(), 1, pluginMetrics); final byte[] rootSpanId1Bytes = ServiceMapTestUtils.getRandomBytes(8); final byte[] traceId1Bytes = ServiceMapTestUtils.getRandomBytes(16); @@ -411,11 +420,9 @@ public void testPrepareForShutdownWithEventRecordData() { @Test public void testGetIdentificationKeys() { - final PluginSetting pluginSetting = new PluginSetting("testPluginSetting", Collections.emptyMap()); - pluginSetting.setProcessWorkers(4); - pluginSetting.setPipelineName("TestPipeline"); - - final ServiceMapStatefulProcessor serviceMapStatefulProcessor = new ServiceMapStatefulProcessor(pluginSetting); + when(pipelineDescription.getNumberOfProcessWorkers()).thenReturn(4); + final ServiceMapStatefulProcessor serviceMapStatefulProcessor = new ServiceMapStatefulProcessor( + serviceMapProcessorConfig, pluginMetrics, pipelineDescription); final Collection expectedIdentificationKeys = serviceMapStatefulProcessor.getIdentificationKeys(); assertThat(expectedIdentificationKeys, equalTo(Collections.singleton("traceId"))); diff --git a/data-prepper-plugins/sns-sink/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-plugins/sns-sink/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 23c33feb6d..0000000000 --- a/data-prepper-plugins/sns-sink/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1,3 +0,0 @@ -# To enable mocking of final classes with vanilla Mockito -# https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods -mock-maker-inline diff --git a/data-prepper-plugins/sqs-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-plugins/sqs-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 23c33feb6d..0000000000 --- a/data-prepper-plugins/sqs-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1,3 +0,0 @@ -# To enable mocking of final classes with vanilla Mockito -# https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods -mock-maker-inline diff --git a/data-prepper-plugins/truncate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/truncate/TruncateProcessorConfig.java b/data-prepper-plugins/truncate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/truncate/TruncateProcessorConfig.java index 7fde949719..02c83f5773 100644 --- a/data-prepper-plugins/truncate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/truncate/TruncateProcessorConfig.java +++ b/data-prepper-plugins/truncate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/truncate/TruncateProcessorConfig.java @@ -6,6 +6,7 @@ package org.opensearch.dataprepper.plugins.processor.truncate; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.AssertTrue; @@ -16,18 +17,25 @@ public class TruncateProcessorConfig { public static class Entry { @JsonProperty("source_keys") + @JsonPropertyDescription("The list of source keys that will be modified by the processor. " + + "The default value is an empty list, which indicates that all values will be truncated.") private List sourceKeys; @JsonProperty("start_at") + @JsonPropertyDescription("Where in the string value to start truncation. " + + "Default is `0`, which specifies to start truncation at the beginning of each key's value.") private Integer startAt; @JsonProperty("length") + @JsonPropertyDescription("The length of the string after truncation. " + + "When not specified, the processor will measure the length based on where the string ends.") private Integer length; @JsonProperty("recursive") private Boolean recurse = false; @JsonProperty("truncate_when") + @JsonPropertyDescription("A condition that, when met, determines when the truncate operation is performed.") private String truncateWhen; public Entry(final List sourceKeys, final Integer startAt, final Integer length, final String truncateWhen, final Boolean recurse) { @@ -77,6 +85,7 @@ public boolean isValidConfig() { @NotEmpty @NotNull + @JsonPropertyDescription("A list of entries to add to an event.") private List<@Valid Entry> entries; public List getEntries() { diff --git a/data-prepper-plugins/user-agent-processor/build.gradle b/data-prepper-plugins/user-agent-processor/build.gradle index 746ee40397..5e92b158f5 100644 --- a/data-prepper-plugins/user-agent-processor/build.gradle +++ b/data-prepper-plugins/user-agent-processor/build.gradle @@ -13,6 +13,7 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.github.ua-parser:uap-java:1.6.1' implementation libs.caffeine + testImplementation project(':data-prepper-test-event') } jacocoTestCoverageVerification { diff --git a/data-prepper-plugins/user-agent-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessor.java b/data-prepper-plugins/user-agent-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessor.java index 32779655dc..c84b308645 100644 --- a/data-prepper-plugins/user-agent-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessor.java +++ b/data-prepper-plugins/user-agent-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessor.java @@ -9,6 +9,8 @@ import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventKey; +import org.opensearch.dataprepper.model.event.EventKeyFactory; import org.opensearch.dataprepper.model.processor.AbstractProcessor; import org.opensearch.dataprepper.model.processor.Processor; import org.opensearch.dataprepper.model.record.Record; @@ -30,12 +32,19 @@ public class UserAgentProcessor extends AbstractProcessor, Record< private static final Logger LOG = LoggerFactory.getLogger(UserAgentProcessor.class); private final UserAgentProcessorConfig config; private final Parser userAgentParser; + private final EventKey sourceKey; + private final EventKey targetKey; @DataPrepperPluginConstructor - public UserAgentProcessor(final PluginMetrics pluginMetrics, final UserAgentProcessorConfig config) { + public UserAgentProcessor( + final UserAgentProcessorConfig config, + final EventKeyFactory eventKeyFactory, + final PluginMetrics pluginMetrics) { super(pluginMetrics); this.config = config; this.userAgentParser = new CaffeineCachingParser(config.getCacheSize()); + this.sourceKey = config.getSource(); + this.targetKey = eventKeyFactory.createEventKey(config.getTarget(), EventKeyFactory.EventAction.PUT); } @Override @@ -44,7 +53,7 @@ public Collection> doExecute(final Collection> recor final Event event = record.getData(); try { - final String userAgentStr = event.get(config.getSource(), String.class); + final String userAgentStr = event.get(sourceKey, String.class); Objects.requireNonNull(userAgentStr); final Client clientInfo = this.userAgentParser.parse(userAgentStr); @@ -53,10 +62,10 @@ public Collection> doExecute(final Collection> recor if (!config.getExcludeOriginal()) { parsedUserAgent.put("original", userAgentStr); } - event.put(config.getTarget(), parsedUserAgent); + event.put(targetKey, parsedUserAgent); } catch (Exception e) { LOG.error(EVENT, "An exception occurred when parsing user agent data from event [{}] with source key [{}]", - event, config.getSource(), e); + event, sourceKey, e); final List tagsOnParseFailure = config.getTagsOnParseFailure(); if (Objects.nonNull(tagsOnParseFailure) && tagsOnParseFailure.size() > 0) { diff --git a/data-prepper-plugins/user-agent-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessorConfig.java b/data-prepper-plugins/user-agent-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessorConfig.java index e62fc5a2da..0dcf46e2a1 100644 --- a/data-prepper-plugins/user-agent-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessorConfig.java +++ b/data-prepper-plugins/user-agent-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessorConfig.java @@ -8,6 +8,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; +import org.opensearch.dataprepper.model.event.EventKey; +import org.opensearch.dataprepper.model.event.EventKeyConfiguration; +import org.opensearch.dataprepper.model.event.EventKeyFactory; import java.util.List; @@ -18,7 +21,8 @@ public class UserAgentProcessorConfig { @NotEmpty @NotNull @JsonProperty("source") - private String source; + @EventKeyConfiguration(EventKeyFactory.EventAction.GET) + private EventKey source; @NotNull @JsonProperty("target") @@ -34,7 +38,7 @@ public class UserAgentProcessorConfig { @JsonProperty("tags_on_parse_failure") private List tagsOnParseFailure; - public String getSource() { + public EventKey getSource() { return source; } diff --git a/data-prepper-plugins/user-agent-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessorTest.java b/data-prepper-plugins/user-agent-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessorTest.java index da0923f509..a346218d0a 100644 --- a/data-prepper-plugins/user-agent-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessorTest.java +++ b/data-prepper-plugins/user-agent-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessorTest.java @@ -12,8 +12,10 @@ import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.event.TestEventKeyFactory; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventKeyFactory; import org.opensearch.dataprepper.model.event.JacksonEvent; import org.opensearch.dataprepper.model.record.Record; @@ -38,11 +40,13 @@ class UserAgentProcessorTest { @Mock private UserAgentProcessorConfig mockConfig; + private final EventKeyFactory eventKeyFactory = TestEventKeyFactory.getTestEventFactory(); + @ParameterizedTest @MethodSource("userAgentStringArguments") public void testParsingUserAgentStrings( String uaString, String uaName, String uaVersion, String osName, String osVersion, String osFull, String deviceName) { - when(mockConfig.getSource()).thenReturn("source"); + when(mockConfig.getSource()).thenReturn(eventKeyFactory.createEventKey("source")); when(mockConfig.getTarget()).thenReturn("user_agent"); when(mockConfig.getCacheSize()).thenReturn(TEST_CACHE_SIZE); @@ -64,7 +68,7 @@ public void testParsingUserAgentStrings( @MethodSource("userAgentStringArguments") public void testParsingUserAgentStringsWithCustomTarget( String uaString, String uaName, String uaVersion, String osName, String osVersion, String osFull, String deviceName) { - when(mockConfig.getSource()).thenReturn("source"); + when(mockConfig.getSource()).thenReturn(eventKeyFactory.createEventKey("source")); when(mockConfig.getTarget()).thenReturn("my_target"); when(mockConfig.getCacheSize()).thenReturn(TEST_CACHE_SIZE); @@ -86,7 +90,7 @@ public void testParsingUserAgentStringsWithCustomTarget( @MethodSource("userAgentStringArguments") public void testParsingUserAgentStringsExcludeOriginal( String uaString, String uaName, String uaVersion, String osName, String osVersion, String osFull, String deviceName) { - when(mockConfig.getSource()).thenReturn("source"); + when(mockConfig.getSource()).thenReturn(eventKeyFactory.createEventKey("source")); when(mockConfig.getTarget()).thenReturn("user_agent"); when(mockConfig.getExcludeOriginal()).thenReturn(true); when(mockConfig.getCacheSize()).thenReturn(TEST_CACHE_SIZE); @@ -107,8 +111,9 @@ public void testParsingUserAgentStringsExcludeOriginal( @Test public void testParsingWhenUserAgentStringNotExist() { - when(mockConfig.getSource()).thenReturn("bad_source"); + when(mockConfig.getSource()).thenReturn(eventKeyFactory.createEventKey("bad_source")); when(mockConfig.getCacheSize()).thenReturn(TEST_CACHE_SIZE); + when(mockConfig.getTarget()).thenReturn("user_agent"); final UserAgentProcessor processor = createObjectUnderTest(); final Record testRecord = createTestRecord(UUID.randomUUID().toString()); @@ -120,8 +125,9 @@ public void testParsingWhenUserAgentStringNotExist() { @Test public void testTagsAddedOnParseFailure() { - when(mockConfig.getSource()).thenReturn("bad_source"); + when(mockConfig.getSource()).thenReturn(eventKeyFactory.createEventKey("bad_source")); when(mockConfig.getCacheSize()).thenReturn(TEST_CACHE_SIZE); + when(mockConfig.getTarget()).thenReturn("user_agent"); final String tagOnFailure1 = UUID.randomUUID().toString(); final String tagOnFailure2 = UUID.randomUUID().toString(); @@ -138,7 +144,7 @@ public void testTagsAddedOnParseFailure() { } private UserAgentProcessor createObjectUnderTest() { - return new UserAgentProcessor(pluginMetrics, mockConfig); + return new UserAgentProcessor(mockConfig, eventKeyFactory, pluginMetrics); } private Record createTestRecord(String uaString) { diff --git a/examples/trace-analytics-sample-app/sample-app/requirements.txt b/examples/trace-analytics-sample-app/sample-app/requirements.txt index df780b836b..a24bef87af 100644 --- a/examples/trace-analytics-sample-app/sample-app/requirements.txt +++ b/examples/trace-analytics-sample-app/sample-app/requirements.txt @@ -1,10 +1,10 @@ dash==2.15.0 mysql-connector==2.2.9 -opentelemetry-exporter-otlp==1.20.0 -opentelemetry-instrumentation-flask==0.41b0 -opentelemetry-instrumentation-mysql==0.41b0 -opentelemetry-instrumentation-requests==0.41b0 -opentelemetry-sdk==1.20.0 +opentelemetry-exporter-otlp==1.25.0 +opentelemetry-instrumentation-flask==0.46b0 +opentelemetry-instrumentation-mysql==0.46b0 +opentelemetry-instrumentation-requests==0.46b0 +opentelemetry-sdk==1.25.0 protobuf==3.20.3 urllib3==2.2.2 werkzeug==3.0.3 \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index b82aa23a4f..a4413138c9 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index 1aa94a4269..b740cf1339 100755 --- a/gradlew +++ b/gradlew @@ -55,7 +55,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. diff --git a/release/smoke-tests/otel-span-exporter/requirements.txt b/release/smoke-tests/otel-span-exporter/requirements.txt index 6968658846..f2e5b97c35 100644 --- a/release/smoke-tests/otel-span-exporter/requirements.txt +++ b/release/smoke-tests/otel-span-exporter/requirements.txt @@ -1,17 +1,17 @@ backoff==1.10.0 -certifi==2023.7.22 +certifi==2024.07.04 charset-normalizer==2.0.9 Deprecated==1.2.13 googleapis-common-protos==1.53.0 grpcio==1.53.2 -idna==3.3 -opentelemetry-api==1.7.1 -opentelemetry-exporter-otlp==1.7.1 -opentelemetry-exporter-otlp-proto-grpc==1.7.1 -opentelemetry-exporter-otlp-proto-http==1.7.1 -opentelemetry-proto==1.7.1 -opentelemetry-sdk==1.7.1 -opentelemetry-semantic-conventions==0.26b1 +idna==3.7 +opentelemetry-api==1.25.0 +opentelemetry-exporter-otlp==1.25.0 +opentelemetry-exporter-otlp-proto-grpc==1.25.0 +opentelemetry-exporter-otlp-proto-http==1.25.0 +opentelemetry-proto==1.25.0 +opentelemetry-sdk==1.25.0 +opentelemetry-semantic-conventions==0.46b0 protobuf==3.19.5 requests==2.32.3 six==1.16.0 diff --git a/settings.gradle b/settings.gradle index ca9fcfbdfb..9d84b2ccf0 100644 --- a/settings.gradle +++ b/settings.gradle @@ -74,7 +74,7 @@ dependencyResolutionManagement { } testLibs { version('junit', '5.8.2') - version('mockito', '3.11.2') + version('mockito', '5.12.0') version('hamcrest', '2.2') version('awaitility', '4.2.0') version('spring', '5.3.28')