Merge pull request #5 from Galactus22625/unitTests

saas-crawler unit tests

data-prepper-plugins/saas-source-plugins/saas-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/saas/crawler/base/CrawlerTest.java

@@ -0,0 +1,110 @@
+package org.opensearch.dataprepper.plugins.source.saas.crawler.base;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.opensearch.dataprepper.model.buffer.Buffer;
+import org.opensearch.dataprepper.model.event.Event;
+import org.opensearch.dataprepper.model.record.Record;
+import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator;
+import org.opensearch.dataprepper.plugins.source.saas.crawler.coordination.partition.SaasSourcePartition;
+import org.opensearch.dataprepper.plugins.source.saas.crawler.coordination.state.SaasWorkerProgressState;
+import org.opensearch.dataprepper.plugins.source.saas.crawler.model.ItemInfo;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.internal.verification.VerificationModeFactory.times;
+
+@ExtendWith(MockitoExtension.class)
+public class CrawlerTest {
+    @Mock
+    private SaasSourceConfig sourceConfig;
+
+    @Mock
+    private EnhancedSourceCoordinator coordinator;
+
+    @Mock
+    private Buffer<Record<Event>> buffer;
+
+    @Mock
+    private SaasClient client;
+
+    @Mock
+    private SaasWorkerProgressState state;
+
+    @Mock
+    private ItemInfo item;
+
+
+    private Crawler crawler;
+
+
+    @BeforeEach
+    public void setup() {
+        crawler = new Crawler(client);
+    }
+
+    @Test
+    public void crawlerConstructionTest() {
+        assertNotNull(crawler);
+    }
+
+    @Test
+    public void executePartitionTest() {
+        crawler.executePartition(state, buffer, sourceConfig);
+        verify(client).executePartition(state, buffer, sourceConfig);
+    }
+
+    @Test
+    void testCrawlWithEmptyList(){
+        long lastPollTime = 0;
+        when(client.listItems()).thenReturn(Collections.emptyIterator());
+        crawler.crawl(sourceConfig, lastPollTime, coordinator);
+        verify(coordinator, never()).createPartition(any(SaasSourcePartition.class));
+    }
+
+    @Test
+    void testCrawlWithNonEmptyList() throws NoSuchFieldException, IllegalAccessException {
+        long lastPollTime = 0;
+        List<ItemInfo> itemInfoList = new ArrayList<>();
+        int maxItemsPerPage = getMaxItemsPerPage();
+        for (int i = 0; i < maxItemsPerPage; i++) {
+            itemInfoList.add(item);
+        }
+        when(client.listItems()).thenReturn(itemInfoList.iterator());
+        crawler.crawl(sourceConfig, lastPollTime, coordinator);
+        verify(coordinator, times(1)).createPartition(any(SaasSourcePartition.class));
+
+    }
+
+    @Test
+    void testCrawlWithMultiplePartitions()  throws NoSuchFieldException, IllegalAccessException{
+        long lastPollTime = 0;
+        List<ItemInfo> itemInfoList = new ArrayList<>();
+        int maxItemsPerPage = getMaxItemsPerPage();
+        for (int i = 0; i < maxItemsPerPage + 1; i++) {
+            itemInfoList.add(item);
+        }
+        when(client.listItems()).thenReturn(itemInfoList.iterator());
+        crawler.crawl(sourceConfig, lastPollTime, coordinator);
+        verify(coordinator, times(2)).createPartition(any(SaasSourcePartition.class));
+
+    }
+
+    private int getMaxItemsPerPage() throws NoSuchFieldException, IllegalAccessException {
+    Field maxItemsPerPageField = Crawler.class.getDeclaredField("maxItemsPerPage");
+    maxItemsPerPageField.setAccessible(true);
+    return (int) maxItemsPerPageField.get(null);
+}
+
+}

data-prepper-plugins/saas-source-plugins/saas-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/saas/crawler/base/SaasPluginExecutorServiceProviderTest.java

@@ -0,0 +1,44 @@
+package org.opensearch.dataprepper.plugins.source.saas.crawler.base;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.util.concurrent.ExecutorService;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@ExtendWith(MockitoExtension.class)
+public class SaasPluginExecutorServiceProviderTest {
+
+    private SaasPluginExecutorServiceProvider provider;
+    private ExecutorService executorService;
+
+    @BeforeEach
+    void setUp() {
+        provider = new SaasPluginExecutorServiceProvider();
+        executorService = provider.get();
+    }
+
+    @AfterEach
+    void tearDown() {
+        provider.terminateExecutor();
+    }
+
+    @Test
+    void testConstruction() {
+        assertNotNull(executorService);
+        assertNotNull(provider);
+    }
+
+
+    @Test
+    void testTerminateExecutor() {
+        provider.terminateExecutor();
+        assertTrue(executorService.isShutdown());
+        assertTrue(executorService.isTerminated());
+    }
+}

data-prepper-plugins/saas-source-plugins/saas-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/saas/crawler/base/SaasSourcePluginTest.java

@@ -0,0 +1,124 @@
+package org.opensearch.dataprepper.plugins.source.saas.crawler.base;
+
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.opensearch.dataprepper.metrics.PluginMetrics;
+import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager;
+import org.opensearch.dataprepper.model.buffer.Buffer;
+import org.opensearch.dataprepper.model.codec.ByteDecoder;
+import org.opensearch.dataprepper.model.event.Event;
+import org.opensearch.dataprepper.model.plugin.PluginFactory;
+import org.opensearch.dataprepper.model.record.Record;
+import org.opensearch.dataprepper.model.source.coordinator.SourcePartitionStoreItem;
+import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator;
+import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourcePartition;
+import org.opensearch.dataprepper.plugins.source.saas.crawler.coordination.scheduler.LeaderScheduler;
+
+import java.util.concurrent.ExecutorService;
+import java.util.function.Function;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.internal.verification.VerificationModeFactory.times;
+
+@ExtendWith(MockitoExtension.class)
+public class SaasSourcePluginTest {
+    @Mock
+    private PluginMetrics pluginMetrics;
+
+    @Mock
+    private PluginFactory pluginFactory;
+
+    @Mock
+    private Crawler crawler;
+
+    @Mock
+    private SaasPluginExecutorServiceProvider executorServiceProvider;
+
+    @Mock
+    private ExecutorService executorService;
+
+    @Mock
+    private SaasSourceConfig sourceConfig;
+
+    @Mock
+    private Buffer<Record<Event>> buffer;
+
+    @Mock
+    private AcknowledgementSetManager acknowledgementSetManager;
+
+    @Mock
+    SourcePartitionStoreItem mockItem;
+
+    @Mock
+    EnhancedSourcePartition mockPartition;
+
+    @Mock
+    private EnhancedSourceCoordinator sourceCoordinator;
+
+    private SaasSourcePlugin saasSourcePlugin;
+
+    @BeforeEach
+    void setUp() {
+        when(executorServiceProvider.get()).thenReturn(executorService);
+        saasSourcePlugin = new SaasSourcePlugin(pluginMetrics, sourceConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider);
+    }
+
+    @Test
+    void pluginConstructorTest() {
+        assertNotNull(saasSourcePlugin);
+        verify(executorServiceProvider).get();
+    }
+
+    @Test
+    void testSetEnhancedSourceCoordinator() {
+        saasSourcePlugin.setEnhancedSourceCoordinator(sourceCoordinator);
+        verify(sourceCoordinator).initialize();
+    }
+
+    @Test
+    void startTest() {
+        saasSourcePlugin.setEnhancedSourceCoordinator(sourceCoordinator);
+        saasSourcePlugin.start(buffer);
+        assertFalse(executorService.isShutdown());
+        assertFalse(executorService.isTerminated());
+    }
+
+    @Test
+    void testExecutorServiceSchedulersSubmitted(){
+        saasSourcePlugin.setEnhancedSourceCoordinator(sourceCoordinator);
+        saasSourcePlugin.start(buffer);
+        verify(executorService, times(1)).submit(any(LeaderScheduler.class));
+        verify(executorService, times(SaasSourceConfig.DEFAULT_NUMBER_OF_WORKERS))
+                .submit(any(Thread.class));
+    }
+
+    @Test
+    void testStop() {
+        saasSourcePlugin.stop();
+        verify(executorService).shutdownNow();
+    }
+
+    @Test
+    void testGetPartitionFactory() {
+        Function<SourcePartitionStoreItem, EnhancedSourcePartition> factory = saasSourcePlugin.getPartitionFactory();
+        assertNotNull(factory);
+
+    }
+
+    @Test
+    void testGetDecoder() {
+        ByteDecoder decoder = saasSourcePlugin.getDecoder();
+        assertNull(decoder);
+
+    }
+
+}