diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/README.md b/data-prepper-plugins/saas-source-plugins/jira-source/README.md index f2a1148a2e..a5f5d962c1 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/README.md +++ b/data-prepper-plugins/saas-source-plugins/jira-source/README.md @@ -1,8 +1,9 @@ - # Metrics ### Counter + - `issuesRequested`: measures total number of issue Requests sent. ### Timer + - `requestProcessDuration`: measures latency of requests processed by the jira source plugin. diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/build.gradle b/data-prepper-plugins/saas-source-plugins/jira-source/build.gradle index 9979e0151d..3edfc6c4f7 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/build.gradle +++ b/data-prepper-plugins/saas-source-plugins/jira-source/build.gradle @@ -20,6 +20,8 @@ dependencies { implementation 'org.projectlombok:lombok:1.18.30' annotationProcessor 'org.projectlombok:lombok:1.18.30' + testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.4' + implementation(libs.spring.context) { exclude group: 'commons-logging', module: 'commons-logging' } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraClient.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraClient.java index d7a39a46c1..7e463f4534 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraClient.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraClient.java @@ -1,18 +1,36 @@ package org.opensearch.dataprepper.plugins.source.jira; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventType; +import org.opensearch.dataprepper.model.event.JacksonEvent; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerClient; import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state.SaasWorkerProgressState; import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.inject.Named; +import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static java.util.concurrent.CompletableFuture.supplyAsync; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT; /** * This class represents a Jira client. @@ -21,25 +39,91 @@ public class JiraClient implements CrawlerClient { private static final Logger log = LoggerFactory.getLogger(JiraClient.class); + private final JiraService service; + private final JiraIterator jiraIterator; + private final ExecutorService executorService; + private final CrawlerSourceConfig configuration; + private final int bufferWriteTimeoutInSeconds = 10; + private ObjectMapper objectMapper = new ObjectMapper(); private Instant lastPollTime; - public JiraClient() { + public JiraClient(JiraService service, + JiraIterator jiraIterator, + PluginExecutorServiceProvider executorServiceProvider, + JiraSourceConfig sourceConfig) { + this.service = service; + this.jiraIterator = jiraIterator; + this.executorService = executorServiceProvider.get(); + this.configuration = sourceConfig; } - @Override public Iterator listItems() { - return null; + jiraIterator.initialize(lastPollTime); + return jiraIterator; } @Override public void setLastPollTime(Instant lastPollTime) { - log.info("Setting the lastPollTime: {}", lastPollTime); + log.trace("Setting the lastPollTime: {}", lastPollTime); this.lastPollTime = lastPollTime; } + @VisibleForTesting + void injectObjectMapper(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + @Override - public void executePartition(SaasWorkerProgressState state, Buffer> buffer, CrawlerSourceConfig configuration) { - log.info("Logic for executing the partitions"); + public void executePartition(SaasWorkerProgressState state, + Buffer> buffer, + CrawlerSourceConfig configuration) { + log.trace("Executing the partition: {} with {} ticket(s)", + state.getKeyAttributes(), state.getItemIds().size()); + List itemIds = state.getItemIds(); + Map keyAttributes = state.getKeyAttributes(); + String project = (String) keyAttributes.get(PROJECT); + Instant eventTime = state.getExportStartTime(); + List itemInfos = new ArrayList<>(); + for (String itemId : itemIds) { + if (itemId == null) { + continue; + } + ItemInfo itemInfo = JiraItemInfo.builder() + .withItemId(itemId) + .withId(itemId) + .withProject(project) + .withEventTime(eventTime) + .withMetadata(keyAttributes).build(); + itemInfos.add(itemInfo); + } + + String eventType = EventType.DOCUMENT.toString(); + List> recordsToWrite = itemInfos + .parallelStream() + .map(t -> (Supplier) (() -> service.getIssue(t.getId()))) + .map(supplier -> supplyAsync(supplier, this.executorService)) + .map(CompletableFuture::join) + .map(ticketJson -> { + try { + return objectMapper.readValue(ticketJson, new TypeReference<>() { + }); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }) + .map(t -> (Event) JacksonEvent.builder() + .withEventType(eventType) + .withData(t) + .build()) + .map(event -> new Record<>(event)) + .collect(Collectors.toList()); + + try { + buffer.writeAll(recordsToWrite, (int) Duration.ofSeconds(bufferWriteTimeoutInSeconds).toMillis()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java new file mode 100644 index 0000000000..25bc042b8c --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java @@ -0,0 +1,138 @@ +package org.opensearch.dataprepper.plugins.source.jira; + +import lombok.Getter; +import lombok.Setter; +import org.opensearch.dataprepper.plugins.source.jira.models.IssueBean; +import org.opensearch.dataprepper.plugins.source.jira.utils.Constants; +import org.opensearch.dataprepper.plugins.source.jira.utils.JiraContentType; +import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; + +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import static org.opensearch.dataprepper.plugins.source.jira.JiraService.CONTENT_TYPE; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CREATED; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.ISSUE_KEY; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_KEY; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_NAME; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.UPDATED; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants._ISSUE; + +@Setter +@Getter +public class JiraItemInfo implements ItemInfo { + private String project; + private String issueType; + private String id; + private String itemId; + private Map metadata; + private Instant eventTime; + + public JiraItemInfo(String id, + String itemId, + String project, + String issueType, + Map metadata, + Instant eventTime + ) { + this.id = id; + this.project = project; + this.issueType = issueType; + this.itemId = itemId; + this.metadata = metadata; + this.eventTime = eventTime; + } + + public static JiraItemInfoBuilder builder() { + return new JiraItemInfoBuilder(); + } + + @Override + public String getPartitionKey() { + return project + "|" + issueType + "|" + UUID.randomUUID(); + } + + @Override + public String getId() { + return id; + } + + @Override + public Map getKeyAttributes() { + return Map.of(Constants.PROJECT, project); + } + + @Override + public Instant getLastModifiedAt() { + long updatedAtMillis = Long.parseLong((String) this.metadata.getOrDefault(Constants.UPDATED, "0")); + long createdAtMillis = Long.parseLong((String) this.metadata.getOrDefault(Constants.CREATED, "0")); + return createdAtMillis > updatedAtMillis ? + Instant.ofEpochMilli(createdAtMillis) : Instant.ofEpochMilli(updatedAtMillis); + } + + public static class JiraItemInfoBuilder { + private Map metadata; + private Instant eventTime; + private String id; + private String itemId; + private String project; + private String issueType; + + public JiraItemInfoBuilder() { + } + + public JiraItemInfo build() { + return new JiraItemInfo(id, itemId, project, issueType, metadata, eventTime); + } + + public JiraItemInfoBuilder withMetadata(Map metadata) { + this.metadata = metadata; + return this; + } + + public JiraItemInfoBuilder withEventTime(Instant eventTime) { + this.eventTime = eventTime; + return this; + } + + public JiraItemInfoBuilder withItemId(String itemId) { + this.itemId = itemId; + return this; + } + + public JiraItemInfoBuilder withId(String id) { + this.id = id; + return this; + } + + public JiraItemInfoBuilder withProject(String project) { + this.project = project; + return this; + } + + public JiraItemInfoBuilder withIssueType(String issueType) { + this.issueType = issueType; + return this; + } + + public JiraItemInfoBuilder withIssueBean(IssueBean issue) { + Map issueMetadata = new HashMap<>(); + issueMetadata.put(PROJECT_KEY, issue.getProject()); + issueMetadata.put(PROJECT_NAME, issue.getProjectName()); + issueMetadata.put(CREATED, issue.getCreatedTimeMillis()); + issueMetadata.put(UPDATED, issue.getUpdatedTimeMillis()); + issueMetadata.put(ISSUE_KEY, issue.getKey()); + issueMetadata.put(CONTENT_TYPE, JiraContentType.ISSUE.getType()); + + this.project = issue.getProject(); + this.id = issue.getKey(); + this.issueType = JiraContentType.ISSUE.getType(); + this.itemId = _ISSUE + issueMetadata.get(PROJECT_KEY) + "-" + issue.getKey(); + this.metadata = issueMetadata; + return this; + } + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraIterator.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraIterator.java new file mode 100644 index 0000000000..1e033581b4 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraIterator.java @@ -0,0 +1,64 @@ +package org.opensearch.dataprepper.plugins.source.jira; + + +import lombok.Setter; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; +import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Named; +import java.time.Instant; +import java.util.Iterator; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; + +@Named +public class JiraIterator implements Iterator { + + private static final Logger log = LoggerFactory.getLogger(JiraIterator.class); + private final JiraSourceConfig sourceConfig; + private final JiraService service; + private final ExecutorService crawlerTaskExecutor; + @Setter + private long crawlerQWaitTimeMillis = 2000; + private Queue itemInfoQueue; + private Instant lastPollTime; + private boolean firstTime = true; + + public JiraIterator(final JiraService service, + PluginExecutorServiceProvider executorServiceProvider, + JiraSourceConfig sourceConfig) { + this.service = service; + this.crawlerTaskExecutor = executorServiceProvider.get(); + this.sourceConfig = sourceConfig; + } + + @Override + public boolean hasNext() { + if (firstTime) { + log.info("Crawling has been started"); + itemInfoQueue = service.getJiraEntities(sourceConfig, lastPollTime); + firstTime = false; + } + return !this.itemInfoQueue.isEmpty(); + } + + @Override + public ItemInfo next() { + return this.itemInfoQueue.remove(); + } + + /** + * Initialize. + * + * @param jiraChangeLogToken the jira change log token + */ + public void initialize(Instant jiraChangeLogToken) { + this.itemInfoQueue = new ConcurrentLinkedQueue<>(); + this.lastPollTime = jiraChangeLogToken; + this.firstTime = true; + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java new file mode 100644 index 0000000000..1eaf67e1df --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java @@ -0,0 +1,213 @@ +package org.opensearch.dataprepper.plugins.source.jira; + +import io.micrometer.core.instrument.Counter; +import lombok.extern.slf4j.Slf4j; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.plugins.source.jira.exception.BadRequestException; +import org.opensearch.dataprepper.plugins.source.jira.models.IssueBean; +import org.opensearch.dataprepper.plugins.source.jira.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.jira.rest.JiraRestClient; +import org.opensearch.dataprepper.plugins.source.jira.utils.JiraConfigHelper; +import org.opensearch.dataprepper.plugins.source.jira.utils.JiraContentType; +import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; +import org.springframework.util.CollectionUtils; + +import javax.inject.Named; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.ISSUE_KEY; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.KEY; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.LIVE; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_KEY; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.UPDATED; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants._PROJECT; +import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.CLOSING_ROUND_BRACKET; +import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.DELIMITER; +import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.GREATER_THAN_EQUALS; +import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.ISSUE_TYPE_IN; +import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.PREFIX; +import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.PROJECT_IN; +import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.STATUS_IN; +import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.SUFFIX; + + +/** + * Service class for interactive external Atlassian jira SaaS service and fetch required details using their rest apis. + */ + +@Slf4j +@Named +public class JiraService { + + + public static final String CONTENT_TYPE = "ContentType"; + private static final String SEARCH_RESULTS_FOUND = "searchResultsFound"; + private static final Map jiraProjectCache = new ConcurrentHashMap<>(); + + private final JiraSourceConfig jiraSourceConfig; + private final JiraRestClient jiraRestClient; + private final Counter searchResultsFoundCounter; + private final PluginMetrics jiraPluginMetrics = PluginMetrics.fromNames("jiraService", "aws"); + + + public JiraService(JiraSourceConfig jiraSourceConfig, JiraRestClient jiraRestClient) { + this.jiraSourceConfig = jiraSourceConfig; + this.jiraRestClient = jiraRestClient; + this.searchResultsFoundCounter = jiraPluginMetrics.counter(SEARCH_RESULTS_FOUND); + } + + /** + * Get jira entities. + * + * @param configuration the configuration. + * @param timestamp timestamp. + */ + public Queue getJiraEntities(JiraSourceConfig configuration, Instant timestamp) { + log.info("Started to fetch entities"); + Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); + jiraProjectCache.clear(); + searchForNewTicketsAndAddToQueue(configuration, timestamp, itemInfoQueue); + log.trace("Creating item information and adding in queue"); + jiraProjectCache.keySet().forEach(key -> { + Map metadata = new HashMap<>(); + metadata.put(CONTENT_TYPE, JiraContentType.PROJECT.getType()); + ItemInfo itemInfo = createItemInfo(_PROJECT + key, metadata); + itemInfoQueue.add(itemInfo); + }); + return itemInfoQueue; + } + + public String getIssue(String issueKey) { + return jiraRestClient.getIssue(issueKey); + } + + /** + * Method for building Issue Item Info. + * + * @param configuration Input Parameter + * @param timestamp Input Parameter + */ + private void searchForNewTicketsAndAddToQueue(JiraSourceConfig configuration, Instant timestamp, + Queue itemInfoQueue) { + log.trace("Looking for Add/Modified tickets with a Search API call"); + StringBuilder jql = createIssueFilterCriteria(configuration, timestamp); + int total; + int startAt = 0; + do { + SearchResults searchIssues = jiraRestClient.getAllIssues(jql, startAt, configuration); + List issueList = new ArrayList<>(searchIssues.getIssues()); + total = searchIssues.getTotal(); + startAt += searchIssues.getIssues().size(); + addItemsToQueue(issueList, itemInfoQueue); + } while (startAt < total); + searchResultsFoundCounter.increment(total); + log.info("Number of tickets found in search api call: {}", total); + } + + /** + * Add items to queue. + * + * @param issueList Issue list. + * @param itemInfoQueue Item info queue. + */ + private void addItemsToQueue(List issueList, Queue itemInfoQueue) { + issueList.forEach(issue -> { + itemInfoQueue.add(JiraItemInfo.builder().withEventTime(Instant.now()).withIssueBean(issue).build()); + + if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(KEY))) { + String projectKey = ((Map) issue.getFields().get(PROJECT)).get(KEY).toString(); + if (!jiraProjectCache.containsKey(projectKey)) { + jiraProjectCache.put(projectKey, LIVE); + } + } + }); + } + + + /** + * Method for creating Issue Filter Criteria. + * + * @param configuration Input Parameter + * @param ts Input Parameter + * @return String Builder + */ + private StringBuilder createIssueFilterCriteria(JiraSourceConfig configuration, Instant ts) { + + log.info("Creating issue filter criteria"); + if (!CollectionUtils.isEmpty(JiraConfigHelper.getProjectKeyFilter(configuration))) { + validateProjectFilters(configuration); + } + StringBuilder jiraQl = new StringBuilder(UPDATED + GREATER_THAN_EQUALS + ts.toEpochMilli()); + if (!CollectionUtils.isEmpty(JiraConfigHelper.getProjectKeyFilter(configuration))) { + jiraQl.append(PROJECT_IN).append(JiraConfigHelper.getProjectKeyFilter(configuration).stream() + .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) + .append(CLOSING_ROUND_BRACKET); + } + if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueTypeFilter(configuration))) { + jiraQl.append(ISSUE_TYPE_IN).append(JiraConfigHelper.getIssueTypeFilter(configuration).stream() + .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) + .append(CLOSING_ROUND_BRACKET); + } + if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueStatusFilter(configuration))) { + jiraQl.append(STATUS_IN).append(JiraConfigHelper.getIssueStatusFilter(configuration).stream() + .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) + .append(CLOSING_ROUND_BRACKET); + } + log.info("Created issue filter criteria JiraQl query: {}", jiraQl); + return jiraQl; + } + + /** + * Method for Validating Project Filters. + * + * @param configuration Input Parameter + */ + private void validateProjectFilters(JiraSourceConfig configuration) { + log.info("Validating project filters"); + List badFilters = new ArrayList<>(); + Pattern regex = Pattern.compile("[^A-Z0-9]"); + JiraConfigHelper.getProjectKeyFilter(configuration).forEach(projectFilter -> { + Matcher matcher = regex.matcher(projectFilter); + if (matcher.find() || projectFilter.length() <= 1 || projectFilter.length() > 10) { + badFilters.add(projectFilter); + } + }); + if (!badFilters.isEmpty()) { + String filters = String.join("\"" + badFilters + "\"", ", "); + log.error("One or more invalid project keys found in filter configuration: {}", badFilters); + throw new BadRequestException("Bad request exception occurred " + + "Invalid project key found in filter configuration for " + + filters); + } + } + + /** + * Method for creating Item Info. + * + * @param key Input Parameter + * @param metadata Input Parameter + * @return Item Info + */ + private ItemInfo createItemInfo(String key, Map metadata) { + return JiraItemInfo.builder().withEventTime(Instant.now()) + .withId((String) metadata.get(ISSUE_KEY)) + .withItemId(key) + .withMetadata(metadata) + .withProject((String) metadata.get(PROJECT_KEY)) + .withIssueType((String) metadata.get(CONTENT_TYPE)) + .build(); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSource.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSource.java index be13c54755..2641ab60ad 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSource.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSource.java @@ -6,52 +6,63 @@ import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.buffer.Buffer; -import org.opensearch.dataprepper.model.codec.ByteDecoder; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.source.Source; +import org.opensearch.dataprepper.plugins.source.jira.rest.auth.JiraAuthConfig; +import org.opensearch.dataprepper.plugins.source.jira.utils.JiraConfigHelper; import org.opensearch.dataprepper.plugins.source.source_crawler.CrawlerApplicationContextMarker; import org.opensearch.dataprepper.plugins.source.source_crawler.base.Crawler; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourcePlugin; import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PLUGIN_NAME; + /** * JiraConnector connector entry point. */ -@DataPrepperPlugin(name = "jira", +@DataPrepperPlugin(name = PLUGIN_NAME, pluginType = Source.class, + pluginConfigurationType = JiraSourceConfig.class, packagesToScan = {CrawlerApplicationContextMarker.class, JiraSource.class} ) -public class JiraSource implements Source> { +public class JiraSource extends CrawlerSourcePlugin { private static final Logger log = LoggerFactory.getLogger(JiraSource.class); - + private final JiraSourceConfig jiraSourceConfig; + private final JiraAuthConfig jiraOauthConfig; @DataPrepperPluginConstructor public JiraSource(final PluginMetrics pluginMetrics, + final JiraSourceConfig jiraSourceConfig, + final JiraAuthConfig jiraOauthConfig, final PluginFactory pluginFactory, final AcknowledgementSetManager acknowledgementSetManager, Crawler crawler, PluginExecutorServiceProvider executorServiceProvider) { - log.info("Create Jira Source Connector"); + super(PLUGIN_NAME, pluginMetrics, jiraSourceConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + log.info("Creating Jira Source Plugin"); + this.jiraSourceConfig = jiraSourceConfig; + this.jiraOauthConfig = jiraOauthConfig; } + @Override public void start(Buffer> buffer) { log.info("Starting Jira Source Plugin... "); + JiraConfigHelper.validateConfig(jiraSourceConfig); + jiraOauthConfig.initCredentials(); + super.start(buffer); } @Override public void stop() { - - } - - @Override - public ByteDecoder getDecoder() { - return Source.super.getDecoder(); + log.info("Stopping Jira Source Plugin"); + super.stop(); } } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfig.java new file mode 100644 index 0000000000..f37434cf9d --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfig.java @@ -0,0 +1,120 @@ +package org.opensearch.dataprepper.plugins.source.jira; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.Size; +import lombok.Getter; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2; + +@Getter +public class JiraSourceConfig implements CrawlerSourceConfig { + + private static final Duration DEFAULT_BACKOFF_MILLIS = Duration.ofMinutes(2); + + /** + * Jira account url + */ + @JsonProperty("account_url") + private String accountUrl; + + /** + * A map of connector credentials specific to this source + */ + @JsonProperty("connector_credentials") + private Map connectorCredentials; + + /** + * List of projects to ingest + */ + @JsonProperty("projects") + @Size(max = 1000, message = "Project type filter should not be more than 1000") + private List project = new ArrayList<>(); + + /** + * List of specific issue types to ingest. + * Ex: Story, Epic, Task etc + */ + @JsonProperty("issue_types") + @Size(max = 1000, message = "Issue type filter should be less than 1000") + private List issueType = new ArrayList<>(); + + /** + * Optional Inclusion patterns for filtering some tickets + */ + @JsonProperty("inclusion_patterns") + @Size(max = 100, message = "inclusion pattern filters should not be more than 1000") + private List inclusionPatterns; + + /** + * Optional Exclusion patterns for excluding some tickets + */ + @JsonProperty("exclusion_patterns") + @Size(max = 1000, message = "exclusion pattern filter should be less than 1000") + private List exclusionPatterns; + + /** + * Optional Status filter to ingest the tickets + */ + @JsonProperty("statuses") + @Size(max = 1000, message = "Status filter should be less than 1000") + private List status = new ArrayList<>(); + + /** + * Number of worker threads to spawn to parallel source fetching + */ + @JsonProperty("workers") + private int numWorkers = DEFAULT_NUMBER_OF_WORKERS; + + /** + * Default time to wait (with exponential backOff) in the case of + * waiting for the source service to respond + */ + @JsonProperty("backoff_time") + private Duration backOff = DEFAULT_BACKOFF_MILLIS; + + public String getJiraId() { + return this.getConnectorCredentials().get("jira_id"); + } + + public String getJiraCredential() { + return this.getConnectorCredentials().get("jira_credential"); + } + + public String getAuthType() { + return this.getConnectorCredentials().get("auth_type"); + } + + public String getAccessToken() { + return fetchGivenOAuthAttribute("access_token"); + } + + public String getRefreshToken() { + return fetchGivenOAuthAttribute("refresh_token"); + } + + public String getClientId() { + return fetchGivenOAuthAttribute("client_id"); + } + + public String getClientSecret() { + return fetchGivenOAuthAttribute("client_secret"); + } + + private String fetchGivenOAuthAttribute(String givenAttribute) { + if (!OAUTH2.equals(getAuthType())) { + throw new RuntimeException("Authentication Type is not OAuth2."); + } + String attributeValue = this.getConnectorCredentials().get(givenAttribute); + if (attributeValue == null || attributeValue.isEmpty()) { + throw new RuntimeException(String.format("%s is required for OAuth2 AuthType", givenAttribute)); + } + return attributeValue; + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/exception/BadRequestException.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/exception/BadRequestException.java new file mode 100644 index 0000000000..589e8626ab --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/exception/BadRequestException.java @@ -0,0 +1,15 @@ +package org.opensearch.dataprepper.plugins.source.jira.exception; + +/** + * Exception to indicate a bad REST call has been made. + * It could either be caused by bad user inputs or wrong url construction in the logic. + */ +public final class BadRequestException extends RuntimeException { + public BadRequestException(final String message, final Throwable throwable) { + super(message, throwable); + } + + public BadRequestException(final String message) { + super(message); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/exception/UnAuthorizedException.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/exception/UnAuthorizedException.java new file mode 100644 index 0000000000..1efa30e032 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/exception/UnAuthorizedException.java @@ -0,0 +1,15 @@ +package org.opensearch.dataprepper.plugins.source.jira.exception; + +/** + * Exception to indicate unauthorized access. + * It could either be caused by invalid credentials supplied by the user or failed renew the credentials. + */ +public final class UnAuthorizedException extends RuntimeException { + public UnAuthorizedException(final String message, final Throwable throwable) { + super(message, throwable); + } + + public UnAuthorizedException(final String message) { + super(message); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/models/IssueBean.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/models/IssueBean.java new file mode 100644 index 0000000000..ec437ac25d --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/models/IssueBean.java @@ -0,0 +1,124 @@ +package org.opensearch.dataprepper.plugins.source.jira.models; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import lombok.Setter; + +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Date; +import java.util.Map; +import java.util.Objects; +import java.util.regex.Pattern; + +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CREATED; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.KEY; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.NAME; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.UPDATED; + + +public class IssueBean { + + @JsonIgnore + private final Pattern JiraDateTimePattern = Pattern.compile( + "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}[-+]\\d{4}$"); + @JsonIgnore + private final DateTimeFormatter offsetDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); + + /** + * Expand options that include additional issue details in the response. + */ + @Getter + @Setter + @JsonProperty("expand") + private String expand = null; + + /** + * The ID of the issue. + */ + @Getter + @Setter + @JsonProperty("id") + private String id = null; + + /** + * The URL of the issue details. + */ + @Getter + @Setter + @JsonProperty("self") + private String self = null; + + /** + * The key of the issue. + */ + @Getter + @Setter + @JsonProperty("key") + private String key = null; + + @Getter + @Setter + @JsonProperty("renderedFields") + private Map renderedFields = null; + + @Getter + @Setter + @JsonProperty("properties") + private Map properties = null; + + @Getter + @Setter + @JsonProperty("names") + private Map names = null; + + @Getter + @Setter + @JsonProperty("fields") + private Map fields = null; + + + @JsonIgnore + public String getProject() { + if (fields != null && Objects.nonNull(((Map) fields.get(PROJECT)).get(KEY))) { + return ((Map) fields.get(PROJECT)).get(KEY).toString(); + } + return null; + } + + @JsonIgnore + public String getProjectName() { + if (fields != null && Objects.nonNull(((Map) fields.get(PROJECT)).get(NAME))) { + return ((Map) fields.get(PROJECT)).get(NAME).toString(); + } + return null; + } + + @JsonIgnore + public long getCreatedTimeMillis() { + return getGivenDateField(CREATED); + } + + @JsonIgnore + public long getUpdatedTimeMillis() { + return getGivenDateField(UPDATED); + } + + @JsonIgnore + private long getGivenDateField(String dateTimeFieldToPull) { + long dateTimeField = 0; + + if (fields != null && Objects.nonNull(fields.get(dateTimeFieldToPull)) && JiraDateTimePattern.matcher(fields.get(dateTimeFieldToPull) + .toString()).matches()) { + String charSequence = fields.get(dateTimeFieldToPull).toString(); + OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence, offsetDateTimeFormatter); + new Date(offsetDateTime.toInstant().toEpochMilli()); + dateTimeField = offsetDateTime.toEpochSecond() * 1000; + } + return dateTimeField; + } + + +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResults.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResults.java new file mode 100644 index 0000000000..96bc445cdb --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResults.java @@ -0,0 +1,28 @@ +package org.opensearch.dataprepper.plugins.source.jira.models; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; + +import java.util.List; + +/** + * The result of a JQL search. + */ +@Getter +public class SearchResults { + @JsonProperty("expand") + private String expand = null; + + @JsonProperty("startAt") + private Integer startAt = null; + + @JsonProperty("maxResults") + private Integer maxResults = null; + + @JsonProperty("total") + private Integer total = null; + + @JsonProperty("issues") + private List issues = null; + +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/BasicAuthInterceptor.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/BasicAuthInterceptor.java new file mode 100644 index 0000000000..57dbb65f94 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/BasicAuthInterceptor.java @@ -0,0 +1,32 @@ +package org.opensearch.dataprepper.plugins.source.jira.rest; + +import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpRequest; +import org.springframework.http.client.ClientHttpRequestExecution; +import org.springframework.http.client.ClientHttpRequestInterceptor; +import org.springframework.http.client.ClientHttpResponse; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + + +public class BasicAuthInterceptor implements ClientHttpRequestInterceptor { + private final String username; + private final String password; + + public BasicAuthInterceptor(JiraSourceConfig config) { + this.username = config.getJiraId(); + this.password = config.getJiraCredential(); + } + + @Override + public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException { + String auth = username + ":" + password; + byte[] encodedAuth = Base64.getEncoder().encode(auth.getBytes(StandardCharsets.US_ASCII)); + String authHeader = "Basic " + new String(encodedAuth); + request.getHeaders().set(HttpHeaders.AUTHORIZATION, authHeader); + return execution.execute(request, body); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/CustomRestTemplateConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/CustomRestTemplateConfig.java new file mode 100644 index 0000000000..231351bcee --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/CustomRestTemplateConfig.java @@ -0,0 +1,32 @@ +package org.opensearch.dataprepper.plugins.source.jira.rest; + + +import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; +import org.opensearch.dataprepper.plugins.source.jira.rest.auth.JiraAuthConfig; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.client.ClientHttpRequestInterceptor; +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.web.client.RestTemplate; + +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2; + +@Configuration +public class CustomRestTemplateConfig { + + @Bean + public RestTemplate basicAuthRestTemplate(JiraSourceConfig config, JiraAuthConfig authConfig) { + RestTemplate restTemplate = new RestTemplate(); + restTemplate.setRequestFactory(new HttpComponentsClientHttpRequestFactory()); + ClientHttpRequestInterceptor httpInterceptor; + if (OAUTH2.equals(config.getAuthType())) { + httpInterceptor = new OAuth2RequestInterceptor(authConfig); + } else { + httpInterceptor = new BasicAuthInterceptor(config); + } + restTemplate.getInterceptors().add(httpInterceptor); + return restTemplate; + } + + +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java new file mode 100644 index 0000000000..d87985c6fd --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java @@ -0,0 +1,135 @@ +package org.opensearch.dataprepper.plugins.source.jira.rest; + +import com.google.common.annotations.VisibleForTesting; +import io.micrometer.core.annotation.Timed; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Timer; +import lombok.extern.slf4j.Slf4j; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; +import org.opensearch.dataprepper.plugins.source.jira.exception.UnAuthorizedException; +import org.opensearch.dataprepper.plugins.source.jira.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.jira.rest.auth.JiraAuthConfig; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +import javax.inject.Named; +import java.net.URI; +import java.util.List; + +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.RETRY_ATTEMPT; +import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.EXPAND_FIELD; +import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.EXPAND_VALUE; +import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.JQL_FIELD; + +@Slf4j +@Named +public class JiraRestClient { + + public static final String REST_API_SEARCH = "rest/api/3/search"; + public static final String REST_API_FETCH_ISSUE = "rest/api/3/issue"; + public static final String FIFTY = "50"; + public static final String START_AT = "startAt"; + public static final String MAX_RESULT = "maxResults"; + public static final List RETRY_ATTEMPT_SLEEP_TIME = List.of(1, 2, 5, 10, 20, 40); + private static final String TICKET_FETCH_LATENCY_TIMER = "ticketFetchLatency"; + private static final String SEARCH_CALL_LATENCY_TIMER = "searchCallLatency"; + private static final String ISSUES_REQUESTED = "issuesRequested"; + private final RestTemplate restTemplate; + private final JiraAuthConfig authConfig; + private final Timer ticketFetchLatencyTimer; + private final Timer searchCallLatencyTimer; + private final Counter issuesRequestedCounter; + private final PluginMetrics jiraPluginMetrics = PluginMetrics.fromNames("jiraRestClient", "aws"); + private int sleepTimeMultiplier = 1000; + + public JiraRestClient(RestTemplate restTemplate, JiraAuthConfig authConfig) { + this.restTemplate = restTemplate; + this.authConfig = authConfig; + + ticketFetchLatencyTimer = jiraPluginMetrics.timer(TICKET_FETCH_LATENCY_TIMER); + searchCallLatencyTimer = jiraPluginMetrics.timer(SEARCH_CALL_LATENCY_TIMER); + issuesRequestedCounter = jiraPluginMetrics.counter(ISSUES_REQUESTED); + } + + /** + * Method to get Issues. + * + * @param jql input parameter. + * @param startAt the start at + * @param configuration input parameter. + * @return InputStream input stream + */ + @Timed(SEARCH_CALL_LATENCY_TIMER) + public SearchResults getAllIssues(StringBuilder jql, int startAt, + JiraSourceConfig configuration) { + + String url = configuration.getAccountUrl() + REST_API_SEARCH; + if (configuration.getAuthType().equals(OAUTH2)) { + url = authConfig.getUrl() + REST_API_SEARCH; + } + + URI uri = UriComponentsBuilder.fromHttpUrl(url) + .queryParam(MAX_RESULT, FIFTY) + .queryParam(START_AT, startAt) + .queryParam(JQL_FIELD, jql) + .queryParam(EXPAND_FIELD, EXPAND_VALUE) + .buildAndExpand().toUri(); + return invokeRestApi(uri, SearchResults.class).getBody(); + } + + /** + * Gets issue. + * + * @param issueKey the item info + * @return the issue + */ + @Timed(TICKET_FETCH_LATENCY_TIMER) + public String getIssue(String issueKey) { + issuesRequestedCounter.increment(); + String url = authConfig.getUrl() + REST_API_FETCH_ISSUE + "/" + issueKey; + URI uri = UriComponentsBuilder.fromHttpUrl(url).buildAndExpand().toUri(); + return invokeRestApi(uri, String.class).getBody(); + } + + private ResponseEntity invokeRestApi(URI uri, Class responseType) { + + int retryCount = 0; + while (retryCount < RETRY_ATTEMPT) { + try { + return restTemplate.getForEntity(uri, responseType); + } catch (HttpClientErrorException ex) { + HttpStatus statusCode = ex.getStatusCode(); + String statusMessage = ex.getMessage(); + log.error("An exception has occurred while getting response from Jira search API {}", ex.getMessage(), ex); + if (statusCode == HttpStatus.FORBIDDEN) { + throw new UnAuthorizedException(statusMessage); + } else if (statusCode == HttpStatus.UNAUTHORIZED) { + log.error(NOISY, "Token expired. We will try to renew the tokens now", ex); + authConfig.renewCredentials(); + } else if (statusCode == HttpStatus.TOO_MANY_REQUESTS) { + log.error(NOISY, "Hitting API rate limit. Backing off with sleep timer.", ex); + } + try { + Thread.sleep((long) RETRY_ATTEMPT_SLEEP_TIME.get(retryCount) * sleepTimeMultiplier); + } catch (InterruptedException e) { + throw new RuntimeException("Sleep in the retry attempt got interrupted", e); + } + } + retryCount++; + } + String errorMessage = String.format("Exceeded max retry attempts. Failed to execute the Rest API call %s", uri.toString()); + log.error(errorMessage); + throw new RuntimeException(errorMessage); + } + + @VisibleForTesting + public void setSleepTimeMultiplier(int multiplier) { + sleepTimeMultiplier = multiplier; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/OAuth2RequestInterceptor.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/OAuth2RequestInterceptor.java new file mode 100644 index 0000000000..bf748ceb26 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/OAuth2RequestInterceptor.java @@ -0,0 +1,26 @@ +package org.opensearch.dataprepper.plugins.source.jira.rest; + +import org.opensearch.dataprepper.plugins.source.jira.rest.auth.JiraAuthConfig; +import org.opensearch.dataprepper.plugins.source.jira.rest.auth.JiraOauthConfig; +import org.springframework.http.HttpRequest; +import org.springframework.http.client.ClientHttpRequestExecution; +import org.springframework.http.client.ClientHttpRequestInterceptor; +import org.springframework.http.client.ClientHttpResponse; + +import java.io.IOException; + +public class OAuth2RequestInterceptor implements ClientHttpRequestInterceptor { + + private final JiraAuthConfig config; + + public OAuth2RequestInterceptor(JiraAuthConfig config) { + this.config = config; + } + + @Override + public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException { + request.getHeaders().setBearerAuth(((JiraOauthConfig) config).getAccessToken()); + return execution.execute(request, body); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthConfig.java new file mode 100644 index 0000000000..274ce8b5d7 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthConfig.java @@ -0,0 +1,24 @@ +package org.opensearch.dataprepper.plugins.source.jira.rest.auth; + +/** + * The interface that defines the behaviour for Jira auth configs. + */ +public interface JiraAuthConfig { + + /** + * Returns the URL for the Jira instance. + * + * @return the URL for the Jira instance. + */ + String getUrl(); + + /** + * Initializes the credentials for the Jira instance. + */ + void initCredentials(); + + /** + * Renews the credentials for the Jira instance. + */ + void renewCredentials(); +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthFactory.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthFactory.java new file mode 100644 index 0000000000..1ba1d9717f --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthFactory.java @@ -0,0 +1,31 @@ +package org.opensearch.dataprepper.plugins.source.jira.rest.auth; + +import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; +import org.springframework.beans.factory.FactoryBean; +import org.springframework.context.annotation.Configuration; + +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2; + +@Configuration +public class JiraAuthFactory implements FactoryBean { + + private final JiraSourceConfig sourceConfig; + + public JiraAuthFactory(JiraSourceConfig sourceConfig) { + this.sourceConfig = sourceConfig; + } + + @Override + public JiraAuthConfig getObject() { + String authType = sourceConfig.getAuthType(); + if (OAUTH2.equals(authType)) { + return new JiraOauthConfig(sourceConfig); + } + return new JiraBasicAuthConfig(sourceConfig); + } + + @Override + public Class getObjectType() { + return JiraAuthConfig.class; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraBasicAuthConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraBasicAuthConfig.java new file mode 100644 index 0000000000..751f56f6d1 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraBasicAuthConfig.java @@ -0,0 +1,30 @@ +package org.opensearch.dataprepper.plugins.source.jira.rest.auth; + + +import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; + +public class JiraBasicAuthConfig implements JiraAuthConfig { + + private final JiraSourceConfig jiraSourceConfig; + + public JiraBasicAuthConfig(JiraSourceConfig jiraSourceConfig) { + this.jiraSourceConfig = jiraSourceConfig; + } + + @Override + public String getUrl() { + return jiraSourceConfig.getAccountUrl(); + } + + @Override + public void initCredentials() { + //do nothing for basic authentication + } + + @Override + public void renewCredentials() { + //do nothing for basic authentication + } + + +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java new file mode 100644 index 0000000000..f7f4e8493f --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java @@ -0,0 +1,155 @@ +package org.opensearch.dataprepper.plugins.source.jira.rest.auth; + +import lombok.Getter; +import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; +import org.opensearch.dataprepper.plugins.source.jira.exception.UnAuthorizedException; +import org.slf4j.Logger; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.util.StringUtils; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; + +import java.time.Instant; +import java.util.List; +import java.util.Map; + +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.RETRY_ATTEMPT; +import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.SLASH; + +/** + * The type Jira service. + */ + +public class JiraOauthConfig implements JiraAuthConfig { + + public static final String OAuth2_URL = "https://api.atlassian.com/ex/jira/"; + public static final String ACCESSIBLE_RESOURCES = "https://api.atlassian.com/oauth/token/accessible-resources"; + public static final String TOKEN_LOCATION = "https://auth.atlassian.com/oauth/token"; + + public static final String EXPIRES_IN = "expires_in"; + public static final String REFRESH_TOKEN = "refresh_token"; + public static final String ACCESS_TOKEN = "access_token"; + private static final Logger log = + org.slf4j.LoggerFactory.getLogger(JiraOauthConfig.class); + private final String clientId; + private final String clientSecret; + private final JiraSourceConfig jiraSourceConfig; + private final Object cloudIdFetchLock = new Object(); + private final Object tokenRenewLock = new Object(); + RestTemplate restTemplate = new RestTemplate(); + private String url; + @Getter + private int expiresInSeconds = 0; + @Getter + private Instant expireTime = Instant.ofEpochMilli(0); + @Getter + private String accessToken; + @Getter + private String refreshToken; + private String cloudId = null; + + public JiraOauthConfig(JiraSourceConfig jiraSourceConfig) { + this.jiraSourceConfig = jiraSourceConfig; + this.accessToken = jiraSourceConfig.getAccessToken(); + this.refreshToken = jiraSourceConfig.getRefreshToken(); + this.clientId = jiraSourceConfig.getClientId(); + this.clientSecret = jiraSourceConfig.getClientSecret(); + } + + public String getJiraAccountCloudId() { + log.info("Getting Jira Account Cloud ID"); + synchronized (cloudIdFetchLock) { + if (this.cloudId != null) { + //Someone else must have initialized it + return this.cloudId; + } + + int retryCount = 0; + while (retryCount < RETRY_ATTEMPT) { + retryCount++; + try { + HttpHeaders headers = new HttpHeaders(); + headers.setBearerAuth(accessToken); + headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); + HttpEntity entity = new HttpEntity<>(headers); + ResponseEntity exchangeResponse = + restTemplate.exchange(ACCESSIBLE_RESOURCES, HttpMethod.GET, entity, Object.class); + List> listResponse = (List>) exchangeResponse.getBody(); + Map response = listResponse.get(0); + this.cloudId = (String) response.get("id"); + return this.cloudId; + } catch (HttpClientErrorException e) { + if (e.getRawStatusCode() == HttpStatus.UNAUTHORIZED.value()) { + renewCredentials(); + } + log.error("Error occurred while accessing resources: ", e); + } + } + throw new UnAuthorizedException(String.format("Access token expired. Unable to renew even after %s attempts", RETRY_ATTEMPT)); + } + } + + public void renewCredentials() { + Instant currentTime = Instant.now(); + if (expireTime.isAfter(currentTime)) { + //There is still time to renew or someone else must have already renewed it + return; + } + + synchronized (tokenRenewLock) { + if (expireTime.isAfter(currentTime)) { + //Someone else must have already renewed it + return; + } + + log.info("Renewing access-refresh token pair for Jira Connector."); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + String payloadTemplate = "{\"grant_type\": \"%s\", \"client_id\": \"%s\", \"client_secret\": \"%s\", \"refresh_token\": \"%s\"}"; + String payload = String.format(payloadTemplate, "refresh_token", clientId, clientSecret, refreshToken); + HttpEntity entity = new HttpEntity<>(payload, headers); + + try { + ResponseEntity responseEntity = restTemplate.postForEntity(TOKEN_LOCATION, entity, Map.class); + Map oauthClientResponse = responseEntity.getBody(); + this.accessToken = (String) oauthClientResponse.get(ACCESS_TOKEN); + this.refreshToken = (String) oauthClientResponse.get(REFRESH_TOKEN); + this.expiresInSeconds = (int) oauthClientResponse.get(EXPIRES_IN); + this.expireTime = Instant.ofEpochMilli(System.currentTimeMillis() + (expiresInSeconds * 1000L)); + } catch (HttpClientErrorException ex) { + this.expireTime = Instant.ofEpochMilli(0); + this.expiresInSeconds = 0; + log.error("Failed to renew access token. Status code: {}, Error Message: {}", + ex.getRawStatusCode(), ex.getMessage()); + throw new RuntimeException("Failed to renew access token" + ex.getMessage(), ex); + } + } + } + + @Override + public String getUrl() { + if (!StringUtils.hasLength(url)) { + synchronized (cloudIdFetchLock) { + if (!StringUtils.hasLength(url)) { + initCredentials(); + } + } + } + return url; + } + + /** + * Method for getting Jira url based on auth type. + */ + @Override + public void initCredentials() { + //For OAuth based flow, we use a different Jira url + this.cloudId = getJiraAccountCloudId(); + this.url = OAuth2_URL + this.cloudId + SLASH; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidation.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidation.java new file mode 100644 index 0000000000..0fc1c379f5 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidation.java @@ -0,0 +1,50 @@ +package org.opensearch.dataprepper.plugins.source.jira.utils; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.opensearch.dataprepper.plugins.source.jira.exception.BadRequestException; + +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.UnknownHostException; + + +/** + * This is the AddressValidation Class. + */ + +@Slf4j +public class AddressValidation { + + public static final String INVALID_URL = "URL is not valid "; + + /** + * Method for getInetAddress. + * + * @param url input parameter. + */ + public static InetAddress getInetAddress(String url) { + try { + return InetAddress.getByName(new URL(url).getHost()); + } catch (UnknownHostException | MalformedURLException e) { + log.error(INVALID_URL, e); + throw new BadRequestException(e.getMessage(), e); + } + } + + /** + * Validates the InetAddress and throws if the address is any of the following: 1. Link Local + * Address 2. Loopback + * Address 3. Multicast Address 4. Any Local Address 5. Site Local Address + * + * @param address the {@link InetAddress} to validate. + * @throws BadRequestException if the address is invalid. + */ + public static void validateInetAddress(@NonNull final InetAddress address) { + if (address.isMulticastAddress() || address.isAnyLocalAddress() || address.isLinkLocalAddress() + || address.isSiteLocalAddress() || address.isLoopbackAddress()) { + throw new BadRequestException(INVALID_URL); + } + } +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/Constants.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/Constants.java new file mode 100644 index 0000000000..ff6a780bfd --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/Constants.java @@ -0,0 +1,24 @@ +package org.opensearch.dataprepper.plugins.source.jira.utils; + +/** + * The type Constants. + */ +public class Constants { + + public static final int RETRY_ATTEMPT = 6; + + public static final String KEY = "key"; + public static final String NAME = "name"; + public static final String PROJECT = "project"; + public static final String OAUTH2 = "OAuth2"; + public static final String _PROJECT = "project-"; + public static final String _ISSUE = "ISSUE-"; + public static final String UPDATED = "updated"; + public static final String PROJECT_KEY = "j_project_key"; + public static final String PROJECT_NAME = "j_project_name"; + public static final String ISSUE_KEY = "j_issue_key"; + public static final String CREATED = "created"; + public static final String BASIC = "Basic"; + public static final String LIVE = "live"; + public static final String PLUGIN_NAME = "jira"; +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraConfigHelper.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraConfigHelper.java new file mode 100644 index 0000000000..07b1e1a213 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraConfigHelper.java @@ -0,0 +1,80 @@ +package org.opensearch.dataprepper.plugins.source.jira.utils; + + +import lombok.extern.slf4j.Slf4j; +import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; + +import java.util.List; + +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2; + +/** + * The type Jira configuration. + */ +@Slf4j +public class JiraConfigHelper { + + public static final String ISSUE_STATUS_FILTER = "status"; + public static final String ISSUE_TYPE_FILTER = "issuetype"; + + + /** + * Get Issue Status Filter from repository configuration. + * + * @return List Issue Status Filter. + */ + public static List getIssueStatusFilter(JiraSourceConfig repositoryConfiguration) { + return repositoryConfiguration.getStatus(); + } + + /** + * Get Issue Types Filter from repository configuration. + * + * @return List Issue Type Filter. + */ + public static List getIssueTypeFilter(JiraSourceConfig repositoryConfiguration) { + return repositoryConfiguration.getIssueType(); + } + + /** + * Get Project Filter Types from repository configuration. + * public static final String ST = "status"; + * + * @return List Project Filter. + */ + public static List getProjectKeyFilter(JiraSourceConfig repositoryConfiguration) { + return repositoryConfiguration.getProject(); + } + + + public static boolean validateConfig(JiraSourceConfig config) { + if (config.getAccountUrl() == null) { + throw new RuntimeException("Account URL is missing."); + } + //At least one of the AuthType should be present + if (config.getAuthType() == null) { + throw new RuntimeException("Authentication Type is missing."); + } + String authType = config.getAuthType(); + if (!OAUTH2.equals(authType) && !BASIC.equals(authType)) { + throw new RuntimeException("Invalid AuthType is given"); + } + + if (BASIC.equals(authType)) { + if (config.getJiraId() == null || config.getJiraCredential() == null) { + throw new RuntimeException("Jira ID or Credential are required for Basic AuthType"); + } + } + + if (OAUTH2.equals(authType)) { + if (config.getAccessToken() == null || config.getRefreshToken() == null) { + throw new RuntimeException("Access Token or Refresh Token are required for OAuth2 AuthType"); + } + } + + AddressValidation.validateInetAddress(AddressValidation + .getInetAddress(config.getAccountUrl())); + return true; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraContentType.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraContentType.java new file mode 100644 index 0000000000..9d37a6e8ca --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraContentType.java @@ -0,0 +1,16 @@ +package org.opensearch.dataprepper.plugins.source.jira.utils; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@AllArgsConstructor +public enum JiraContentType { + PROJECT("PROJECT"), + ISSUE("ISSUE"), + COMMENT("COMMENT"), + ATTACHMENT("ATTACHMENT"), + WORKLOG("WORKLOG"); + + @Getter + private final String type; +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JqlConstants.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JqlConstants.java new file mode 100644 index 0000000000..5b88208f74 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JqlConstants.java @@ -0,0 +1,17 @@ +package org.opensearch.dataprepper.plugins.source.jira.utils; + +public class JqlConstants { + public static final String GREATER_THAN_EQUALS = ">="; + public static final String CLOSING_ROUND_BRACKET = ")"; + + public static final String SLASH = "/"; + public static final String PROJECT_IN = " AND project in ("; + public static final String STATUS_IN = " AND status in ("; + public static final String DELIMITER = "\",\""; + public static final String PREFIX = "\""; + public static final String SUFFIX = "\""; + public static final String ISSUE_TYPE_IN = " AND issueType in ("; + public static final String JQL_FIELD = "jql"; + public static final String EXPAND_FIELD = "expand"; + public static final String EXPAND_VALUE = "all"; +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraClientTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraClientTest.java new file mode 100644 index 0000000000..4720ffc89b --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraClientTest.java @@ -0,0 +1,142 @@ +package org.opensearch.dataprepper.plugins.source.jira; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; +import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state.SaasWorkerProgressState; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class JiraClientTest { + + @Mock + private Buffer> buffer; + + @Mock + private SaasWorkerProgressState saasWorkerProgressState; + + @Mock + private CrawlerSourceConfig crawlerSourceConfig; + + @Mock + private JiraSourceConfig jiraSourceConfig; + + @Mock + private JiraService jiraService; + + @Mock + private JiraIterator jiraIterator; + + private PluginExecutorServiceProvider executorServiceProvider = new PluginExecutorServiceProvider(); + + @Test + void testConstructor() { + JiraClient jiraClient = new JiraClient(jiraService, jiraIterator, executorServiceProvider, jiraSourceConfig); + jiraClient.setLastPollTime(Instant.ofEpochSecond(1234L)); + assertNotNull(jiraClient); + } + + @Test + void testListItems() { + JiraClient jiraClient = new JiraClient(jiraService, jiraIterator, executorServiceProvider, jiraSourceConfig); + assertNotNull(jiraClient.listItems()); + } + + + @Test + void testExecutePartition() throws Exception { + JiraClient jiraClient = new JiraClient(jiraService, jiraIterator, executorServiceProvider, jiraSourceConfig); + Map keyAttributes = new HashMap<>(); + keyAttributes.put("project", "test"); + when(saasWorkerProgressState.getKeyAttributes()).thenReturn(keyAttributes); + List itemIds = new ArrayList<>(); + itemIds.add(null); + itemIds.add("ID2"); + itemIds.add("ID3"); + when(saasWorkerProgressState.getItemIds()).thenReturn(itemIds); + Instant exportStartTime = Instant.now(); + when(saasWorkerProgressState.getExportStartTime()).thenReturn(Instant.ofEpochSecond(exportStartTime.toEpochMilli())); + + when(jiraService.getIssue(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); + + ArgumentCaptor>> recordsCaptor = ArgumentCaptor.forClass((Class) Collection.class); + + jiraClient.executePartition(saasWorkerProgressState, buffer, crawlerSourceConfig); + + verify(buffer).writeAll(recordsCaptor.capture(), anyInt()); + Collection> capturedRecords = recordsCaptor.getValue(); + assertFalse(capturedRecords.isEmpty()); + for (Record record : capturedRecords) { + assertNotNull(record.getData()); + } + } + + @Test + void testExecutePartitionError() throws Exception { + JiraClient jiraClient = new JiraClient(jiraService, jiraIterator, executorServiceProvider, jiraSourceConfig); + Map keyAttributes = new HashMap<>(); + keyAttributes.put("project", "test"); + when(saasWorkerProgressState.getKeyAttributes()).thenReturn(keyAttributes); + List itemIds = List.of("ID1", "ID2", "ID3", "ID4"); + when(saasWorkerProgressState.getItemIds()).thenReturn(itemIds); + Instant exportStartTime = Instant.now(); + when(saasWorkerProgressState.getExportStartTime()).thenReturn(Instant.ofEpochSecond(exportStartTime.toEpochMilli())); + + when(jiraService.getIssue(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); + + ArgumentCaptor>> recordsCaptor = ArgumentCaptor.forClass((Class) Collection.class); + + ObjectMapper mockObjectMapper = mock(ObjectMapper.class); + when(mockObjectMapper.readValue(any(String.class), any(TypeReference.class))).thenThrow(new JsonProcessingException("test") { + }); + jiraClient.injectObjectMapper(mockObjectMapper); + + assertThrows(RuntimeException.class, () -> jiraClient.executePartition(saasWorkerProgressState, buffer, crawlerSourceConfig)); + } + + @Test + void bufferWriteRuntimeTest() throws Exception { + JiraClient jiraClient = new JiraClient(jiraService, jiraIterator, executorServiceProvider, jiraSourceConfig); + Map keyAttributes = new HashMap<>(); + keyAttributes.put("project", "test"); + when(saasWorkerProgressState.getKeyAttributes()).thenReturn(keyAttributes); + List itemIds = List.of("ID1", "ID2", "ID3", "ID4"); + when(saasWorkerProgressState.getItemIds()).thenReturn(itemIds); + Instant exportStartTime = Instant.now(); + when(saasWorkerProgressState.getExportStartTime()).thenReturn(Instant.ofEpochSecond(exportStartTime.toEpochMilli())); + + when(jiraService.getIssue(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); + + ArgumentCaptor>> recordsCaptor = ArgumentCaptor.forClass((Class) Collection.class); + + doThrow(new RuntimeException()).when(buffer).writeAll(recordsCaptor.capture(), anyInt()); + assertThrows(RuntimeException.class, () -> jiraClient.executePartition(saasWorkerProgressState, buffer, crawlerSourceConfig)); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraConfigHelperTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraConfigHelperTest.java new file mode 100644 index 0000000000..ac26864a25 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraConfigHelperTest.java @@ -0,0 +1,100 @@ +package org.opensearch.dataprepper.plugins.source.jira; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.jira.utils.JiraConfigHelper; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2; + +@ExtendWith(MockitoExtension.class) +public class JiraConfigHelperTest { + + @Mock + JiraSourceConfig jiraSourceConfig; + + @Test + void testInitialization() { + JiraConfigHelper jiraConfigHelper = new JiraConfigHelper(); + assertNotNull(jiraConfigHelper); + } + + @Test + void testGetIssueStatusFilter() { + assertTrue(JiraConfigHelper.getIssueStatusFilter(jiraSourceConfig).isEmpty()); + List issueStatusFilter = List.of("Done", "In Progress"); + when(jiraSourceConfig.getProject()).thenReturn(issueStatusFilter); + assertEquals(issueStatusFilter, JiraConfigHelper.getProjectKeyFilter(jiraSourceConfig)); + } + + @Test + void testGetIssueTypeFilter() { + assertTrue(JiraConfigHelper.getProjectKeyFilter(jiraSourceConfig).isEmpty()); + List issueTypeFilter = List.of("Bug", "Story"); + when(jiraSourceConfig.getProject()).thenReturn(issueTypeFilter); + assertEquals(issueTypeFilter, JiraConfigHelper.getProjectKeyFilter(jiraSourceConfig)); + } + + @Test + void testGetProjectKeyFilter() { + assertTrue(JiraConfigHelper.getProjectKeyFilter(jiraSourceConfig).isEmpty()); + List projectKeyFilter = List.of("TEST", "TEST2"); + when(jiraSourceConfig.getProject()).thenReturn(projectKeyFilter); + assertEquals(projectKeyFilter, JiraConfigHelper.getProjectKeyFilter(jiraSourceConfig)); + } + + @Test + void testValidateConfig() { + assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); + + when(jiraSourceConfig.getAccountUrl()).thenReturn("https://test.com"); + assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); + + when(jiraSourceConfig.getAuthType()).thenReturn("fakeType"); + assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); + } + + @Test + void testValidateConfigBasic() { + when(jiraSourceConfig.getAccountUrl()).thenReturn("https://test.com"); + when(jiraSourceConfig.getAuthType()).thenReturn(BASIC); + assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); + + when(jiraSourceConfig.getJiraId()).thenReturn("id"); + assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); + + when(jiraSourceConfig.getJiraCredential()).thenReturn("credential"); + when(jiraSourceConfig.getJiraId()).thenReturn(null); + assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); + + when(jiraSourceConfig.getJiraId()).thenReturn("id"); + assertDoesNotThrow(() -> JiraConfigHelper.validateConfig(jiraSourceConfig)); + } + + @Test + void testValidateConfigOauth2() { + when(jiraSourceConfig.getAccountUrl()).thenReturn("https://test.com"); + when(jiraSourceConfig.getAuthType()).thenReturn(OAUTH2); + assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); + + when(jiraSourceConfig.getAccessToken()).thenReturn("id"); + assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); + + when(jiraSourceConfig.getRefreshToken()).thenReturn("credential"); + when(jiraSourceConfig.getAccessToken()).thenReturn(null); + assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); + + when(jiraSourceConfig.getAccessToken()).thenReturn("id"); + assertDoesNotThrow(() -> JiraConfigHelper.validateConfig(jiraSourceConfig)); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java new file mode 100644 index 0000000000..49dc6873bd --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java @@ -0,0 +1,97 @@ +package org.opensearch.dataprepper.plugins.source.jira; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.time.Instant; +import java.util.Map; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class JiraItemInfoTest { + private String project; + private String issueType; + private String id; + private String itemId; + private Instant eventTime; + + @Mock + private Map metadata; + + @Mock + private Map newMetadata; + + @Mock + private JiraItemInfo jiraItemInfo; + + @BeforeEach + void setUP() { + issueType = "TestIssue"; + id = UUID.randomUUID().toString(); + project = "TestProject"; + itemId = UUID.randomUUID().toString(); + eventTime = Instant.ofEpochSecond(0); + jiraItemInfo = new JiraItemInfo(id, itemId, project, issueType, metadata, eventTime); + } + + @Test + void testGetters() { + assertEquals(jiraItemInfo.getItemId(), itemId); + assertEquals(jiraItemInfo.getId(), id); + assertEquals(jiraItemInfo.getProject(), project); + assertEquals(jiraItemInfo.getIssueType(), issueType); + assertEquals(jiraItemInfo.getMetadata(), metadata); + assertEquals(jiraItemInfo.getEventTime(), eventTime); + } + + @Test + void testGetKeyAttributes() { + assertInstanceOf(Map.class, jiraItemInfo.getKeyAttributes()); + } + + @Test + void testSetter() { + jiraItemInfo.setEventTime(Instant.now()); + assertNotEquals(jiraItemInfo.getEventTime(), eventTime); + jiraItemInfo.setItemId("newItemID"); + assertNotEquals(jiraItemInfo.getItemId(), itemId); + jiraItemInfo.setId("newID"); + assertNotEquals(jiraItemInfo.getId(), id); + jiraItemInfo.setProject("newProject"); + assertNotEquals(jiraItemInfo.getProject(), project); + jiraItemInfo.setMetadata(newMetadata); + assertNotEquals(jiraItemInfo.getMetadata(), metadata); + jiraItemInfo.setIssueType("newIssueType"); + assertNotEquals(jiraItemInfo.getIssueType(), issueType); + + } + + @Test + void testGetPartitionKey() { + String partitionKey = jiraItemInfo.getPartitionKey(); + assertTrue(partitionKey.contains(project)); + assertTrue(partitionKey.contains(issueType)); + } + + + @Test + void testGetLastModifiedAt() { + when(metadata.getOrDefault("updated", "0")).thenReturn("5"); + when(metadata.getOrDefault("created", "0")).thenReturn("0"); + assertEquals(Instant.ofEpochMilli(5), jiraItemInfo.getLastModifiedAt()); + + when(metadata.getOrDefault("updated", "0")).thenReturn("5"); + when(metadata.getOrDefault("created", "0")).thenReturn("7"); + assertEquals(Instant.ofEpochMilli(7), jiraItemInfo.getLastModifiedAt()); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraIteratorTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraIteratorTest.java new file mode 100644 index 0000000000..b8560d69a4 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraIteratorTest.java @@ -0,0 +1,147 @@ +package org.opensearch.dataprepper.plugins.source.jira; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.jira.models.IssueBean; +import org.opensearch.dataprepper.plugins.source.jira.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.jira.rest.JiraRestClient; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CREATED; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.KEY; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.UPDATED; + +@ExtendWith(MockitoExtension.class) +public class JiraIteratorTest { + + private final PluginExecutorServiceProvider executorServiceProvider = new PluginExecutorServiceProvider(); + + @Mock + private SearchResults mockSearchResults; + @Mock + private JiraRestClient jiraRestClient; + private JiraService jiraService; + @Mock + private JiraSourceConfig jiraSourceConfig; + + private JiraIterator jiraIterator; + + @BeforeEach + void setUp() { + jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient)); + } + + public JiraIterator createObjectUnderTest() { + return new JiraIterator(jiraService, executorServiceProvider, jiraSourceConfig); + } + + @Test + void testInitialization() { + jiraIterator = createObjectUnderTest(); + assertNotNull(jiraIterator); + jiraIterator.initialize(Instant.ofEpochSecond(0)); + when(mockSearchResults.getIssues()).thenReturn(new ArrayList<>()); + when(mockSearchResults.getTotal()).thenReturn(0); + doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(JiraSourceConfig.class)); + assertFalse(jiraIterator.hasNext()); + } + + @Test + void sleepInterruptionTest() { + jiraIterator = createObjectUnderTest(); + jiraIterator.initialize(Instant.ofEpochSecond(0)); + + Thread testThread = new Thread(() -> { + assertThrows(InterruptedException.class, () -> { + try { + jiraIterator.hasNext(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + }); + + testThread.start(); + testThread.interrupt(); + } + + @Test + void testItemInfoQueueNotEmpty() { + jiraIterator = createObjectUnderTest(); + List mockIssues = new ArrayList<>(); + IssueBean issue1 = createIssueBean(false); + mockIssues.add(issue1); + when(mockSearchResults.getIssues()).thenReturn(mockIssues); + when(mockSearchResults.getTotal()).thenReturn(0); + doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(JiraSourceConfig.class)); + + jiraIterator.initialize(Instant.ofEpochSecond(0)); + jiraIterator.setCrawlerQWaitTimeMillis(1); + assertTrue(jiraIterator.hasNext()); + assertNotNull(jiraIterator.next()); + assertNotNull(jiraIterator.next()); + assertFalse(jiraIterator.hasNext()); + } + + + private IssueBean createIssueBean(boolean nullFields) { + IssueBean issue1 = new IssueBean(); + issue1.setId(UUID.randomUUID().toString()); + issue1.setKey("issue_1_key"); + issue1.setSelf("https://example.com/rest/api/2/issue/123"); + issue1.setExpand("operations,versionedRepresentations,editmeta"); + + Map fieldMap = new HashMap<>(); + if (!nullFields) { + fieldMap.put(CREATED, Instant.now()); + fieldMap.put(UPDATED, Instant.now()); + } else { + fieldMap.put(CREATED, 0); + fieldMap.put(UPDATED, 0); + } + + Map issueTypeMap = new HashMap<>(); + issueTypeMap.put("name", "Task"); + issueTypeMap.put("self", "https://example.com/rest/api/2/issuetype/1"); + issueTypeMap.put("id", "1"); + fieldMap.put("issuetype", issueTypeMap); + + Map projectMap = new HashMap<>(); + if (!nullFields) { + projectMap.put("name", "project name test"); + projectMap.put(KEY, "TEST"); + } + fieldMap.put("project", projectMap); + + Map priorityMap = new HashMap<>(); + priorityMap.put("name", "Medium"); + fieldMap.put("priority", priorityMap); + + Map statusMap = new HashMap<>(); + statusMap.put("name", "In Progress"); + fieldMap.put("status", statusMap); + + issue1.setFields(fieldMap); + + return issue1; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java new file mode 100644 index 0000000000..cfa8a32733 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java @@ -0,0 +1,253 @@ +package org.opensearch.dataprepper.plugins.source.jira; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.jira.exception.BadRequestException; +import org.opensearch.dataprepper.plugins.source.jira.models.IssueBean; +import org.opensearch.dataprepper.plugins.source.jira.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.jira.rest.JiraRestClient; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; +import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.jira.rest.auth.JiraOauthConfig.ACCESSIBLE_RESOURCES; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CREATED; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.KEY; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.NAME; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.UPDATED; + + +/** + * The type Jira service. + */ +@ExtendWith(MockitoExtension.class) +public class JiraServiceTest { + + private static final Logger log = LoggerFactory.getLogger(JiraServiceTest.class); + private final PluginExecutorServiceProvider executorServiceProvider = new PluginExecutorServiceProvider(); + + @Mock + private JiraRestClient jiraRestClient; + + + private static InputStream getResourceAsStream(String resourceName) { + InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourceName); + if (inputStream == null) { + inputStream = JiraServiceTest.class.getResourceAsStream("/" + resourceName); + } + return inputStream; + } + + public static JiraSourceConfig createJiraConfigurationFromYaml(String fileName) { + ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); + try (InputStream inputStream = getResourceAsStream(fileName)) { + return objectMapper.readValue(inputStream, JiraSourceConfig.class); + } catch (IOException ex) { + log.error("Failed to parse pipeline Yaml", ex); + } + return null; + } + + public static JiraSourceConfig createJiraConfiguration(String auth_type, + List issueType, + List issueStatus, + List projectKey) throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(); + Map connectorCredentialsMap = new HashMap<>(); + connectorCredentialsMap.put("auth_type", auth_type); + + Map jiraSourceConfigMap = new HashMap<>(); + jiraSourceConfigMap.put("account_url", ACCESSIBLE_RESOURCES); + jiraSourceConfigMap.put("connector_credentials", connectorCredentialsMap); + jiraSourceConfigMap.put("issue_types", issueType); + jiraSourceConfigMap.put("statuses", issueStatus); + jiraSourceConfigMap.put("projects", projectKey); + + + String jiraSourceConfigJsonString = objectMapper.writeValueAsString(jiraSourceConfigMap); + return objectMapper.readValue(jiraSourceConfigJsonString, JiraSourceConfig.class); + } + + @AfterEach + void tearDown() { + executorServiceProvider.terminateExecutor(); + } + + @Test + void testJiraServiceInitialization() throws JsonProcessingException { + List issueType = new ArrayList<>(); + List issueStatus = new ArrayList<>(); + List projectKey = new ArrayList<>(); + JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); + JiraService jiraService = new JiraService(jiraSourceConfig, jiraRestClient); + assertNotNull(jiraService); + when(jiraRestClient.getIssue(anyString())).thenReturn("test String"); + assertNotNull(jiraService.getIssue("test Key")); + } + + @Test + public void testGetJiraEntities() throws JsonProcessingException { + List issueType = new ArrayList<>(); + List issueStatus = new ArrayList<>(); + List projectKey = new ArrayList<>(); + issueType.add("Task"); + issueStatus.add("Done"); + projectKey.add("KAN"); + JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); + JiraService jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient)); + List mockIssues = new ArrayList<>(); + IssueBean issue1 = createIssueBean(false, false); + mockIssues.add(issue1); + IssueBean issue2 = createIssueBean(true, false); + mockIssues.add(issue2); + IssueBean issue3 = createIssueBean(false, true); + mockIssues.add(issue3); + + SearchResults mockSearchResults = mock(SearchResults.class); + when(mockSearchResults.getIssues()).thenReturn(mockIssues); + when(mockSearchResults.getTotal()).thenReturn(mockIssues.size()); + + doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(JiraSourceConfig.class)); + + Instant timestamp = Instant.ofEpochSecond(0); + Queue itemInfoQueue = jiraService.getJiraEntities(jiraSourceConfig, timestamp); + assertEquals(mockIssues.size() + 1, itemInfoQueue.size()); + } + + @Test + public void buildIssueItemInfoMultipleFutureThreads() throws JsonProcessingException { + List issueType = new ArrayList<>(); + List issueStatus = new ArrayList<>(); + List projectKey = new ArrayList<>(); + issueType.add("Task"); + JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); + JiraService jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient)); + List mockIssues = new ArrayList<>(); + for (int i = 0; i < 50; i++) { + IssueBean issue1 = createIssueBean(false, false); + mockIssues.add(issue1); + } + + SearchResults mockSearchResults = mock(SearchResults.class); + when(mockSearchResults.getIssues()).thenReturn(mockIssues); + when(mockSearchResults.getTotal()).thenReturn(100); + + doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(JiraSourceConfig.class)); + + Instant timestamp = Instant.ofEpochSecond(0); + Queue itemInfoQueue = jiraService.getJiraEntities(jiraSourceConfig, timestamp); + assertTrue(itemInfoQueue.size() >= 100); + } + + @Test + public void testBadProjectKeys() throws JsonProcessingException { + List issueType = new ArrayList<>(); + List issueStatus = new ArrayList<>(); + List projectKey = new ArrayList<>(); + issueType.add("Task"); + issueStatus.add("Done"); + projectKey.add("Bad Project Key"); + projectKey.add("A"); + projectKey.add("!@#$"); + projectKey.add("AAAAAAAAAAAAAA"); + + JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); + JiraService jiraService = new JiraService(jiraSourceConfig, jiraRestClient); + + Instant timestamp = Instant.ofEpochSecond(0); + assertThrows(BadRequestException.class, () -> jiraService.getJiraEntities(jiraSourceConfig, timestamp)); + } + + @Test + public void testGetJiraEntitiesException() throws JsonProcessingException { + List issueType = new ArrayList<>(); + List issueStatus = new ArrayList<>(); + List projectKey = new ArrayList<>(); + issueType.add("Task"); + JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); + JiraService jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient)); + + doThrow(RuntimeException.class).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(JiraSourceConfig.class)); + + Instant timestamp = Instant.ofEpochSecond(0); + assertThrows(RuntimeException.class, () -> jiraService.getJiraEntities(jiraSourceConfig, timestamp)); + } + + + private IssueBean createIssueBean(boolean nullFields, boolean createdNull) { + IssueBean issue1 = new IssueBean(); + issue1.setId(UUID.randomUUID().toString()); + issue1.setKey("issue_1_key"); + issue1.setSelf("https://example.com/rest/api/2/issue/123"); + issue1.setExpand("operations,versionedRepresentations,editmeta"); + + Map fieldMap = new HashMap<>(); + if (!nullFields) { + fieldMap.put(CREATED, "2024-07-06T21:12:23.437-0700"); + fieldMap.put(UPDATED, "2024-07-06T21:12:23.106-0700"); + } else { + fieldMap.put(CREATED, 0); + fieldMap.put(UPDATED, 0); + } + if (createdNull) { + fieldMap.put(CREATED, null); + } + + Map issueTypeMap = new HashMap<>(); + issueTypeMap.put("name", "Task"); + issueTypeMap.put("self", "https://example.com/rest/api/2/issuetype/1"); + issueTypeMap.put("id", "1"); + fieldMap.put("issuetype", issueTypeMap); + + Map projectMap = new HashMap<>(); + if (!nullFields) { + projectMap.put(NAME, "project name test"); + projectMap.put(KEY, "TEST"); + } + fieldMap.put(PROJECT, projectMap); + + Map priorityMap = new HashMap<>(); + priorityMap.put(NAME, "Medium"); + fieldMap.put("priority", priorityMap); + + Map statusMap = new HashMap<>(); + statusMap.put(NAME, "In Progress"); + fieldMap.put("statuses", statusMap); + + issue1.setFields(fieldMap); + + return issue1; + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfigTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfigTest.java new file mode 100644 index 0000000000..9da7011d0d --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfigTest.java @@ -0,0 +1,118 @@ +package org.opensearch.dataprepper.plugins.source.jira; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2; +import static org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig.DEFAULT_NUMBER_OF_WORKERS; + +public class JiraSourceConfigTest { + private final String accessToken = "access token test"; + private final String refreshToken = "refresh token test"; + private final String clientId = "client id test"; + private final String clientSecret = "client secret test"; + private final String jiraCredential = "test Jira Credential"; + private final String jiraId = "test Jira Id"; + private final String accountUrl = "https://example.atlassian.net"; + private List projectList = new ArrayList<>(); + private List issueTypeList = new ArrayList<>(); + private List inclusionPatternList = new ArrayList<>(); + private List exclusionPatternList = new ArrayList<>(); + private List statusList = new ArrayList<>(); + private Map connectorCredentialMap = new HashMap<>(); + private JiraSourceConfig jiraSourceConfig; + + private JiraSourceConfig createJiraSourceConfig(String authtype, boolean hasToken) throws JsonProcessingException { + Map configMap = new HashMap<>(); + configMap.put("account_url", accountUrl); + + connectorCredentialMap.put("auth_type", authtype); + if (hasToken) { + connectorCredentialMap.put("access_token", accessToken); + connectorCredentialMap.put("refresh_token", refreshToken); + } else { + connectorCredentialMap.put("refresh_token", ""); + } + connectorCredentialMap.put("jira_id", jiraId); + connectorCredentialMap.put("jira_credential", jiraCredential); + connectorCredentialMap.put("client_id", clientId); + connectorCredentialMap.put("client_secret", clientSecret); + + configMap.put("connector_credentials", connectorCredentialMap); + + projectList.add("project1"); + projectList.add("project2"); + configMap.put("projects", projectList); + + issueTypeList.add("issue type 1"); + issueTypeList.add("issue type 2"); + configMap.put("issue_types", issueTypeList); + + inclusionPatternList.add("pattern 1"); + inclusionPatternList.add("pattern 2"); + configMap.put("inclusion_patterns", inclusionPatternList); + + exclusionPatternList.add("pattern 3"); + exclusionPatternList.add("pattern 4"); + configMap.put("exclusion_patterns", exclusionPatternList); + + statusList.add("status 1"); + statusList.add("status 2"); + configMap.put("statuses", statusList); + + ObjectMapper objectMapper = new ObjectMapper(); + String jsonConfig = objectMapper.writeValueAsString(configMap); + JiraSourceConfig config = objectMapper.readValue(jsonConfig, JiraSourceConfig.class); + return config; + } + + @Test + void testGetters() throws JsonProcessingException { + jiraSourceConfig = createJiraSourceConfig(BASIC, false); + assertEquals(jiraSourceConfig.getInclusionPatterns(), inclusionPatternList); + assertEquals(jiraSourceConfig.getIssueType(), issueTypeList); + assertEquals(jiraSourceConfig.getExclusionPatterns(), exclusionPatternList); + assertEquals(jiraSourceConfig.getNumWorkers(), DEFAULT_NUMBER_OF_WORKERS); + assertEquals(jiraSourceConfig.getProject(), projectList); + assertEquals(jiraSourceConfig.getStatus(), statusList); + assertEquals(jiraSourceConfig.getConnectorCredentials(), connectorCredentialMap); + assertEquals(jiraSourceConfig.getAccountUrl(), accountUrl); + assertNotNull(jiraSourceConfig.getBackOff()); + assertEquals(jiraSourceConfig.getJiraCredential(), jiraCredential); + assertEquals(jiraSourceConfig.getJiraId(), jiraId); + } + + @Test + void testFetchGivenOauthAttributeWrongAuthType() throws JsonProcessingException { + jiraSourceConfig = createJiraSourceConfig(BASIC, true); + assertThrows(RuntimeException.class, () -> jiraSourceConfig.getAccessToken()); + } + + @Test + void testFetchGivenOauthAtrribute() throws JsonProcessingException { + jiraSourceConfig = createJiraSourceConfig(OAUTH2, true); + assertEquals(accessToken, jiraSourceConfig.getAccessToken()); + assertEquals(refreshToken, jiraSourceConfig.getRefreshToken()); + assertEquals(clientId, jiraSourceConfig.getClientId()); + assertEquals(clientSecret, jiraSourceConfig.getClientSecret()); + } + + @Test + void testFetchGivenOauthAtrributeMissing() throws JsonProcessingException { + jiraSourceConfig = createJiraSourceConfig(OAUTH2, false); + assertThrows(RuntimeException.class, () -> jiraSourceConfig.getAccessToken()); + assertThrows(RuntimeException.class, () -> jiraSourceConfig.getRefreshToken()); + + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceTest.java new file mode 100644 index 0000000000..46cf58b7a9 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceTest.java @@ -0,0 +1,108 @@ +package org.opensearch.dataprepper.plugins.source.jira; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator; +import org.opensearch.dataprepper.plugins.source.jira.rest.auth.JiraAuthConfig; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.Crawler; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; + +import java.util.concurrent.ExecutorService; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.jira.rest.auth.JiraOauthConfig.ACCESSIBLE_RESOURCES; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.BASIC; + +@ExtendWith(MockitoExtension.class) +public class JiraSourceTest { + + @Mock + private PluginMetrics pluginMetrics; + + @Mock + private JiraSourceConfig jiraSourceConfig; + + @Mock + private JiraAuthConfig jiraOauthConfig; + + @Mock + private PluginFactory pluginFactory; + + @Mock + private AcknowledgementSetManager acknowledgementSetManager; + + @Mock + private Crawler crawler; + + @Mock + private EnhancedSourceCoordinator sourceCooridinator; + + @Mock + Buffer> buffer; + + @Mock + private PluginExecutorServiceProvider executorServiceProvider; + + @Mock + private ExecutorService executorService; +// = new PluginExecutorServiceProvider(); + + @Test + void initialization() { + when(executorServiceProvider.get()).thenReturn(executorService); + JiraSource source = new JiraSource(pluginMetrics, jiraSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + assertNotNull(source); + } + + @Test + void testStart() { + when(executorServiceProvider.get()).thenReturn(executorService); + JiraSource source = new JiraSource(pluginMetrics, jiraSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + when(jiraSourceConfig.getAccountUrl()).thenReturn(ACCESSIBLE_RESOURCES); + when(jiraSourceConfig.getAuthType()).thenReturn(BASIC); + when(jiraSourceConfig.getJiraId()).thenReturn("Test Id"); + when(jiraSourceConfig.getJiraCredential()).thenReturn("Test Credential"); + + source.setEnhancedSourceCoordinator(sourceCooridinator); + source.start(buffer); + verify(executorService, atLeast(1)).submit(any(Runnable.class)); + } + + @Test + void testStop() { + when(executorServiceProvider.get()).thenReturn(executorService); + JiraSource source = new JiraSource(pluginMetrics, jiraSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + when(jiraSourceConfig.getAccountUrl()).thenReturn(ACCESSIBLE_RESOURCES); + when(jiraSourceConfig.getAuthType()).thenReturn(BASIC); + when(jiraSourceConfig.getJiraId()).thenReturn("Test Id"); + when(jiraSourceConfig.getJiraCredential()).thenReturn("Test Credential"); + + source.setEnhancedSourceCoordinator(sourceCooridinator); + source.start(buffer); + source.stop(); + verify(executorService).shutdownNow(); + } + + @Test + void testStop_WhenNotStarted() { + when(executorServiceProvider.get()).thenReturn(executorService); + JiraSource source = new JiraSource(pluginMetrics, jiraSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + + source.stop(); + + verify(executorService, never()).shutdown(); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/exception/BadRequestExceptionTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/exception/BadRequestExceptionTest.java new file mode 100644 index 0000000000..74f5873af2 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/exception/BadRequestExceptionTest.java @@ -0,0 +1,54 @@ +package org.opensearch.dataprepper.plugins.source.jira.exception; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; + +public class BadRequestExceptionTest { + private String message; + private Throwable throwable; + + @BeforeEach + void setUp() { + message = "Bad Request"; + throwable = mock(Throwable.class); + } + + @Nested + class MessageOnlyConstructor { + private BadRequestException createObjectUnderTest() { + return new BadRequestException(message); + } + + @Test + void getMessage_returns_message() { + assertEquals(createObjectUnderTest().getMessage(), message); + } + + @Test + void getCause_returns_null() { + assertNull(createObjectUnderTest().getCause()); + } + } + + @Nested + class MessageThrowableConstructor { + private BadRequestException createObjectUnderTest() { + return new BadRequestException(message, throwable); + } + + @Test + void getMessage_returns_message() { + assertEquals(createObjectUnderTest().getMessage(), message); + } + + @Test + void getCause_returns_throwable() { + assertEquals(createObjectUnderTest().getCause(), throwable); + } + } +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/exception/UnAuthorizedExceptionTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/exception/UnAuthorizedExceptionTest.java new file mode 100644 index 0000000000..ecedff2d60 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/exception/UnAuthorizedExceptionTest.java @@ -0,0 +1,54 @@ +package org.opensearch.dataprepper.plugins.source.jira.exception; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; + +public class UnAuthorizedExceptionTest { + private String message; + private Throwable throwable; + + @BeforeEach + void setUp() { + message = "UnAuthorized Exception"; + throwable = mock(Throwable.class); + } + + @Nested + class MessageOnlyConstructor { + private UnAuthorizedException createObjectUnderTest() { + return new UnAuthorizedException(message); + } + + @Test + void getMessage_returns_message() { + assertEquals(createObjectUnderTest().getMessage(), message); + } + + @Test + void getCause_returns_null() { + assertNull(createObjectUnderTest().getCause()); + } + } + + @Nested + class MessageThrowableConstructor { + private UnAuthorizedException createObjectUnderTest() { + return new UnAuthorizedException(message, throwable); + } + + @Test + void getMessage_returns_message() { + assertEquals(createObjectUnderTest().getMessage(), message); + } + + @Test + void getCause_returns_throwable() { + assertEquals(createObjectUnderTest().getCause(), throwable); + } + } +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/models/IssueBeanTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/models/IssueBeanTest.java new file mode 100644 index 0000000000..0f99ddccbf --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/models/IssueBeanTest.java @@ -0,0 +1,120 @@ +package org.opensearch.dataprepper.plugins.source.jira.models; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.KEY; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.NAME; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT; + +@ExtendWith(MockitoExtension.class) +public class IssueBeanTest { + + @Mock + private Map renderedFieldsTestObject; + + @Mock + private Map propertiesTestObject; + + @Mock + private Map namesTestObject; + + @Mock + private Map fieldsTestObject; + + private IssueBean issueBean; + + @BeforeEach + void setup() { + issueBean = new IssueBean(); + } + + @Test + public void testInitialization() { + assertNotNull(issueBean); + } + + @Test + public void testNull() { + assertNull(issueBean.getExpand()); + assertNull(issueBean.getId()); + assertNull(issueBean.getSelf()); + assertNull(issueBean.getKey()); + assertNull(issueBean.getRenderedFields()); + assertNull(issueBean.getProperties()); + assertNull(issueBean.getNames()); + assertNull(issueBean.getFields()); + } + + @Test + void testNullCases() { + assertNull(issueBean.getProject()); + assertNull(issueBean.getProjectName()); + assertEquals(issueBean.getUpdatedTimeMillis(), 0); + } + + @Test + void testGivenDateField() { + Map fieldsTestObject = new HashMap<>(); + fieldsTestObject.put("created", "2024-07-06T21:12:23.437-0700"); + fieldsTestObject.put("updated", "2022-07-06T21:12:23.106-0700"); + issueBean.setFields(fieldsTestObject); + assertEquals(issueBean.getCreatedTimeMillis(), 1720325543000L); + assertEquals(issueBean.getUpdatedTimeMillis(), 1657167143000L); + } + + @Test + public void testStringSettersAndGetters() { + String self = "selfTest"; + String key = "keyTest"; + String id = "idTest"; + String expand = "expandTest"; + + issueBean.setExpand(expand); + assertEquals(issueBean.getExpand(), expand); + issueBean.setId(id); + assertEquals(issueBean.getId(), id); + issueBean.setSelf(self); + assertEquals(issueBean.getSelf(), self); + issueBean.setKey(key); + assertEquals(issueBean.getKey(), key); + } + + @Test + public void testMapSettersAndGetters() { + + issueBean.setRenderedFields(renderedFieldsTestObject); + assertEquals(issueBean.getRenderedFields(), renderedFieldsTestObject); + issueBean.setProperties(propertiesTestObject); + assertEquals(issueBean.getProperties(), propertiesTestObject); + issueBean.setNames(namesTestObject); + assertEquals(issueBean.getNames(), namesTestObject); + issueBean.setFields(fieldsTestObject); + assertEquals(issueBean.getFields(), fieldsTestObject); + } + + @Test + public void testFieldPropertyGetters() { + Map fieldsTestObject = new HashMap<>(); + Map projectTestObject = new HashMap<>(); + String projectName = "name of project"; + String projectKey = "PROJKEY"; + projectTestObject.put(KEY, projectKey); + projectTestObject.put(NAME, projectName); + fieldsTestObject.put(PROJECT, projectTestObject); + + issueBean.setFields(fieldsTestObject); + assertEquals(projectKey, issueBean.getProject()); + assertEquals(projectName, issueBean.getProjectName()); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResultsTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResultsTest.java new file mode 100644 index 0000000000..3a569ca39d --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResultsTest.java @@ -0,0 +1,87 @@ +package org.opensearch.dataprepper.plugins.source.jira.models; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +@ExtendWith(MockitoExtension.class) +public class SearchResultsTest { + + private final ObjectMapper objectMapper = new ObjectMapper(); + + private SearchResults searchResults; + + @BeforeEach + public void setUp() throws JsonProcessingException { + String state = "{}"; + searchResults = objectMapper.readValue(state, SearchResults.class); + } + + @Test + public void testConstructor() { + assertNotNull(searchResults); + + assertNull(searchResults.getExpand()); + assertNull(searchResults.getStartAt()); + assertNull(searchResults.getMaxResults()); + assertNull(searchResults.getTotal()); + assertNull(searchResults.getIssues()); + } + + @Test + public void testGetters() throws JsonProcessingException { + String expand = "expandTest"; + Integer startAt = 1; + Integer maxResults = 100; + Integer total = 10; + List testIssues = new ArrayList<>(); + IssueBean issue1 = new IssueBean(); + IssueBean issue2 = new IssueBean(); + issue1.setId("issue 1"); + issue2.setId("issue 2"); + testIssues.add(issue1); + testIssues.add(issue2); + + + Map searchResultsMap = new HashMap<>(); + searchResultsMap.put("expand", expand); + searchResultsMap.put("startAt", startAt); + searchResultsMap.put("maxResults", maxResults); + searchResultsMap.put("total", total); + searchResultsMap.put("issues", testIssues); + + String jsonString = objectMapper.writeValueAsString(searchResultsMap); + + searchResults = objectMapper.readValue(jsonString, SearchResults.class); + + assertEquals(searchResults.getExpand(), expand); + assertEquals(searchResults.getStartAt(), startAt); + assertEquals(searchResults.getMaxResults(), maxResults); + assertEquals(searchResults.getTotal(), total); + + List returnedIssues = searchResults.getIssues(); + assertNotNull(returnedIssues); + assertEquals(testIssues.size(), returnedIssues.size()); + + for (int i = 0; i < testIssues.size(); i++) { + IssueBean originalIssue = testIssues.get(i); + IssueBean returnedIssue = returnedIssues.get(i); + + assertEquals(originalIssue.getId(), returnedIssue.getId()); + } + } + + +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/BasicAuthInterceptorTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/BasicAuthInterceptorTest.java new file mode 100644 index 0000000000..18c94cb426 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/BasicAuthInterceptorTest.java @@ -0,0 +1,65 @@ +package org.opensearch.dataprepper.plugins.source.jira.rest; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpRequest; +import org.springframework.http.client.ClientHttpRequestExecution; +import org.springframework.http.client.ClientHttpResponse; + +import java.io.IOException; +import java.util.Base64; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class BasicAuthInterceptorTest { + + @Mock + private HttpRequest mockRequest; + + @Mock + private ClientHttpRequestExecution mockExecution; + + @Mock + private ClientHttpResponse mockResponse; + + @Mock + private JiraSourceConfig mockConfig; + + @Mock + private HttpHeaders mockHeaders; + + private BasicAuthInterceptor interceptor; + + @BeforeEach + void setUp() { + when(mockConfig.getJiraId()).thenReturn("testUser"); + when(mockConfig.getJiraCredential()).thenReturn("testPassword"); + when(mockRequest.getHeaders()).thenReturn(mockHeaders); + interceptor = new BasicAuthInterceptor(mockConfig); + } + + @Test + void testInterceptAddsAuthorizationHeader() throws IOException { + when(mockExecution.execute(any(HttpRequest.class), any(byte[].class))).thenReturn(mockResponse); + + ClientHttpResponse response = interceptor.intercept(mockRequest, new byte[0], mockExecution); + + verify(mockHeaders).set(eq(HttpHeaders.AUTHORIZATION), argThat(value -> + value.startsWith("Basic ") && + new String(Base64.getDecoder().decode(value.substring(6))).equals("testUser:testPassword") + )); + assertEquals(mockResponse, response); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/CustomRestTemplateConfigTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/CustomRestTemplateConfigTest.java new file mode 100644 index 0000000000..ce0675f22a --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/CustomRestTemplateConfigTest.java @@ -0,0 +1,64 @@ +package org.opensearch.dataprepper.plugins.source.jira.rest; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; +import org.opensearch.dataprepper.plugins.source.jira.rest.auth.JiraAuthConfig; +import org.springframework.http.client.ClientHttpRequestInterceptor; +import org.springframework.http.client.InterceptingClientHttpRequestFactory; +import org.springframework.web.client.RestTemplate; + +import java.util.List; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2; + +@ExtendWith(MockitoExtension.class) +class CustomRestTemplateConfigTest { + + private CustomRestTemplateConfig config; + + @Mock + private JiraSourceConfig mockSourceConfig; + + @Mock + private JiraAuthConfig mockAuthConfig; + + private static Stream provideAuthTypeAndExpectedInterceptorType() { + return Stream.of( + Arguments.of(OAUTH2, OAuth2RequestInterceptor.class), + Arguments.of(BASIC, BasicAuthInterceptor.class), + Arguments.of("Default", BasicAuthInterceptor.class), + Arguments.of(null, BasicAuthInterceptor.class) + ); + } + + @BeforeEach + void setUp() { + config = new CustomRestTemplateConfig(); + } + + @ParameterizedTest + @MethodSource("provideAuthTypeAndExpectedInterceptorType") + void testBasicAuthRestTemplateWithOAuth2(String authType, Class interceptorClassType) { + when(mockSourceConfig.getAuthType()).thenReturn(authType); + RestTemplate restTemplate = config.basicAuthRestTemplate(mockSourceConfig, mockAuthConfig); + assertNotNull(restTemplate); + assertInstanceOf(InterceptingClientHttpRequestFactory.class, restTemplate.getRequestFactory()); + List interceptors = restTemplate.getInterceptors(); + assertEquals(1, interceptors.size()); + assertInstanceOf(interceptorClassType, interceptors.get(0)); + } + +} + diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClientTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClientTest.java new file mode 100644 index 0000000000..c9bdb39625 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClientTest.java @@ -0,0 +1,131 @@ +package org.opensearch.dataprepper.plugins.source.jira.rest; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.jira.JiraServiceTest; +import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; +import org.opensearch.dataprepper.plugins.source.jira.exception.UnAuthorizedException; +import org.opensearch.dataprepper.plugins.source.jira.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.jira.rest.auth.JiraAuthConfig; +import org.opensearch.dataprepper.plugins.source.jira.rest.auth.JiraAuthFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2; + +@ExtendWith(MockitoExtension.class) +public class JiraRestClientTest { + + @Mock + private StringBuilder jql; + + @Mock + private RestTemplate restTemplate; + + @Mock + private JiraAuthConfig authConfig; + + private static Stream provideHttpStatusCodesWithExceptionClass() { + return Stream.of( + Arguments.of(HttpStatus.FORBIDDEN, UnAuthorizedException.class), + Arguments.of(HttpStatus.UNAUTHORIZED, RuntimeException.class), + Arguments.of(HttpStatus.TOO_MANY_REQUESTS, RuntimeException.class), + Arguments.of(HttpStatus.INSUFFICIENT_STORAGE, RuntimeException.class) + ); + } + + @ParameterizedTest + @ValueSource(strings = {"basic-auth-jira-pipeline.yaml"}) + public void testFetchingJiraIssue(String configFileName) { + String exampleTicketResponse = "{\"id\":\"123\",\"key\":\"key\",\"self\":\"https://example.com/rest/api/2/issue/123\"}"; + doReturn(new ResponseEntity<>(exampleTicketResponse, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); + JiraSourceConfig jiraSourceConfig = JiraServiceTest.createJiraConfigurationFromYaml(configFileName); + JiraAuthConfig authConfig = new JiraAuthFactory(jiraSourceConfig).getObject(); + JiraRestClient jiraRestClient = new JiraRestClient(restTemplate, authConfig); + String ticketDetails = jiraRestClient.getIssue("key"); + assertEquals(exampleTicketResponse, ticketDetails); + } + + @ParameterizedTest + @MethodSource("provideHttpStatusCodesWithExceptionClass") + void testInvokeRestApiTokenExpired(HttpStatus statusCode, Class expectedExceptionType) { + JiraRestClient jiraRestClient = new JiraRestClient(restTemplate, authConfig); + jiraRestClient.setSleepTimeMultiplier(1); + when(authConfig.getUrl()).thenReturn("https://example.com/rest/api/2/issue/key"); + when(restTemplate.getForEntity(any(URI.class), any(Class.class))).thenThrow(new HttpClientErrorException(statusCode)); + assertThrows(expectedExceptionType, () -> jiraRestClient.getIssue("key")); + } + + @Test + void testInvokeRestApiTokenExpiredInterruptException() throws InterruptedException { + JiraRestClient jiraRestClient = new JiraRestClient(restTemplate, authConfig); + when(authConfig.getUrl()).thenReturn("https://example.com/rest/api/2/issue/key"); + when(restTemplate.getForEntity(any(URI.class), any(Class.class))).thenThrow(new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS)); + jiraRestClient.setSleepTimeMultiplier(100000); + + Thread testThread = new Thread(() -> { + assertThrows(InterruptedException.class, () -> { + try { + jiraRestClient.getIssue("key"); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + }); + testThread.start(); + Thread.sleep(100); + testThread.interrupt(); + } + + @Test + public void testGetAllIssuesOauth2() throws JsonProcessingException { + List issueType = new ArrayList<>(); + List issueStatus = new ArrayList<>(); + List projectKey = new ArrayList<>(); + issueType.add("Task"); + JiraSourceConfig jiraSourceConfig = JiraServiceTest.createJiraConfiguration(OAUTH2, issueType, issueStatus, projectKey); + JiraRestClient jiraRestClient = new JiraRestClient(restTemplate, authConfig); + SearchResults mockSearchResults = mock(SearchResults.class); + doReturn("http://mock-service.jira.com").when(authConfig).getUrl(); + doReturn(new ResponseEntity<>(mockSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); + SearchResults results = jiraRestClient.getAllIssues(jql, 0, jiraSourceConfig); + assertNotNull(results); + } + + @Test + public void testGetAllIssuesBasic() throws JsonProcessingException { + List issueType = new ArrayList<>(); + List issueStatus = new ArrayList<>(); + List projectKey = new ArrayList<>(); + issueType.add("Task"); + JiraSourceConfig jiraSourceConfig = JiraServiceTest.createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); + JiraRestClient jiraRestClient = new JiraRestClient(restTemplate, authConfig); + SearchResults mockSearchResults = mock(SearchResults.class); + doReturn(new ResponseEntity<>(mockSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); + SearchResults results = jiraRestClient.getAllIssues(jql, 0, jiraSourceConfig); + assertNotNull(results); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/OAuth2RequestInterceptorTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/OAuth2RequestInterceptorTest.java new file mode 100644 index 0000000000..277c42ca2e --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/OAuth2RequestInterceptorTest.java @@ -0,0 +1,56 @@ +package org.opensearch.dataprepper.plugins.source.jira.rest; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.jira.rest.auth.JiraOauthConfig; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpRequest; +import org.springframework.http.client.ClientHttpRequestExecution; +import org.springframework.http.client.ClientHttpResponse; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class OAuth2RequestInterceptorTest { + + @Mock + private HttpRequest mockRequest; + + @Mock + private ClientHttpRequestExecution mockExecution; + + @Mock + private ClientHttpResponse mockResponse; + + @Mock + private JiraOauthConfig mockConfig; + + @Mock + private HttpHeaders mockHeaders; + + private OAuth2RequestInterceptor interceptor; + + @BeforeEach + void setUp() { + when(mockConfig.getAccessToken()).thenReturn("testAccessToken"); + when(mockRequest.getHeaders()).thenReturn(mockHeaders); + interceptor = new OAuth2RequestInterceptor(mockConfig); + } + + + @Test + void testInterceptAddsAuthorizationHeader() throws IOException { + when(mockExecution.execute(any(HttpRequest.class), any(byte[].class))).thenReturn(mockResponse); + ClientHttpResponse response = interceptor.intercept(mockRequest, new byte[0], mockExecution); + verify(mockHeaders).setBearerAuth("testAccessToken"); + assertEquals(mockResponse, response); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthFactoryTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthFactoryTest.java new file mode 100644 index 0000000000..d006f8df3f --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthFactoryTest.java @@ -0,0 +1,43 @@ +package org.opensearch.dataprepper.plugins.source.jira.rest.auth; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2; + +@ExtendWith(MockitoExtension.class) +public class JiraAuthFactoryTest { + + @Mock + private JiraSourceConfig sourceConfig; + + private JiraAuthFactory jiraAuthFactory; + + @BeforeEach + void setUp() { + jiraAuthFactory = new JiraAuthFactory(sourceConfig); + } + + @Test + void testGetObjectOauth2() { + when(sourceConfig.getAuthType()).thenReturn(OAUTH2); + assertInstanceOf(JiraOauthConfig.class, jiraAuthFactory.getObject()); + } + + @Test + void testGetObjectBasicAuth() { + assertInstanceOf(JiraBasicAuthConfig.class, jiraAuthFactory.getObject()); + } + + @Test + void testGetObjectType() { + assertEquals(JiraAuthConfig.class, jiraAuthFactory.getObjectType()); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraBasicAuthConfigTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraBasicAuthConfigTest.java new file mode 100644 index 0000000000..0028525127 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraBasicAuthConfigTest.java @@ -0,0 +1,35 @@ +package org.opensearch.dataprepper.plugins.source.jira.rest.auth; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +@ExtendWith(MockitoExtension.class) +public class JiraBasicAuthConfigTest { + + @Mock + private JiraSourceConfig jiraSourceConfig; + + private JiraBasicAuthConfig jiraBasicAuthConfig; + + @BeforeEach + void setUp() { + jiraBasicAuthConfig = new JiraBasicAuthConfig(jiraSourceConfig); + } + + @Test + void testGetUrl() { + assertEquals(jiraBasicAuthConfig.getUrl(), jiraSourceConfig.getAccountUrl()); + } + + @Test + void DoNothingForBasicAuthentication() { + jiraBasicAuthConfig.initCredentials(); + jiraBasicAuthConfig.renewCredentials(); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfigTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfigTest.java new file mode 100644 index 0000000000..1b648ab404 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfigTest.java @@ -0,0 +1,145 @@ +package org.opensearch.dataprepper.plugins.source.jira.rest.auth; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; +import org.opensearch.dataprepper.plugins.source.jira.exception.UnAuthorizedException; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; + +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.jira.JiraServiceTest.createJiraConfigurationFromYaml; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.RETRY_ATTEMPT; + +@ExtendWith(MockitoExtension.class) +public class JiraOauthConfigTest { + + @Mock + RestTemplate restTemplateMock; + + JiraSourceConfig jiraSourceConfig = createJiraConfigurationFromYaml("oauth2-auth-jira-pipeline.yaml"); + + @Test + void testRenewToken() throws InterruptedException { + Instant testStartTime = Instant.now(); + Map firstMockResponseMap = Map.of("access_token", "first_mock_access_token", + "refresh_token", "first_mock_refresh_token", + "expires_in", 3600); + JiraOauthConfig jiraOauthConfig = new JiraOauthConfig(jiraSourceConfig); + when(restTemplateMock.postForEntity(any(String.class), any(HttpEntity.class), any(Class.class))) + .thenReturn(new ResponseEntity<>(firstMockResponseMap, HttpStatus.OK)); + jiraOauthConfig.restTemplate = restTemplateMock; + ExecutorService executor = Executors.newFixedThreadPool(2); + Future firstCall = executor.submit(jiraOauthConfig::renewCredentials); + Future secondCall = executor.submit(jiraOauthConfig::renewCredentials); + while (!firstCall.isDone() || !secondCall.isDone()) { + // Do nothing. Wait for the calls to complete + Thread.sleep(10); + } + executor.shutdown(); + assertNotNull(jiraOauthConfig.getAccessToken()); + assertNotNull(jiraOauthConfig.getExpireTime()); + assertEquals(jiraOauthConfig.getRefreshToken(), "first_mock_refresh_token"); + assertEquals(jiraOauthConfig.getExpiresInSeconds(), 3600); + assertEquals(jiraOauthConfig.getAccessToken(), "first_mock_access_token"); + assertTrue(jiraOauthConfig.getExpireTime().isAfter(testStartTime)); + Instant expectedNewExpireTime = Instant.ofEpochMilli(testStartTime.toEpochMilli() + 3601 * 1000); + assertTrue(jiraOauthConfig.getExpireTime().isBefore(expectedNewExpireTime), + String.format("Expected that %s time should be before %s", jiraOauthConfig.getExpireTime(), expectedNewExpireTime)); + verify(restTemplateMock, times(1)).postForEntity(any(String.class), any(HttpEntity.class), any(Class.class)); + + } + + @Test + void testFailedToRenewAccessToken() { + JiraOauthConfig jiraOauthConfig = new JiraOauthConfig(jiraSourceConfig); + when(restTemplateMock.postForEntity(any(String.class), any(HttpEntity.class), any(Class.class))) + .thenThrow(HttpClientErrorException.class); + jiraOauthConfig.restTemplate = restTemplateMock; + assertThrows(RuntimeException.class, jiraOauthConfig::renewCredentials); + } + + + @Test + void testGetJiraAccountCloudId() throws InterruptedException { + Map mockGetCallResponse = new HashMap<>(); + mockGetCallResponse.put("id", "test_cloud_id"); + when(restTemplateMock.exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class))) + .thenReturn(new ResponseEntity<>(List.of(mockGetCallResponse), HttpStatus.OK)); + JiraOauthConfig jiraOauthConfig = new JiraOauthConfig(jiraSourceConfig); + jiraOauthConfig.restTemplate = restTemplateMock; + + ExecutorService executor = Executors.newFixedThreadPool(2); + Future firstCall = executor.submit(jiraOauthConfig::getUrl); + Future secondCall = executor.submit(jiraOauthConfig::getUrl); + while (!firstCall.isDone() || !secondCall.isDone()) { + // Do nothing. Wait for the calls to complete + Thread.sleep(10); + } + executor.shutdown(); + + assertEquals("test_cloud_id", jiraOauthConfig.getJiraAccountCloudId()); + assertEquals("https://api.atlassian.com/ex/jira/test_cloud_id/", jiraOauthConfig.getUrl()); + //calling second time shouldn't trigger rest call + jiraOauthConfig.getUrl(); + verify(restTemplateMock, times(1)) + .exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class)); + } + + @Test + void testGetJiraAccountCloudIdUnauthorizedCase() { + + when(restTemplateMock.exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class))) + .thenThrow(new HttpClientErrorException(HttpStatus.UNAUTHORIZED)); + Map mockRenewTokenResponse = Map.of("access_token", "first_mock_access_token", + "refresh_token", "first_mock_refresh_token", + "expires_in", 3600); + when(restTemplateMock.postForEntity(any(String.class), any(HttpEntity.class), any(Class.class))) + .thenReturn(new ResponseEntity<>(mockRenewTokenResponse, HttpStatus.OK)); + JiraOauthConfig jiraOauthConfig = new JiraOauthConfig(jiraSourceConfig); + jiraOauthConfig.restTemplate = restTemplateMock; + + + assertThrows(UnAuthorizedException.class, () -> jiraOauthConfig.initCredentials()); + verify(restTemplateMock, times(6)) + .exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class)); + verify(restTemplateMock, times(1)) + .postForEntity(any(String.class), any(HttpEntity.class), any(Class.class)); + + } + + @Test + void testFailedToGetCloudId() { + when(restTemplateMock.exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class))) + .thenThrow(new HttpClientErrorException(HttpStatus.UNAUTHORIZED)) + .thenThrow(HttpClientErrorException.class); + JiraOauthConfig jiraOauthConfig = new JiraOauthConfig(jiraSourceConfig); + jiraOauthConfig.restTemplate = restTemplateMock; + assertThrows(RuntimeException.class, jiraOauthConfig::getUrl); + for (int i = 0; i <= RETRY_ATTEMPT; i++) { + assertThrows(RuntimeException.class, jiraOauthConfig::getUrl); + } + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidationTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidationTest.java new file mode 100644 index 0000000000..b6f56a11ee --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidationTest.java @@ -0,0 +1,92 @@ +package org.opensearch.dataprepper.plugins.source.jira.utils; + +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.plugins.source.jira.exception.BadRequestException; + +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.UnknownHostException; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class AddressValidationTest { + + @Test + void testInstanceCreation() { + assertNotNull(new AddressValidation()); + assertNotNull(new Constants()); + assertNotNull(new JqlConstants()); + } + + @Test + void testGetInetAddress() { + String testUrl = "https://www.amazon.com"; + System.out.print("Test output"); + System.out.print(AddressValidation.getInetAddress(testUrl)); + } + + @Test + void testGetInetAddressWithMalformedUrl() { + String testUrl = "XXXXXXXXXXXXXXXXXXXXXX"; + assertThrows(BadRequestException.class, () -> AddressValidation.getInetAddress(testUrl)); + } + + @Test + void testGetInetAddressWithUnknownHost() { + String testUrl = "https://www.thisurldoesntexist1384276t5917278481073.com"; + assertThrows(BadRequestException.class, () -> AddressValidation.getInetAddress(testUrl)); + } + + @Test + void testGetInetAddressWithNullUrl() { + String testUrl = null; + assertThrows(BadRequestException.class, () -> AddressValidation.getInetAddress(testUrl)); + } + + + @Test + void testValidateInetAddressAnyLocalAddress() throws UnknownHostException { + InetAddress wildcardAddress = InetAddress.getByName("0.0.0.0"); + assertThrows(BadRequestException.class, () -> AddressValidation.validateInetAddress(wildcardAddress)); + } + + @Test + void testValidateInetAddressMulticastAddress() throws UnknownHostException { + InetAddress multicastAddress = InetAddress.getByName("224.0.0.1"); + assertThrows(BadRequestException.class, () -> AddressValidation.validateInetAddress(multicastAddress)); + } + + @Test + void testValidateInetAddressLinkLocalAddress() throws UnknownHostException { + InetAddress linkLocalAddress = InetAddress.getByName("169.254.1.1"); + assertThrows(BadRequestException.class, () -> AddressValidation.validateInetAddress(linkLocalAddress)); + } + + @Test + void testValidateInetAddressSiteLocalAddress() throws UnknownHostException { + InetAddress siteLocalAddress = InetAddress.getByName("10.0.0.1"); + assertThrows(BadRequestException.class, () -> AddressValidation.validateInetAddress(siteLocalAddress)); + } + + @Test + void testValidateInetAddressLoopbackAddress() throws UnknownHostException { + InetAddress loopbackAddress = InetAddress.getByName("127.0.0.1"); + assertThrows(BadRequestException.class, () -> AddressValidation.validateInetAddress(loopbackAddress)); + } + + @Test + void testValidateInetAddressValidAddress() throws UnknownHostException, MalformedURLException { + InetAddress validAddress = InetAddress.getByName(new URL("https://www.amazon.com").getHost()); + assertDoesNotThrow(() -> AddressValidation.validateInetAddress(validAddress)); + } + + @Test + void testValidateInetAddressNullAddress() { + InetAddress nullAddress = null; + assertThrows(NullPointerException.class, () -> AddressValidation.validateInetAddress(nullAddress)); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraContentTypeTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraContentTypeTest.java new file mode 100644 index 0000000000..c342e9a3cd --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraContentTypeTest.java @@ -0,0 +1,26 @@ +package org.opensearch.dataprepper.plugins.source.jira.utils; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class JiraContentTypeTest { + @Test + void testEnumConstants() { + assertNotNull(JiraContentType.PROJECT); + assertNotNull(JiraContentType.ISSUE); + assertNotNull(JiraContentType.COMMENT); + assertNotNull(JiraContentType.ATTACHMENT); + assertNotNull(JiraContentType.WORKLOG); + } + + @Test + void testTypeValues() { + assertEquals("PROJECT", JiraContentType.PROJECT.getType()); + assertEquals("ISSUE", JiraContentType.ISSUE.getType()); + assertEquals("COMMENT", JiraContentType.COMMENT.getType()); + assertEquals("ATTACHMENT", JiraContentType.ATTACHMENT.getType()); + assertEquals("WORKLOG", JiraContentType.WORKLOG.getType()); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/basic-auth-jira-pipeline.yaml b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/basic-auth-jira-pipeline.yaml new file mode 100644 index 0000000000..09b15d40d6 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/basic-auth-jira-pipeline.yaml @@ -0,0 +1,6 @@ +account_url: "https://jira.com/" +connector_credentials: + auth_type: "Basic" + jira_id: "jiraId" + jira_credential: "jiraApiKey" + diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/oauth2-auth-jira-pipeline.yaml b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/oauth2-auth-jira-pipeline.yaml new file mode 100644 index 0000000000..ae1f0b508d --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/oauth2-auth-jira-pipeline.yaml @@ -0,0 +1,9 @@ +account_url: "https://jira.com/" +connector_credentials: + auth_type: "OAuth2" + jira_id: "jira_id" + client_id: "client_id" + client_secret: "client_secret" + access_token: "access_token" + refresh_token: "refresh_token" + diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/Crawler.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/Crawler.java index e8b01bfa1e..327fff0d89 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/Crawler.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/Crawler.java @@ -23,8 +23,8 @@ @Named public class Crawler { private static final Logger log = LoggerFactory.getLogger(Crawler.class); - private static final int maxItemsPerPage = 20; - private final Timer crawlingTime; + private static final int maxItemsPerPage = 50; + private final Timer crawlingTimer; private final PluginMetrics pluginMetrics = PluginMetrics.fromNames("sourceCrawler", "crawler"); @@ -32,7 +32,7 @@ public class Crawler { public Crawler(CrawlerClient client) { this.client = client; - this.crawlingTime = pluginMetrics.timer("crawlingTime"); + this.crawlingTimer = pluginMetrics.timer("crawlingTime"); } public Instant crawl(Instant lastPollTime, @@ -41,7 +41,6 @@ public Instant crawl(Instant lastPollTime, client.setLastPollTime(lastPollTime); Iterator itemInfoIterator = client.listItems(); log.info("Starting to crawl the source with lastPollTime: {}", lastPollTime); - Instant updatedPollTime = Instant.ofEpochMilli(0); do { final List itemInfoList = new ArrayList<>(); for (int i = 0; i < maxItemsPerPage && itemInfoIterator.hasNext(); i++) { @@ -52,15 +51,14 @@ public Instant crawl(Instant lastPollTime, continue; } itemInfoList.add(nextItem); - Instant lastModifiedTime = nextItem.getLastModifiedAt(); - updatedPollTime = updatedPollTime.isAfter(lastModifiedTime) ? updatedPollTime : lastModifiedTime; } createPartition(itemInfoList, coordinator); + // intermediate updates to master partition state is required here } while (itemInfoIterator.hasNext()); long crawlTimeMillis = System.currentTimeMillis() - startTime; log.debug("Crawling completed in {} ms", crawlTimeMillis); - crawlingTime.record(crawlTimeMillis, TimeUnit.MILLISECONDS); - return updatedPollTime; + crawlingTimer.record(crawlTimeMillis, TimeUnit.MILLISECONDS); + return Instant.ofEpochMilli(startTime); } public void executePartition(SaasWorkerProgressState state, Buffer> buffer, CrawlerSourceConfig sourceConfig) { diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/state/LeaderProgressState.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/state/LeaderProgressState.java index 062753cc03..19f88e3949 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/state/LeaderProgressState.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/state/LeaderProgressState.java @@ -1,10 +1,8 @@ package org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import lombok.Getter; import lombok.Setter; -import org.opensearch.dataprepper.plugins.source.source_crawler.util.CustomInstantDeserializer; import java.time.Instant; @@ -16,7 +14,6 @@ public class LeaderProgressState { private boolean initialized = false; @JsonProperty("last_poll_time") - @JsonDeserialize(using = CustomInstantDeserializer.class) private Instant lastPollTime; public LeaderProgressState(@JsonProperty("last_poll_time") final Instant lastPollTime) { diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/state/SaasWorkerProgressState.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/state/SaasWorkerProgressState.java index 2dbfa28a67..bc2fb4e094 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/state/SaasWorkerProgressState.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/state/SaasWorkerProgressState.java @@ -6,10 +6,8 @@ package org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import lombok.Getter; import lombok.Setter; -import org.opensearch.dataprepper.plugins.source.source_crawler.util.CustomInstantDeserializer; import java.time.Instant; import java.util.HashMap; @@ -27,10 +25,9 @@ public class SaasWorkerProgressState { private int loadedItems; @JsonProperty("exportStartTime") - @JsonDeserialize(using = CustomInstantDeserializer.class) private Instant exportStartTime; - private Map keyAttributes = new HashMap<>(); + private Map keyAttributes = new HashMap<>(); @JsonProperty("itemIds") private List itemIds; diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/model/ItemInfo.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/model/ItemInfo.java index 48a063d64a..fb1f25b581 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/model/ItemInfo.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/model/ItemInfo.java @@ -42,7 +42,7 @@ public interface ItemInfo { * * @return A map of key attributes of this Item. */ - Map getKeyAttributes(); + Map getKeyAttributes(); /** * Service specific Item's last modified time diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/util/CustomInstantDeserializer.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/util/CustomInstantDeserializer.java deleted file mode 100644 index ffc6b0f485..0000000000 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/util/CustomInstantDeserializer.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.opensearch.dataprepper.plugins.source.source_crawler.util; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; - -import java.io.IOException; -import java.time.Instant; - -public class CustomInstantDeserializer extends JsonDeserializer { - @Override - public Instant deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - long millis = p.getLongValue(); - return Instant.ofEpochMilli(millis); - } -} diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/CrawlerTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/CrawlerTest.java index ad8358a1e6..45d2fcb402 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/CrawlerTest.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/CrawlerTest.java @@ -21,7 +21,6 @@ import java.util.HashMap; import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.ArgumentMatchers.any; @@ -134,18 +133,7 @@ void testUpdatedPollTimeNiCreatedLarger() { itemInfoList.add(testItem); when(client.listItems()).thenReturn(itemInfoList.iterator()); Instant updatedPollTime = crawler.crawl(lastPollTime, coordinator); - assertEquals(Instant.ofEpochMilli(10), updatedPollTime); - } - - @Test - void testUpdatedPollTimeNiUpdatedLarger() { - Instant lastPollTime = Instant.ofEpochMilli(0); - List itemInfoList = new ArrayList<>(); - ItemInfo testItem = createTestItemInfo("1"); - itemInfoList.add(testItem); - when(client.listItems()).thenReturn(itemInfoList.iterator()); - Instant updatedPollTime = crawler.crawl(lastPollTime, coordinator); - assertEquals(Instant.ofEpochMilli(10), updatedPollTime); + assertNotEquals(lastPollTime, updatedPollTime); } diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/PartitionFactoryTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/PartitionFactoryTest.java index af60c1e98b..4d3ac034e6 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/PartitionFactoryTest.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/PartitionFactoryTest.java @@ -33,7 +33,7 @@ void testCreateLeaderPartition() { String sourceId = sourceIdentifier + "|" + LeaderPartition.PARTITION_TYPE; when(sourcePartitionStoreItem.getSourceIdentifier()).thenReturn(sourceId); - String state = "{\"last_poll_time\":1729391235717}"; + String state = "{\"last_poll_time\":1730273866.332000000}"; when(sourcePartitionStoreItem.getPartitionProgressState()).thenReturn(state); @@ -46,7 +46,7 @@ void testCreateLeaderPartition() { Optional progressState = leaderParition.getProgressState(); assertThat(progressState.isPresent(), equalTo(true)); - assertThat(progressState.get().getLastPollTime(), equalTo(Instant.ofEpochMilli(1729391235717L))); + assertThat(progressState.get().getLastPollTime().toEpochMilli(), equalTo(1730273866332L)); //Update leader progress state and then verify LeaderProgressState updatedState = new LeaderProgressState(Instant.ofEpochMilli(12345L)); diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/state/LeaderProgressStateTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/state/LeaderProgressStateTest.java index ed6b10ef78..713f1c2117 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/state/LeaderProgressStateTest.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/state/LeaderProgressStateTest.java @@ -1,11 +1,11 @@ package org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state; +import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import org.junit.jupiter.api.Test; -import java.time.Instant; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; @@ -14,7 +14,8 @@ public class LeaderProgressStateTest { - private static final ObjectMapper objectMapper = new ObjectMapper(); + private static final ObjectMapper objectMapper = new ObjectMapper(new JsonFactory()) + .registerModule(new JavaTimeModule()); @Test void testDefaultValues() throws JsonProcessingException { @@ -28,7 +29,7 @@ void testDefaultValues() throws JsonProcessingException { void testInitializedValues() throws JsonProcessingException { String state = "{\"last_poll_time\":1729391235717, \"initialized\": true}"; LeaderProgressState leaderProgressState = objectMapper.readValue(state, LeaderProgressState.class); - assertEquals(Instant.ofEpochMilli(1729391235717L), leaderProgressState.getLastPollTime()); + assertEquals(1729391235717000L, leaderProgressState.getLastPollTime().toEpochMilli()); assertTrue(leaderProgressState.isInitialized()); } } diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/state/SaasWorkerProgressStateTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/state/SaasWorkerProgressStateTest.java index c7a4f48a43..c959cae259 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/state/SaasWorkerProgressStateTest.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/state/SaasWorkerProgressStateTest.java @@ -6,8 +6,6 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import org.junit.jupiter.api.Test; -import java.time.Instant; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -37,7 +35,7 @@ void testInitializedValues() throws JsonProcessingException { assertEquals(10, workderProgressState.getTotalItems()); assertEquals(20, workderProgressState.getLoadedItems()); assertNotNull(workderProgressState.getKeyAttributes()); - assertEquals(workderProgressState.getExportStartTime(), Instant.ofEpochMilli(1729391235717L)); + assertEquals(1729391235717000L, workderProgressState.getExportStartTime().toEpochMilli()); assertNotNull(workderProgressState.getItemIds()); assertEquals(2, workderProgressState.getItemIds().size()); } diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/model/TestItemInfo.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/model/TestItemInfo.java index 6e1a195da3..cb1ce2cb68 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/model/TestItemInfo.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/model/TestItemInfo.java @@ -45,7 +45,7 @@ public String getId() { } @Override - public Map getKeyAttributes() { + public Map getKeyAttributes() { return Map.of(); }