Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jira source fixes #5174

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import javax.inject.Named;
import java.time.Instant;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutorService;
Expand Down Expand Up @@ -38,7 +39,7 @@ public JiraIterator(final JiraService service,
@Override
public boolean hasNext() {
if (firstTime) {
log.info("Crawling has been started");
log.trace("Crawling has been started");
itemInfoQueue = service.getJiraEntities(sourceConfig, lastPollTime);
firstTime = false;
}
Expand All @@ -47,7 +48,11 @@ public boolean hasNext() {

@Override
public ItemInfo next() {
return this.itemInfoQueue.remove();
if (hasNext()) {
return this.itemInfoQueue.remove();
} else {
throw new NoSuchElementException();
}
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,31 +8,23 @@
import org.opensearch.dataprepper.plugins.source.jira.models.SearchResults;
import org.opensearch.dataprepper.plugins.source.jira.rest.JiraRestClient;
import org.opensearch.dataprepper.plugins.source.jira.utils.JiraConfigHelper;
import org.opensearch.dataprepper.plugins.source.jira.utils.JiraContentType;
import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo;
import org.springframework.util.CollectionUtils;

import javax.inject.Named;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Queue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.ISSUE_KEY;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.KEY;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.LIVE;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT_KEY;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.UPDATED;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants._PROJECT;
import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.CLOSING_ROUND_BRACKET;
import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.DELIMITER;
import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.GREATER_THAN_EQUALS;
Expand All @@ -54,7 +46,6 @@ public class JiraService {

public static final String CONTENT_TYPE = "ContentType";
private static final String SEARCH_RESULTS_FOUND = "searchResultsFound";
private static final Map<String, String> jiraProjectCache = new ConcurrentHashMap<>();

private final JiraSourceConfig jiraSourceConfig;
private final JiraRestClient jiraRestClient;
Expand All @@ -75,17 +66,10 @@ public JiraService(JiraSourceConfig jiraSourceConfig, JiraRestClient jiraRestCli
* @param timestamp timestamp.
*/
public Queue<ItemInfo> getJiraEntities(JiraSourceConfig configuration, Instant timestamp) {
log.info("Started to fetch entities");
log.trace("Started to fetch entities");
Queue<ItemInfo> itemInfoQueue = new ConcurrentLinkedQueue<>();
jiraProjectCache.clear();
searchForNewTicketsAndAddToQueue(configuration, timestamp, itemInfoQueue);
log.trace("Creating item information and adding in queue");
jiraProjectCache.keySet().forEach(key -> {
Map<String, Object> metadata = new HashMap<>();
metadata.put(CONTENT_TYPE, JiraContentType.PROJECT.getType());
ItemInfo itemInfo = createItemInfo(_PROJECT + key, metadata);
itemInfoQueue.add(itemInfo);
});
return itemInfoQueue;
}

Expand Down Expand Up @@ -125,13 +109,6 @@ private void searchForNewTicketsAndAddToQueue(JiraSourceConfig configuration, In
private void addItemsToQueue(List<IssueBean> issueList, Queue<ItemInfo> itemInfoQueue) {
issueList.forEach(issue -> {
itemInfoQueue.add(JiraItemInfo.builder().withEventTime(Instant.now()).withIssueBean(issue).build());

if (Objects.nonNull(((Map) issue.getFields().get(PROJECT)).get(KEY))) {
String projectKey = ((Map) issue.getFields().get(PROJECT)).get(KEY).toString();
if (!jiraProjectCache.containsKey(projectKey)) {
jiraProjectCache.put(projectKey, LIVE);
}
}
});
}

Expand Down Expand Up @@ -165,7 +142,7 @@ private StringBuilder createIssueFilterCriteria(JiraSourceConfig configuration,
.collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX)))
.append(CLOSING_ROUND_BRACKET);
}
log.info("Created issue filter criteria JiraQl query: {}", jiraQl);
log.trace("Created issue filter criteria JiraQl query: {}", jiraQl);
return jiraQl;
}

Expand All @@ -175,7 +152,7 @@ private StringBuilder createIssueFilterCriteria(JiraSourceConfig configuration,
* @param configuration Input Parameter
*/
private void validateProjectFilters(JiraSourceConfig configuration) {
log.info("Validating project filters");
log.trace("Validating project filters");
List<String> badFilters = new ArrayList<>();
Pattern regex = Pattern.compile("[^A-Z0-9]");
JiraConfigHelper.getProjectKeyFilter(configuration).forEach(projectFilter -> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import java.util.List;

import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2;
import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.RETRY_ATTEMPT;
import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.EXPAND_FIELD;
import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.EXPAND_VALUE;
Expand Down Expand Up @@ -69,10 +68,7 @@ public JiraRestClient(RestTemplate restTemplate, JiraAuthConfig authConfig) {
public SearchResults getAllIssues(StringBuilder jql, int startAt,
JiraSourceConfig configuration) {

String url = configuration.getAccountUrl() + REST_API_SEARCH;
if (configuration.getAuthType().equals(OAUTH2)) {
url = authConfig.getUrl() + REST_API_SEARCH;
}
String url = authConfig.getUrl() + REST_API_SEARCH;

URI uri = UriComponentsBuilder.fromHttpUrl(url)
.queryParam(MAX_RESULT, FIFTY)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,19 @@
public class JiraBasicAuthConfig implements JiraAuthConfig {

private final JiraSourceConfig jiraSourceConfig;
private String accountUrl;

public JiraBasicAuthConfig(JiraSourceConfig jiraSourceConfig) {
this.jiraSourceConfig = jiraSourceConfig;
accountUrl = jiraSourceConfig.getAccountUrl();
if (!accountUrl.endsWith("/")) {
accountUrl += "/";
}
}

@Override
public String getUrl() {
return jiraSourceConfig.getAccountUrl();
return accountUrl;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.UUID;

import static org.junit.jupiter.api.Assertions.assertFalse;
Expand Down Expand Up @@ -98,8 +99,8 @@ void testItemInfoQueueNotEmpty() {
jiraIterator.setCrawlerQWaitTimeMillis(1);
assertTrue(jiraIterator.hasNext());
assertNotNull(jiraIterator.next());
assertNotNull(jiraIterator.next());
assertFalse(jiraIterator.hasNext());
assertThrows(NoSuchElementException.class, () -> jiraIterator.next());
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ public void testGetJiraEntities() throws JsonProcessingException {

Instant timestamp = Instant.ofEpochSecond(0);
Queue<ItemInfo> itemInfoQueue = jiraService.getJiraEntities(jiraSourceConfig, timestamp);
assertEquals(mockIssues.size() + 1, itemInfoQueue.size());
assertEquals(mockIssues.size(), itemInfoQueue.size());
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ public void testGetAllIssuesBasic() throws JsonProcessingException {
JiraSourceConfig jiraSourceConfig = JiraServiceTest.createJiraConfiguration(BASIC, issueType, issueStatus, projectKey);
JiraRestClient jiraRestClient = new JiraRestClient(restTemplate, authConfig);
SearchResults mockSearchResults = mock(SearchResults.class);
when(authConfig.getUrl()).thenReturn("https://example.com");
doReturn(new ResponseEntity<>(mockSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class));
SearchResults results = jiraRestClient.getAllIssues(jql, 0, jiraSourceConfig);
assertNotNull(results);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.when;

@ExtendWith(MockitoExtension.class)
public class JiraBasicAuthConfigTest {
Expand All @@ -24,7 +25,14 @@ void setUp() {

@Test
void testGetUrl() {
assertEquals(jiraBasicAuthConfig.getUrl(), jiraSourceConfig.getAccountUrl());
String url = "https://example.com";
when(jiraSourceConfig.getAccountUrl()).thenReturn(url);
assertEquals(jiraBasicAuthConfig.getUrl(), url + '/');

String url2 = "https://example.com/";
when(jiraSourceConfig.getAccountUrl()).thenReturn(url2);
assertEquals(jiraBasicAuthConfig.getUrl(), url2);

}

@Test
Expand Down
Loading