Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ism index patterns based on indexAlias/index #5117 #5118

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

ism index patterns based on indexAlias/index #5117 #5118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d06fcb8
ism index patterns based on indexAlias/index #5117
kranthikirang Oct 26, 2024
6739a56
fixed integration tests with indexAlias
kranthikirang Nov 15, 2024
20b9749
change the policy test template to have dummy index-patterns
kranthikirang Nov 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions ...n/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/AbstractIndexManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ public void setupIndex() throws IOException {
private void checkAndCreateIndexTemplate() throws IOException {
final boolean isISMEnabled = checkISMEnabled();
final Optional<String> policyIdOptional = isISMEnabled ?
ismPolicyManagementStrategy.checkAndCreatePolicy() :
ismPolicyManagementStrategy.checkAndCreatePolicy(configuredIndexAlias) :
kranthikirang marked this conversation as resolved.
Show resolved Hide resolved
Optional.empty();
if (!openSearchSinkConfiguration.getIndexConfiguration().getIndexTemplate().isEmpty()) {
checkAndCreateIndexTemplate(isISMEnabled, policyIdOptional.orElse(null));
Expand Down Expand Up @@ -258,8 +258,8 @@ final void checkAndCreateIndexTemplate(final boolean isISMEnabled, final String
templateStrategy.createTemplate(indexTemplate);
}

final Optional<String> checkAndCreatePolicy() throws IOException {
return ismPolicyManagementStrategy.checkAndCreatePolicy();
final Optional<String> checkAndCreatePolicy(final String indexAlias) throws IOException {
return ismPolicyManagementStrategy.checkAndCreatePolicy(indexAlias);
}

public void checkAndCreateIndex() throws IOException {
Expand Down Expand Up @@ -322,6 +322,7 @@ private void attachPolicy(
indexTemplate.putCustomSetting(IndexConstants.ISM_POLICY_ID_SETTING, ismPolicyId);
}
indexTemplate.putCustomSetting(IndexConstants.ISM_ROLLOVER_ALIAS_SETTING, rolloverAlias);
indexTemplate.putCustomSetting(IndexConstants.PLUGINS_ROLLOVER_ALIAS_SETTING, rolloverAlias);
}

}
1 change: 1 addition & 0 deletions ...rc/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ public class IndexConstants {
public static final String ISM_ENABLED_SETTING = "opendistro.index_state_management.enabled";
public static final String ISM_POLICY_ID_SETTING = "opendistro.index_state_management.policy_id";
public static final String ISM_ROLLOVER_ALIAS_SETTING = "opendistro.index_state_management.rollover_alias";
public static final String PLUGINS_ROLLOVER_ALIAS_SETTING = "plugins.index_state_management.rollover_alias";
// TODO: extract out version number into version enum
public static final String SERVICE_MAP_DEFAULT_TEMPLATE_FILE = "otel-v1-apm-service-map-index-template.json";

Expand Down
12 changes: 11 additions & 1 deletion ...in/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IsmPolicyManagement.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import io.micrometer.core.instrument.util.StringUtils;
import org.opensearch.client.Request;
import org.opensearch.client.ResponseException;
Expand Down Expand Up @@ -93,10 +94,19 @@ public IsmPolicyManagement(final OpenSearchClient openSearchClient,
}

@Override
public Optional<String> checkAndCreatePolicy() throws IOException {
public Optional<String> checkAndCreatePolicy(final String indexAlias) throws IOException {
final String policyManagementEndpoint = POLICY_MANAGEMENT_ENDPOINT + policyName;

String policyJsonString = retrievePolicyJsonString(policyFile);
if(!indexAlias.isEmpty()) {
kranthikirang marked this conversation as resolved.
Show resolved Hide resolved
final ObjectMapper mapper = new ObjectMapper();
final JsonNode jsonNode = mapper.readTree(policyJsonString);
final ArrayNode iparray = mapper.createArrayNode();
iparray.add(indexAlias + "*");
((ObjectNode) jsonNode.get("policy").get("ism_template")).put("index_patterns", iparray);
policyJsonString = jsonNode.toString();
}
LOG.debug("Got the policystring as {} and indexAlias as {}", policyJsonString, indexAlias);
Request request = createPolicyRequestFromFile(policyManagementEndpoint, policyJsonString);

try {
Expand Down
2 changes: 1 addition & 1 deletion ...org/opensearch/dataprepper/plugins/sink/opensearch/index/IsmPolicyManagementStrategy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

interface IsmPolicyManagementStrategy {

Optional<String> checkAndCreatePolicy() throws IOException;
Optional<String> checkAndCreatePolicy(final String indexAlias) throws IOException;
kranthikirang marked this conversation as resolved.
Show resolved Hide resolved

List<String> getIndexPatterns(final String indexAlias);

Expand Down
2 changes: 1 addition & 1 deletion .../java/org/opensearch/dataprepper/plugins/sink/opensearch/index/NoIsmPolicyManagement.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public NoIsmPolicyManagement(final OpenSearchClient openSearchClient,
}

@Override
public Optional<String> checkAndCreatePolicy() throws IOException {
public Optional<String> checkAndCreatePolicy(final String indexAlias) throws IOException {
return Optional.empty();
}

Expand Down
6 changes: 3 additions & 3 deletions ...va/org/opensearch/dataprepper/plugins/sink/opensearch/index/DefaultIndexManagerTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -412,7 +412,7 @@ void checkAndCreatePolicy_Normal() throws IOException {
defaultIndexManager = indexManagerFactory.getIndexManager(
IndexType.CUSTOM, openSearchClient, restHighLevelClient, openSearchSinkConfiguration, templateStrategy);
when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
assertEquals(Optional.empty(), defaultIndexManager.checkAndCreatePolicy());
assertEquals(Optional.empty(), defaultIndexManager.checkAndCreatePolicy(INDEX_ALIAS));
verify(restHighLevelClient).getLowLevelClient();
verify(restClient).performRequest(any());
verify(openSearchSinkConfiguration, times(4)).getIndexConfiguration();
Expand All @@ -428,7 +428,7 @@ void checkAndCreatePolicy_Exception() throws IOException {
when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
when(restClient.performRequest(any())).thenThrow(responseException);
when(responseException.getMessage()).thenReturn("Invalid field: [ism_template]");
assertThrows(ResponseException.class, () -> defaultIndexManager.checkAndCreatePolicy());
assertThrows(ResponseException.class, () -> defaultIndexManager.checkAndCreatePolicy(INDEX_ALIAS));
verify(restHighLevelClient, times(2)).getLowLevelClient();
verify(restClient, times(2)).performRequest(any());
verify(openSearchSinkConfiguration, times(2)).getIndexConfiguration();
Expand All @@ -441,7 +441,7 @@ void checkAndCreatePolicy_Exception() throws IOException {
void checkAndCreatePolicy() throws IOException {
defaultIndexManager = indexManagerFactory.getIndexManager(
IndexType.CUSTOM, openSearchClient, restHighLevelClient, openSearchSinkConfiguration, templateStrategy);
assertEquals(Optional.empty(), defaultIndexManager.checkAndCreatePolicy());
assertEquals(Optional.empty(), defaultIndexManager.checkAndCreatePolicy(INDEX_ALIAS));
verify(indexConfiguration).getIndexAlias();
verify(openSearchSinkConfiguration, times(2)).getIndexConfiguration();
verify(indexConfiguration).getIsmPolicyFile();
Expand Down
10 changes: 5 additions & 5 deletions ...va/org/opensearch/dataprepper/plugins/sink/opensearch/index/IsmPolicyManagementTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ public void constructor_NullRestClient() {
@Test
public void checkAndCreatePolicy_Normal() throws IOException {
when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
assertEquals(Optional.empty(), ismPolicyManagementStrategy.checkAndCreatePolicy());
assertEquals(Optional.empty(), ismPolicyManagementStrategy.checkAndCreatePolicy(INDEX_ALIAS));
verify(restHighLevelClient).getLowLevelClient();
verify(restClient).performRequest(any());
}
Expand All @@ -114,7 +114,7 @@ public void checkAndCreatePolicy_OnlyOnePolicyFile_TwoExceptions() throws IOExce
when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
when(restClient.performRequest(any())).thenThrow(responseException);
when(responseException.getMessage()).thenReturn("Invalid field: [ism_template]");
assertThrows(ResponseException.class, () -> ismPolicyManagementStrategy.checkAndCreatePolicy());
assertThrows(ResponseException.class, () -> ismPolicyManagementStrategy.checkAndCreatePolicy(INDEX_ALIAS));
verify(restHighLevelClient, times(2)).getLowLevelClient();
verify(restClient, times(2)).performRequest(any());
}
Expand All @@ -129,7 +129,7 @@ public void checkAndCreatePolicy_OnlyOnePolicyFile_FirstExceptionThenSucceeds()
when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
when(restClient.performRequest(any())).thenThrow(responseException).thenReturn(null);
when(responseException.getMessage()).thenReturn("Invalid field: [ism_template]");
assertEquals(Optional.of(POLICY_NAME), ismPolicyManagementStrategy.checkAndCreatePolicy());
assertEquals(Optional.of(POLICY_NAME), ismPolicyManagementStrategy.checkAndCreatePolicy(INDEX_ALIAS));
verify(restHighLevelClient, times(2)).getLowLevelClient();
verify(restClient, times(2)).performRequest(any());
}
Expand All @@ -156,7 +156,7 @@ public void checkAndCreatePolicy_with_custom_ism_policy_from_s3() throws IOExcep
when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
when(restClient.performRequest(any())).thenThrow(responseException).thenReturn(null);
when(responseException.getMessage()).thenReturn("Invalid field: [ism_template]");
assertEquals(Optional.of(POLICY_NAME), ismPolicyManagementStrategyWithTemplate.checkAndCreatePolicy());
assertEquals(Optional.of(POLICY_NAME), ismPolicyManagementStrategyWithTemplate.checkAndCreatePolicy(INDEX_ALIAS));
verify(restHighLevelClient, times(2)).getLowLevelClient();
verify(restClient, times(2)).performRequest(any());
}
Expand All @@ -166,7 +166,7 @@ public void checkAndCreatePolicy_ExceptionFirstThenSucceed() throws IOException
when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
when(restClient.performRequest(any())).thenThrow(responseException).thenReturn(null);
when(responseException.getMessage()).thenReturn("Invalid field: [ism_template]");
assertEquals(Optional.of(POLICY_NAME), ismPolicyManagementStrategy.checkAndCreatePolicy());
assertEquals(Optional.of(POLICY_NAME), ismPolicyManagementStrategy.checkAndCreatePolicy(INDEX_ALIAS));
verify(restHighLevelClient, times(2)).getLowLevelClient();
verify(restClient, times(2)).performRequest(any());
}
Expand Down
2 changes: 1 addition & 1 deletion .../org/opensearch/dataprepper/plugins/sink/opensearch/index/NoIsmPolicyManagementTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public void constructor_NullRestClient() {

@Test
public void checkAndCreatePolicy() throws IOException {
assertEquals(Optional.empty(), ismPolicyManagementStrategy.checkAndCreatePolicy());
assertEquals(Optional.empty(), ismPolicyManagementStrategy.checkAndCreatePolicy(INDEX_ALIAS));
}

@ParameterizedTest
Expand Down
4 changes: 2 additions & 2 deletions ...nsearch/dataprepper/plugins/sink/opensearch/index/TraceAnalyticsRawIndexManagerTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ void checkISMEnabled_False() throws IOException {
@Test
void checkAndCreatePolicy_Normal() throws IOException {
when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
assertEquals(Optional.empty(), traceAnalyticsRawIndexManager.checkAndCreatePolicy());
assertEquals(Optional.empty(), traceAnalyticsRawIndexManager.checkAndCreatePolicy(INDEX_ALIAS));
verify(openSearchSinkConfiguration).getIndexConfiguration();
verify(indexConfiguration).getIndexAlias();
verify(restHighLevelClient).getLowLevelClient();
Expand All @@ -189,7 +189,7 @@ void checkAndCreatePolicy_ExceptionFirstThenSucceeds() throws IOException {
when(restHighLevelClient.getLowLevelClient()).thenReturn(restClient);
when(restClient.performRequest(any())).thenThrow(responseException).thenReturn(null);
when(responseException.getMessage()).thenReturn("Invalid field: [ism_template]");
assertEquals(Optional.of("raw-span-policy"), traceAnalyticsRawIndexManager.checkAndCreatePolicy());
assertEquals(Optional.of("raw-span-policy"), traceAnalyticsRawIndexManager.checkAndCreatePolicy(INDEX_ALIAS));
verify(restHighLevelClient, times(2)).getLowLevelClient();
verify(restClient, times(2)).performRequest(any());
verify(openSearchSinkConfiguration).getIndexConfiguration();
Expand Down
2 changes: 1 addition & 1 deletion .../dataprepper/plugins/sink/opensearch/index/TraceAnalyticsServiceMapIndexManagerTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ void checkISMEnabledByDefault_False() throws IOException {

@Test
void checkAndCreatePolicy() throws IOException {
assertEquals(Optional.empty(), traceAnalyticsServiceMapIndexManager.checkAndCreatePolicy());
assertEquals(Optional.empty(), traceAnalyticsServiceMapIndexManager.checkAndCreatePolicy(INDEX_ALIAS));
verify(openSearchSinkConfiguration).getIndexConfiguration();
verify(indexConfiguration).getIndexAlias();
}
Expand Down
5 changes: 3 additions & 2 deletions data-prepper-plugins/opensearch/src/test/resources/test-custom-index-policy-file.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
}
],
"ism_template": {
"index_patterns": ["sink-custom-index-ism-test-alias-*"]
"priority": 100,
"index_patterns": ["dummy-pattern-*"]
}
}
}
}
Loading