Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Exclude keys #3055

Merged
merged 13 commits into from
Jul 27, 2023
Merged

Exclude keys #3055

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
b8c5fe5
readme and config
shenkw1 Jul 19, 2023
e19f5e9
reorganize code, impl exclude keys check
shenkw1 Jul 19, 2023
7d7cf03
rerun tests
shenkw1 Jul 21, 2023
c23c4c3
merging main
shenkw1 Jul 24, 2023
0760cea
Merge branch 'opensearch-project:main' into exclude-keys
shenkw1 Jul 24, 2023
4d965bc
exclude impl done
shenkw1 Jul 24, 2023
2a8e686
exclude keys tests and empty list check
shenkw1 Jul 24, 2023
901f20d
rerun checks
shenkw1 Jul 24, 2023
186952e
rerun checks
shenkw1 Jul 24, 2023
cd6980c
simplify include and exclude keys validation, add overlap check and t…
shenkw1 Jul 26, 2023
229ca7b
remove equals check logic (covered by overlap logic already)
shenkw1 Jul 26, 2023
1108ee7
reorganize code for logic clarity, fix overlap edge case
shenkw1 Jul 26, 2023
766b108
remove equal check and moved validation to private method
shenkw1 Jul 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 12 additions & 10 deletions ...rc/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,18 +100,20 @@ public KeyValueProcessor(final PluginMetrics pluginMetrics, final KeyValueProces
throw new PatternSyntaxException("delete_value_regex is not a valid regex string", keyValueProcessorConfig.getDeleteValueRegex(), -1);
}

if (keyValueProcessorConfig.getIncludeKeys() != null) {
if (keyValueProcessorConfig.getExcludeKeys() != null) {
if (keyValueProcessorConfig.getIncludeKeys().equals(keyValueProcessorConfig.getExcludeKeys())) {
if (!keyValueProcessorConfig.getIncludeKeys().isEmpty()) {
throw new IllegalStateException("Include keys and exclude keys set cannot be the same", null);
}
}

excludeKeysSet.addAll(keyValueProcessorConfig.getExcludeKeys());
if (keyValueProcessorConfig.getIncludeKeys().equals(keyValueProcessorConfig.getExcludeKeys())) {
shenkw1 marked this conversation as resolved.
Show resolved Hide resolved
if (!keyValueProcessorConfig.getIncludeKeys().isEmpty()) {
throw new IllegalArgumentException("Include keys and exclude keys set cannot be the same", null);
}
}

includeKeysSet.addAll(keyValueProcessorConfig.getIncludeKeys());
excludeKeysSet.addAll(keyValueProcessorConfig.getExcludeKeys());

includeKeysSet.addAll(keyValueProcessorConfig.getIncludeKeys());
Set<String> intersectionSet = new HashSet<String>(includeKeysSet);
if (intersectionSet.retainAll(excludeKeysSet)) {
shenkw1 marked this conversation as resolved.
Show resolved Hide resolved
if (!intersectionSet.isEmpty()) {
throw new IllegalArgumentException("Include keys and exclude keys set cannot have any overlap", null);
}
}

if (!validTransformOptionSet.contains(keyValueProcessorConfig.getTransformKey())) {
Expand Down
18 changes: 14 additions & 4 deletions ...st/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,8 +264,8 @@ void testIncludeKeysNoMatchKeyValueProcessor() {
}

@Test
void testIncludeKeysAsNullKeyValueProcessor() {
when(mockConfig.getIncludeKeys()).thenReturn(null);
void testIncludeKeysAsDefaultKeyValueProcessor() {
when(mockConfig.getIncludeKeys()).thenReturn(List.of());
keyValueProcessor = new KeyValueProcessor(pluginMetrics, mockConfig);

final Record<Event> record = getMessage("key1=value1&key2=value2");
Expand All @@ -292,8 +292,8 @@ void testExcludeKeysKeyValueProcessor() {
}

@Test
void testExcludeKeysAsNullKeyValueProcessor() {
when(mockConfig.getExcludeKeys()).thenReturn(null);
void testExcludeKeysAsDefaultKeyValueProcessor() {
when(mockConfig.getExcludeKeys()).thenReturn(List.of());
keyValueProcessor = new KeyValueProcessor(pluginMetrics, mockConfig);

final Record<Event> record = getMessage("key1=value1&key2=value2");
Expand All @@ -305,6 +305,16 @@ void testExcludeKeysAsNullKeyValueProcessor() {
assertThatKeyEquals(parsed_message, "key2", "value2");
}

shenkw1 marked this conversation as resolved.
Show resolved Hide resolved
@Test
void testIncludeExcludeKeysOverlapKeyValueProcessor() {
final List<String> includeKeys = List.of("key1", "key3");
final List<String> excludeKeys = List.of("key3");
when(mockConfig.getIncludeKeys()).thenReturn(includeKeys);
when(mockConfig.getExcludeKeys()).thenReturn(excludeKeys);

assertThrows(IllegalArgumentException.class, () -> new KeyValueProcessor(pluginMetrics, mockConfig));
}

@Test
void testCustomPrefixKvProcessor() {
when(mockConfig.getPrefix()).thenReturn("TEST_");
Expand Down
Loading