From 8b674cd1bd26447685e0739454ce134c5d2dae50 Mon Sep 17 00:00:00 2001 From: Taylor Gray Date: Wed, 25 Oct 2023 09:52:13 -0500 Subject: [PATCH] Add obfuscate_when parameter and tags_on_match failure to obfuscate processor (#3544) Add obfuscate_when parameter to obfuscate processor Signed-off-by: Taylor Gray --- .../obfuscation/ObfuscationProcessor.java | 24 +++++- .../ObfuscationProcessorConfig.java | 25 ++++++- .../obfuscation/ObfuscationProcessorTest.java | 75 +++++++++++++++---- 3 files changed, 107 insertions(+), 17 deletions(-) diff --git a/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessor.java b/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessor.java index f349ce0ebc..0ffc309797 100644 --- a/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessor.java +++ b/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessor.java @@ -5,6 +5,7 @@ package org.opensearch.dataprepper.plugins.processor.obfuscation; +import org.opensearch.dataprepper.expression.ExpressionEvaluator; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; @@ -33,6 +34,10 @@ public class ObfuscationProcessor extends AbstractProcessor, Recor private static final String COMMON_PATTERN_REGEX = "^%\\{([A-Z_0-9]+)}$"; private static final Logger LOG = LoggerFactory.getLogger(ObfuscationProcessor.class); + + private final ExpressionEvaluator expressionEvaluator; + private final ObfuscationProcessorConfig obfuscationProcessorConfig; + private final String source; private final String target; @@ -41,13 +46,20 @@ public class ObfuscationProcessor extends AbstractProcessor, Recor @DataPrepperPluginConstructor - public ObfuscationProcessor(final PluginMetrics pluginMetrics, final ObfuscationProcessorConfig config, final PluginFactory pluginFactory) { + public ObfuscationProcessor(final PluginMetrics pluginMetrics, + final ObfuscationProcessorConfig config, + final PluginFactory pluginFactory, + final ExpressionEvaluator expressionEvaluator) { // No special metrics generate by this processor. super(pluginMetrics); this.source = config.getSource(); this.target = config.getTarget(); this.patterns = new ArrayList<>(); + this.expressionEvaluator = expressionEvaluator; + this.obfuscationProcessorConfig = config; + + config.validateObfuscateWhen(expressionEvaluator); final PluginModel actionPlugin = config.getAction(); if (actionPlugin == null) { @@ -94,14 +106,24 @@ public Collection> doExecute(Collection> records) { for (final Record record : records) { final Event recordEvent = record.getData(); + if (obfuscationProcessorConfig.getObfuscateWhen() != null && !expressionEvaluator.evaluateConditional(obfuscationProcessorConfig.getObfuscateWhen(), recordEvent)) { + continue; + } + if (!recordEvent.containsKey(source)) { continue; } String rawValue = recordEvent.get(source, String.class); + // Call obfuscation action String newValue = this.action.obfuscate(rawValue, patterns); + // No changes means it does not match any patterns + if (rawValue.equals(newValue)) { + recordEvent.getMetadata().addTags(obfuscationProcessorConfig.getTagsOnMatchFailure()); + } + // Update the event record. if (target == null || target.isEmpty()) { recordEvent.put(source, newValue); diff --git a/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java b/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java index 714f41cb85..56defb6baf 100644 --- a/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java +++ b/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java @@ -8,7 +8,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; +import org.opensearch.dataprepper.expression.ExpressionEvaluator; import org.opensearch.dataprepper.model.configuration.PluginModel; +import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import java.util.List; @@ -28,14 +30,21 @@ public class ObfuscationProcessorConfig { @JsonProperty("action") private PluginModel action; + @JsonProperty("obfuscate_when") + private String obfuscateWhen; + + @JsonProperty("tags_on_match_failure") + private List tagsOnMatchFailure; + public ObfuscationProcessorConfig() { } - public ObfuscationProcessorConfig(String source, List patterns, String target, PluginModel action) { + public ObfuscationProcessorConfig(String source, List patterns, String target, PluginModel action, List tagsOnMatchFailure) { this.source = source; this.patterns = patterns; this.target = target; this.action = action; + this.tagsOnMatchFailure = tagsOnMatchFailure; } public String getSource() { @@ -53,4 +62,18 @@ public String getTarget() { public PluginModel getAction() { return action; } + + public String getObfuscateWhen() { + return obfuscateWhen; + } + + public List getTagsOnMatchFailure() { + return tagsOnMatchFailure; + } + + void validateObfuscateWhen(final ExpressionEvaluator expressionEvaluator) { + if (obfuscateWhen != null && !expressionEvaluator.isValidExpressionStatement(obfuscateWhen)) { + throw new InvalidPluginConfigurationException(String.format("obfuscate_when value %s is not a valid Data Prepper expression statement", obfuscateWhen)); + } + } } diff --git a/data-prepper-plugins/obfuscate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorTest.java b/data-prepper-plugins/obfuscate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorTest.java index 50e232d75d..b29ad3b0f4 100644 --- a/data-prepper-plugins/obfuscate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorTest.java +++ b/data-prepper-plugins/obfuscate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorTest.java @@ -13,6 +13,7 @@ import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.expression.ExpressionEvaluator; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.model.configuration.PluginSetting; @@ -27,9 +28,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -53,6 +56,9 @@ class ObfuscationProcessorTest { @Mock private ObfuscationProcessorConfig mockConfig; + @Mock + private ExpressionEvaluator expressionEvaluator; + private ObfuscationProcessor obfuscationProcessor; static Record buildRecordWithEvent(final Map data) { @@ -70,12 +76,51 @@ private Record createRecord(String message) { @BeforeEach void setup() { - final ObfuscationProcessorConfig defaultConfig = new ObfuscationProcessorConfig("message", null, null, null); + final ObfuscationProcessorConfig defaultConfig = new ObfuscationProcessorConfig("message", null, null, null, null); lenient().when(mockConfig.getSource()).thenReturn(defaultConfig.getSource()); lenient().when(mockConfig.getAction()).thenReturn(defaultConfig.getAction()); lenient().when(mockConfig.getPatterns()).thenReturn(defaultConfig.getPatterns()); lenient().when(mockConfig.getTarget()).thenReturn(defaultConfig.getTarget()); - obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory); + lenient().when(mockConfig.getObfuscateWhen()).thenReturn(null); + lenient().when(mockConfig.getTagsOnMatchFailure()).thenReturn(List.of(UUID.randomUUID().toString())); + obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator); + } + + @Test + void obfuscate_when_evaluates_to_false_does_not_modify_event() { + final String expression = "/test == success"; + final Record record = createRecord(UUID.randomUUID().toString()); + when(mockConfig.getObfuscateWhen()).thenReturn(expression); + when(expressionEvaluator.evaluateConditional(expression, record.getData())).thenReturn(false); + + final ObfuscationProcessor objectUnderTest = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator); + + final Map expectedEventMap = record.getData().toMap(); + final List> editedRecords = (List>) objectUnderTest.doExecute(Collections.singletonList(record)); + + assertThat(editedRecords.size(), equalTo(1)); + assertThat(editedRecords.get(0).getData().toMap(), equalTo(expectedEventMap)); + } + + @Test + void event_is_tagged_with_match_failure_tags_when_it_does_not_match_any_patterns_and_when_condition_is_true() { + final Record record = createRecord(UUID.randomUUID().toString()); + + final String expression = UUID.randomUUID().toString(); + when(mockConfig.getObfuscateWhen()).thenReturn(expression); + when(expressionEvaluator.evaluateConditional(expression, record.getData())).thenReturn(true); + when(mockConfig.getPatterns()).thenReturn(List.of(UUID.randomUUID().toString())); + + final ObfuscationProcessor objectUnderTest = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator); + + final Map expectedEventMap = record.getData().toMap(); + final List> editedRecords = (List>) objectUnderTest.doExecute(Collections.singletonList(record)); + + assertThat(editedRecords.size(), equalTo(1)); + assertThat(editedRecords.get(0).getData().toMap(), equalTo(expectedEventMap)); + assertThat(editedRecords.get(0).getData().getMetadata().getTags(), notNullValue()); + assertThat(editedRecords.get(0).getData().getMetadata().getTags().size(), equalTo(1)); + assertThat(editedRecords.get(0).getData().getMetadata().getTags().contains(mockConfig.getTagsOnMatchFailure().get(0)), equalTo(true)); } @@ -102,7 +147,7 @@ void testProcessorWithDifferentAction() { when(mockFactory.loadPlugin(eq(ObfuscationAction.class), any(PluginSetting.class))) .thenReturn(mockAction); - obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory); + obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator); final Record record = createRecord("Hello"); final List> editedRecords = (List>) obfuscationProcessor.doExecute(Collections.singletonList(record)); @@ -117,7 +162,7 @@ void testProcessorWithDifferentAction() { @ValueSource(strings = {"hello", "hello, world", "This is a message", "123", "你好"}) void testProcessorWithTarget(String message) { when(mockConfig.getTarget()).thenReturn("new_message"); - obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory); + obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator); final Record record = createRecord(message); final List> editedRecords = (List>) obfuscationProcessor.doExecute(Collections.singletonList(record)); @@ -135,7 +180,7 @@ void testProcessorWithTarget(String message) { @Test void testProcessorWithUnknownSource() { when(mockConfig.getSource()).thenReturn("email"); - obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory); + obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator); final Record record = createRecord("Hello"); final List> editedRecords = (List>) obfuscationProcessor.doExecute(Collections.singletonList(record)); @@ -158,7 +203,7 @@ void testProcessorWithUnknownSource() { }) void testProcessorWithPattern(String message, String pattern, String expected) { when(mockConfig.getPatterns()).thenReturn(List.of(pattern)); - obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory); + obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator); final Record record = createRecord(message); final List> editedRecords = (List>) obfuscationProcessor.doExecute(Collections.singletonList(record)); @@ -171,13 +216,13 @@ void testProcessorWithPattern(String message, String pattern, String expected) { @Test void testProcessorWithUnknownPattern() { when(mockConfig.getPatterns()).thenReturn(List.of("%{UNKNOWN}")); - assertThrows(InvalidPluginConfigurationException.class, () -> new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory)); + assertThrows(InvalidPluginConfigurationException.class, () -> new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator)); } @Test void testProcessorInvalidPattern() { when(mockConfig.getPatterns()).thenReturn(List.of("[")); - assertThrows(InvalidPluginConfigurationException.class, () -> new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory)); + assertThrows(InvalidPluginConfigurationException.class, () -> new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator)); } @ParameterizedTest @@ -194,7 +239,7 @@ void testProcessorInvalidPattern() { }) void testProcessorWithEmailAddressPattern(String message, String expected) { when(mockConfig.getPatterns()).thenReturn(List.of("%{EMAIL_ADDRESS}")); - obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory); + obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator); final Record record = createRecord(message); @@ -221,7 +266,7 @@ void testProcessorWithEmailAddressPattern(String message, String expected) { }) void testProcessorWithUSPhoneNumberPattern(String message, String expected) { when(mockConfig.getPatterns()).thenReturn(List.of("%{US_PHONE_NUMBER}")); - obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory); + obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator); final Record record = createRecord(message); @@ -247,7 +292,7 @@ void testProcessorWithUSPhoneNumberPattern(String message, String expected) { }) void testProcessorWithCreditNumberPattern(String message, String expected) { when(mockConfig.getPatterns()).thenReturn(List.of("%{CREDIT_CARD_NUMBER}")); - obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory); + obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator); final Record record = createRecord(message); @@ -271,7 +316,7 @@ void testProcessorWithCreditNumberPattern(String message, String expected) { }) void testProcessorWithIPAddressV4Pattern(String message, String expected) { when(mockConfig.getPatterns()).thenReturn(List.of("%{IP_ADDRESS_V4}")); - obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory); + obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator); final Record record = createRecord(message); final List> editedRecords = (List>) obfuscationProcessor.doExecute(Collections.singletonList(record)); @@ -291,7 +336,7 @@ void testProcessorWithIPAddressV4Pattern(String message, String expected) { }) void testProcessorWithUSSSNPattern(String message, String expected) { when(mockConfig.getPatterns()).thenReturn(List.of("%{US_SSN_NUMBER}")); - obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory); + obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator); final Record record = createRecord(message); final List> editedRecords = (List>) obfuscationProcessor.doExecute(Collections.singletonList(record)); @@ -314,7 +359,7 @@ void testProcessorWithUSSSNPattern(String message, String expected) { }) void testProcessorWithBaseNumberPattern(String message, String expected) { when(mockConfig.getPatterns()).thenReturn(List.of("%{BASE_NUMBER}")); - obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory); + obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator); final Record record = createRecord(message); final List> editedRecords = (List>) obfuscationProcessor.doExecute(Collections.singletonList(record)); @@ -333,7 +378,7 @@ void testProcessorWithBaseNumberPattern(String message, String expected) { }) void testProcessorWithMultiplePatterns(String message, String expected) { when(mockConfig.getPatterns()).thenReturn(List.of("%{EMAIL_ADDRESS}", "%{IP_ADDRESS_V4}")); - obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory); + obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator); final Record record = createRecord(message); final List> editedRecords = (List>) obfuscationProcessor.doExecute(Collections.singletonList(record));