Skip to content

Commit

Permalink
Add obfuscate_when parameter and tags_on_match failure to obfuscate p…
Browse files Browse the repository at this point in the history 
…rocessor (opensearch-project#3544)

Add obfuscate_when parameter to obfuscate processor

Signed-off-by: Taylor Gray <[email protected]>
  • Loading branch information
@graytaylor0
graytaylor0 authored Oct 25, 2023
1 parent a9e419a commit 8b674cd
Show file tree
Hide file tree
Showing 3 changed files with 107 additions and 17 deletions.
24 changes: 23 additions & 1 deletion ...n/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

package org.opensearch.dataprepper.plugins.processor.obfuscation;

import org.opensearch.dataprepper.expression.ExpressionEvaluator;
import org.opensearch.dataprepper.metrics.PluginMetrics;
import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor;
Expand Down Expand Up @@ -33,6 +34,10 @@ public class ObfuscationProcessor extends AbstractProcessor<Record<Event>, Recor

private static final String COMMON_PATTERN_REGEX = "^%\\{([A-Z_0-9]+)}$";
private static final Logger LOG = LoggerFactory.getLogger(ObfuscationProcessor.class);

private final ExpressionEvaluator expressionEvaluator;
private final ObfuscationProcessorConfig obfuscationProcessorConfig;

private final String source;
private final String target;

Expand All @@ -41,13 +46,20 @@ public class ObfuscationProcessor extends AbstractProcessor<Record<Event>, Recor


@DataPrepperPluginConstructor
public ObfuscationProcessor(final PluginMetrics pluginMetrics, final ObfuscationProcessorConfig config, final PluginFactory pluginFactory) {
public ObfuscationProcessor(final PluginMetrics pluginMetrics,
final ObfuscationProcessorConfig config,
final PluginFactory pluginFactory,
final ExpressionEvaluator expressionEvaluator) {
// No special metrics generate by this processor.
super(pluginMetrics);

this.source = config.getSource();
this.target = config.getTarget();
this.patterns = new ArrayList<>();
this.expressionEvaluator = expressionEvaluator;
this.obfuscationProcessorConfig = config;

config.validateObfuscateWhen(expressionEvaluator);

final PluginModel actionPlugin = config.getAction();
if (actionPlugin == null) {
Expand Down Expand Up @@ -94,14 +106,24 @@ public Collection<Record<Event>> doExecute(Collection<Record<Event>> records) {
for (final Record<Event> record : records) {
final Event recordEvent = record.getData();

if (obfuscationProcessorConfig.getObfuscateWhen() != null && !expressionEvaluator.evaluateConditional(obfuscationProcessorConfig.getObfuscateWhen(), recordEvent)) {
continue;
}

if (!recordEvent.containsKey(source)) {
continue;
}

String rawValue = recordEvent.get(source, String.class);

// Call obfuscation action
String newValue = this.action.obfuscate(rawValue, patterns);

// No changes means it does not match any patterns
if (rawValue.equals(newValue)) {
recordEvent.getMetadata().addTags(obfuscationProcessorConfig.getTagsOnMatchFailure());
}

// Update the event record.
if (target == null || target.isEmpty()) {
recordEvent.put(source, newValue);
Expand Down
25 changes: 24 additions & 1 deletion .../org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import org.opensearch.dataprepper.expression.ExpressionEvaluator;
import org.opensearch.dataprepper.model.configuration.PluginModel;
import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException;

import java.util.List;

Expand All @@ -28,14 +30,21 @@ public class ObfuscationProcessorConfig {
@JsonProperty("action")
private PluginModel action;

@JsonProperty("obfuscate_when")
private String obfuscateWhen;

@JsonProperty("tags_on_match_failure")
private List<String> tagsOnMatchFailure;

public ObfuscationProcessorConfig() {
}

public ObfuscationProcessorConfig(String source, List<String> patterns, String target, PluginModel action) {
public ObfuscationProcessorConfig(String source, List<String> patterns, String target, PluginModel action, List<String> tagsOnMatchFailure) {
this.source = source;
this.patterns = patterns;
this.target = target;
this.action = action;
this.tagsOnMatchFailure = tagsOnMatchFailure;
}

public String getSource() {
Expand All @@ -53,4 +62,18 @@ public String getTarget() {
public PluginModel getAction() {
return action;
}

public String getObfuscateWhen() {
return obfuscateWhen;
}

public List<String> getTagsOnMatchFailure() {
return tagsOnMatchFailure;
}

void validateObfuscateWhen(final ExpressionEvaluator expressionEvaluator) {
if (obfuscateWhen != null && !expressionEvaluator.isValidExpressionStatement(obfuscateWhen)) {
throw new InvalidPluginConfigurationException(String.format("obfuscate_when value %s is not a valid Data Prepper expression statement", obfuscateWhen));
}
}
}
75 changes: 60 additions & 15 deletions ...va/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opensearch.dataprepper.expression.ExpressionEvaluator;
import org.opensearch.dataprepper.metrics.PluginMetrics;
import org.opensearch.dataprepper.model.configuration.PluginModel;
import org.opensearch.dataprepper.model.configuration.PluginSetting;
Expand All @@ -27,9 +28,11 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
Expand All @@ -53,6 +56,9 @@ class ObfuscationProcessorTest {
@Mock
private ObfuscationProcessorConfig mockConfig;

@Mock
private ExpressionEvaluator expressionEvaluator;

private ObfuscationProcessor obfuscationProcessor;

static Record<Event> buildRecordWithEvent(final Map<String, Object> data) {
Expand All @@ -70,12 +76,51 @@ private Record<Event> createRecord(String message) {

@BeforeEach
void setup() {
final ObfuscationProcessorConfig defaultConfig = new ObfuscationProcessorConfig("message", null, null, null);
final ObfuscationProcessorConfig defaultConfig = new ObfuscationProcessorConfig("message", null, null, null, null);
lenient().when(mockConfig.getSource()).thenReturn(defaultConfig.getSource());
lenient().when(mockConfig.getAction()).thenReturn(defaultConfig.getAction());
lenient().when(mockConfig.getPatterns()).thenReturn(defaultConfig.getPatterns());
lenient().when(mockConfig.getTarget()).thenReturn(defaultConfig.getTarget());
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
lenient().when(mockConfig.getObfuscateWhen()).thenReturn(null);
lenient().when(mockConfig.getTagsOnMatchFailure()).thenReturn(List.of(UUID.randomUUID().toString()));
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);
}

@Test
void obfuscate_when_evaluates_to_false_does_not_modify_event() {
final String expression = "/test == success";
final Record<Event> record = createRecord(UUID.randomUUID().toString());
when(mockConfig.getObfuscateWhen()).thenReturn(expression);
when(expressionEvaluator.evaluateConditional(expression, record.getData())).thenReturn(false);

final ObfuscationProcessor objectUnderTest = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);

final Map<String, Object> expectedEventMap = record.getData().toMap();
final List<Record<Event>> editedRecords = (List<Record<Event>>) objectUnderTest.doExecute(Collections.singletonList(record));

assertThat(editedRecords.size(), equalTo(1));
assertThat(editedRecords.get(0).getData().toMap(), equalTo(expectedEventMap));
}

@Test
void event_is_tagged_with_match_failure_tags_when_it_does_not_match_any_patterns_and_when_condition_is_true() {
final Record<Event> record = createRecord(UUID.randomUUID().toString());

final String expression = UUID.randomUUID().toString();
when(mockConfig.getObfuscateWhen()).thenReturn(expression);
when(expressionEvaluator.evaluateConditional(expression, record.getData())).thenReturn(true);
when(mockConfig.getPatterns()).thenReturn(List.of(UUID.randomUUID().toString()));

final ObfuscationProcessor objectUnderTest = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);

final Map<String, Object> expectedEventMap = record.getData().toMap();
final List<Record<Event>> editedRecords = (List<Record<Event>>) objectUnderTest.doExecute(Collections.singletonList(record));

assertThat(editedRecords.size(), equalTo(1));
assertThat(editedRecords.get(0).getData().toMap(), equalTo(expectedEventMap));
assertThat(editedRecords.get(0).getData().getMetadata().getTags(), notNullValue());
assertThat(editedRecords.get(0).getData().getMetadata().getTags().size(), equalTo(1));
assertThat(editedRecords.get(0).getData().getMetadata().getTags().contains(mockConfig.getTagsOnMatchFailure().get(0)), equalTo(true));
}


Expand All @@ -102,7 +147,7 @@ void testProcessorWithDifferentAction() {

when(mockFactory.loadPlugin(eq(ObfuscationAction.class), any(PluginSetting.class)))
.thenReturn(mockAction);
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);

final Record<Event> record = createRecord("Hello");
final List<Record<Event>> editedRecords = (List<Record<Event>>) obfuscationProcessor.doExecute(Collections.singletonList(record));
Expand All @@ -117,7 +162,7 @@ void testProcessorWithDifferentAction() {
@ValueSource(strings = {"hello", "hello, world", "This is a message", "123", "你好"})
void testProcessorWithTarget(String message) {
when(mockConfig.getTarget()).thenReturn("new_message");
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);

final Record<Event> record = createRecord(message);
final List<Record<Event>> editedRecords = (List<Record<Event>>) obfuscationProcessor.doExecute(Collections.singletonList(record));
Expand All @@ -135,7 +180,7 @@ void testProcessorWithTarget(String message) {
@Test
void testProcessorWithUnknownSource() {
when(mockConfig.getSource()).thenReturn("email");
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);

final Record<Event> record = createRecord("Hello");
final List<Record<Event>> editedRecords = (List<Record<Event>>) obfuscationProcessor.doExecute(Collections.singletonList(record));
Expand All @@ -158,7 +203,7 @@ void testProcessorWithUnknownSource() {
})
void testProcessorWithPattern(String message, String pattern, String expected) {
when(mockConfig.getPatterns()).thenReturn(List.of(pattern));
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);

final Record<Event> record = createRecord(message);
final List<Record<Event>> editedRecords = (List<Record<Event>>) obfuscationProcessor.doExecute(Collections.singletonList(record));
Expand All @@ -171,13 +216,13 @@ void testProcessorWithPattern(String message, String pattern, String expected) {
@Test
void testProcessorWithUnknownPattern() {
when(mockConfig.getPatterns()).thenReturn(List.of("%{UNKNOWN}"));
assertThrows(InvalidPluginConfigurationException.class, () -> new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory));
assertThrows(InvalidPluginConfigurationException.class, () -> new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator));
}

@Test
void testProcessorInvalidPattern() {
when(mockConfig.getPatterns()).thenReturn(List.of("["));
assertThrows(InvalidPluginConfigurationException.class, () -> new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory));
assertThrows(InvalidPluginConfigurationException.class, () -> new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator));
}

@ParameterizedTest
Expand All @@ -194,7 +239,7 @@ void testProcessorInvalidPattern() {
})
void testProcessorWithEmailAddressPattern(String message, String expected) {
when(mockConfig.getPatterns()).thenReturn(List.of("%{EMAIL_ADDRESS}"));
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);

final Record<Event> record = createRecord(message);

Expand All @@ -221,7 +266,7 @@ void testProcessorWithEmailAddressPattern(String message, String expected) {
})
void testProcessorWithUSPhoneNumberPattern(String message, String expected) {
when(mockConfig.getPatterns()).thenReturn(List.of("%{US_PHONE_NUMBER}"));
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);

final Record<Event> record = createRecord(message);

Expand All @@ -247,7 +292,7 @@ void testProcessorWithUSPhoneNumberPattern(String message, String expected) {
})
void testProcessorWithCreditNumberPattern(String message, String expected) {
when(mockConfig.getPatterns()).thenReturn(List.of("%{CREDIT_CARD_NUMBER}"));
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);

final Record<Event> record = createRecord(message);

Expand All @@ -271,7 +316,7 @@ void testProcessorWithCreditNumberPattern(String message, String expected) {
})
void testProcessorWithIPAddressV4Pattern(String message, String expected) {
when(mockConfig.getPatterns()).thenReturn(List.of("%{IP_ADDRESS_V4}"));
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);

final Record<Event> record = createRecord(message);
final List<Record<Event>> editedRecords = (List<Record<Event>>) obfuscationProcessor.doExecute(Collections.singletonList(record));
Expand All @@ -291,7 +336,7 @@ void testProcessorWithIPAddressV4Pattern(String message, String expected) {
})
void testProcessorWithUSSSNPattern(String message, String expected) {
when(mockConfig.getPatterns()).thenReturn(List.of("%{US_SSN_NUMBER}"));
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);

final Record<Event> record = createRecord(message);
final List<Record<Event>> editedRecords = (List<Record<Event>>) obfuscationProcessor.doExecute(Collections.singletonList(record));
Expand All @@ -314,7 +359,7 @@ void testProcessorWithUSSSNPattern(String message, String expected) {
})
void testProcessorWithBaseNumberPattern(String message, String expected) {
when(mockConfig.getPatterns()).thenReturn(List.of("%{BASE_NUMBER}"));
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);

final Record<Event> record = createRecord(message);
final List<Record<Event>> editedRecords = (List<Record<Event>>) obfuscationProcessor.doExecute(Collections.singletonList(record));
Expand All @@ -333,7 +378,7 @@ void testProcessorWithBaseNumberPattern(String message, String expected) {
})
void testProcessorWithMultiplePatterns(String message, String expected) {
when(mockConfig.getPatterns()).thenReturn(List.of("%{EMAIL_ADDRESS}", "%{IP_ADDRESS_V4}"));
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);

final Record<Event> record = createRecord(message);
final List<Record<Event>> editedRecords = (List<Record<Event>>) obfuscationProcessor.doExecute(Collections.singletonList(record));
Expand Down

0 comments on commit 8b674cd

Please sign in to comment.