From 62e99ef741b0fc9b91b3465f6384d009d5c2067b Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Mon, 18 Sep 2023 21:48:02 -0700 Subject: [PATCH] FEAT: AWS secret extension (#3340) Signed-off-by: George Chen --- .../DataPrepperExtensionPlugin.java | 13 +- .../model/configuration/PluginSetting.java | 10 +- .../plugin/PluginConfigValueTranslator.java | 7 + .../configuration/PluginSettingsTests.java | 9 ++ .../plugin/DataPrepperExtensionPoints.java | 7 + .../DataPrepperScalarTypeDeserializer.java | 22 +++ .../dataprepper/plugin/ExtensionLoader.java | 3 +- ...ExtensionPluginConfigurationConverter.java | 27 +++- .../ExtensionPluginConfigurationResolver.java | 17 +- .../plugin/ObjectMapperConfiguration.java | 20 ++- .../plugin/PluginBeanFactoryProvider.java | 10 +- .../plugin/PluginConfigurationConverter.java | 10 +- .../plugin/SupportSecretString.java | 17 ++ .../dataprepper/plugin/VariableExpander.java | 54 +++++++ .../DataPrepperExtensionPointsTest.java | 57 ++++++- ...DataPrepperScalarTypeDeserializerTest.java | 53 ++++++ .../plugin/ExtensionLoaderTest.java | 4 +- ...nsionPluginConfigurationConverterTest.java | 43 +++-- ...ensionPluginConfigurationResolverTest.java | 6 +- .../plugin/PluginBeanFactoryProviderTest.java | 7 +- .../PluginConfigurationConverterTest.java | 7 +- .../plugin/VariableExpanderTest.java | 127 +++++++++++++++ .../plugins/test/TestExtensionWithConfig.java | 3 +- ...ine_configuration_with_test_extension1.yml | 13 ++ ...ine_configuration_with_test_extension2.yml | 11 ++ .../pipeline_configuration2.yml | 8 + ...line_configuration_with_test_extension.yml | 13 ++ data-prepper-plugins/aws-plugin/build.gradle | 4 + .../aws/AwsSecretExtensionProvider.java | 23 +++ .../aws/AwsSecretManagerConfiguration.java | 104 ++++++++++++ .../plugins/aws/AwsSecretPlugin.java | 30 ++++ .../plugins/aws/AwsSecretPluginConfig.java | 16 ++ ...AwsSecretsPluginConfigValueTranslator.java | 41 +++++ .../plugins/aws/AwsSecretsSupplier.java | 96 +++++++++++ .../plugins/aws/SecretsSupplier.java | 7 + .../aws/AwsSecretExtensionProviderTest.java | 51 ++++++ .../AwsSecretManagerConfigurationTest.java | 151 ++++++++++++++++++ .../aws/AwsSecretPluginConfigTest.java | 36 +++++ .../plugins/aws/AwsSecretPluginIT.java | 63 ++++++++ ...ecretsPluginConfigValueTranslatorTest.java | 63 ++++++++ .../plugins/aws/AwsSecretsSupplierTest.java | 130 +++++++++++++++ ...-secret-manager-configuration-default.yaml | 1 + ...t-manager-configuration-invalid-sts-1.yaml | 3 + ...t-manager-configuration-invalid-sts-2.yaml | 3 + ...t-manager-configuration-invalid-sts-3.yaml | 3 + ...nager-configuration-missing-secret-id.yaml | 2 + ...secret-manager-configuration-with-sts.yaml | 3 + .../test-aws-secret-plugin-config.yaml | 4 + .../opensearch/ConnectionConfiguration.java | 9 +- .../sink/opensearch/RetryConfiguration.java | 2 +- .../opensearch/index/IndexConfiguration.java | 4 +- 51 files changed, 1372 insertions(+), 55 deletions(-) create mode 100644 data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/PluginConfigValueTranslator.java create mode 100644 data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/DataPrepperScalarTypeDeserializer.java create mode 100644 data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/SupportSecretString.java create mode 100644 data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/VariableExpander.java create mode 100644 data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/DataPrepperScalarTypeDeserializerTest.java create mode 100644 data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/VariableExpanderTest.java create mode 100644 data-prepper-core/src/test/resources/multi-pipelines-distributed-pipeline-extensions/pipeline_configuration_with_test_extension1.yml create mode 100644 data-prepper-core/src/test/resources/multi-pipelines-distributed-pipeline-extensions/pipeline_configuration_with_test_extension2.yml create mode 100644 data-prepper-core/src/test/resources/multi-pipelines-single-pipeline-extensions/pipeline_configuration2.yml create mode 100644 data-prepper-core/src/test/resources/multi-pipelines-single-pipeline-extensions/pipeline_configuration_with_test_extension.yml create mode 100644 data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretExtensionProvider.java create mode 100644 data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretManagerConfiguration.java create mode 100644 data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretPlugin.java create mode 100644 data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretPluginConfig.java create mode 100644 data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsPluginConfigValueTranslator.java create mode 100644 data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsSupplier.java create mode 100644 data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/SecretsSupplier.java create mode 100644 data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretExtensionProviderTest.java create mode 100644 data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretManagerConfigurationTest.java create mode 100644 data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretPluginConfigTest.java create mode 100644 data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretPluginIT.java create mode 100644 data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsPluginConfigValueTranslatorTest.java create mode 100644 data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsSupplierTest.java create mode 100644 data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-default.yaml create mode 100644 data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-invalid-sts-1.yaml create mode 100644 data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-invalid-sts-2.yaml create mode 100644 data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-invalid-sts-3.yaml create mode 100644 data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-missing-secret-id.yaml create mode 100644 data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-with-sts.yaml create mode 100644 data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-plugin-config.yaml diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/DataPrepperExtensionPlugin.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/DataPrepperExtensionPlugin.java index fb115cd622..da191bcb98 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/DataPrepperExtensionPlugin.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/DataPrepperExtensionPlugin.java @@ -6,11 +6,22 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +/** + * Annotates a Data Prepper extension plugin which includes a configuration model class. + */ @Documented @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.TYPE}) public @interface DataPrepperExtensionPlugin { + /** + * @return extension plugin configuration class. + */ Class modelType(); - String rootKey(); + /** + * @return valid JSON path string starts with "/" pointing towards the configuration block. + */ + String rootKeyJsonPath(); + + boolean allowInPipelineConfigurations() default false; } diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginSetting.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginSetting.java index e431f144cc..ab7e16455e 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginSetting.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginSetting.java @@ -14,7 +14,7 @@ public class PluginSetting implements PipelineDescription { private static final String UNEXPECTED_ATTRIBUTE_TYPE_MSG = "Unexpected type [%s] for attribute [%s]"; private final String name; - private final Map settings; + private Map settings; private int processWorkers; private String pipelineName; @@ -31,6 +31,10 @@ public Map getSettings() { return settings; } + public void setSettings(final Map settings) { + this.settings = settings; + } + /** * Returns the number of process workers the pipeline is using; This is only required for special plugin use-cases * where plugin implementation depends on the number of process workers. For example, Trace analytics service map @@ -99,7 +103,7 @@ public Object getAttributeOrDefault(final String attribute, final Object default * @return the value of the specified attribute, or {@code defaultValue} if this settings contains no value for * the attribute. If the value is null, null will be returned. */ - public Integer getIntegerOrDefault(final String attribute, final int defaultValue) { + public Integer getIntegerOrDefault(final String attribute, final Integer defaultValue) { Object object = getAttributeOrDefault(attribute, defaultValue); if (object == null) { return null; @@ -218,7 +222,7 @@ public Map> getTypedListMap(final String attribute, final Clas * @return the value of the specified attribute, or {@code defaultValue} if this settings contains no value for * the attribute */ - public Boolean getBooleanOrDefault(final String attribute, final boolean defaultValue) { + public Boolean getBooleanOrDefault(final String attribute, final Boolean defaultValue) { Object object = getAttributeOrDefault(attribute, defaultValue); if (object == null) { return null; diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/PluginConfigValueTranslator.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/PluginConfigValueTranslator.java new file mode 100644 index 0000000000..2c3a20fbe6 --- /dev/null +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/PluginConfigValueTranslator.java @@ -0,0 +1,7 @@ +package org.opensearch.dataprepper.model.plugin; + +public interface PluginConfigValueTranslator { + Object translate(final String value); + + String getPrefix(); +} diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/PluginSettingsTests.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/PluginSettingsTests.java index fcbdeb7c5b..d1a058165d 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/PluginSettingsTests.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/PluginSettingsTests.java @@ -537,4 +537,13 @@ public void testGetLongOrDefault_UnsupportedType() { assertThrows(IllegalArgumentException.class, () -> pluginSetting.getLongOrDefault(TEST_LONG_ATTRIBUTE, TEST_LONG_DEFAULT_VALUE)); } + + @Test + public void testSetSettings() { + final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, null); + + final Map settings = Map.of("test", 1); + pluginSetting.setSettings(settings); + assertThat(pluginSetting.getSettings(), equalTo(settings)); + } } diff --git a/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/DataPrepperExtensionPoints.java b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/DataPrepperExtensionPoints.java index 64b712cf24..d97d5b30ce 100644 --- a/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/DataPrepperExtensionPoints.java +++ b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/DataPrepperExtensionPoints.java @@ -18,17 +18,24 @@ public class DataPrepperExtensionPoints implements ExtensionPoints { private static final ExtensionProvider.Context EMPTY_CONTEXT = new EmptyContext(); private final GenericApplicationContext sharedApplicationContext; + private final GenericApplicationContext coreApplicationContext; @Inject public DataPrepperExtensionPoints( final PluginBeanFactoryProvider pluginBeanFactoryProvider) { Objects.requireNonNull(pluginBeanFactoryProvider); + Objects.requireNonNull(pluginBeanFactoryProvider.getCoreApplicationContext()); Objects.requireNonNull(pluginBeanFactoryProvider.getSharedPluginApplicationContext()); this.sharedApplicationContext = pluginBeanFactoryProvider.getSharedPluginApplicationContext(); + this.coreApplicationContext = pluginBeanFactoryProvider.getCoreApplicationContext(); } @Override public void addExtensionProvider(final ExtensionProvider extensionProvider) { + coreApplicationContext.registerBean( + extensionProvider.supportedClass(), + () -> extensionProvider.provideInstance(EMPTY_CONTEXT).orElse(null), + b -> b.setScope(BeanDefinition.SCOPE_PROTOTYPE)); sharedApplicationContext.registerBean( extensionProvider.supportedClass(), () -> extensionProvider.provideInstance(EMPTY_CONTEXT), diff --git a/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/DataPrepperScalarTypeDeserializer.java b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/DataPrepperScalarTypeDeserializer.java new file mode 100644 index 0000000000..91fc8a6f76 --- /dev/null +++ b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/DataPrepperScalarTypeDeserializer.java @@ -0,0 +1,22 @@ +package org.opensearch.dataprepper.plugin; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; + +import java.io.IOException; + +public class DataPrepperScalarTypeDeserializer extends JsonDeserializer { + private final VariableExpander variableExpander; + private final Class scalarType; + + public DataPrepperScalarTypeDeserializer(final VariableExpander variableExpander, final Class scalarType) { + this.variableExpander = variableExpander; + this.scalarType = scalarType; + } + + @Override + public T deserialize(final JsonParser jsonParser, final DeserializationContext ctxt) throws IOException { + return variableExpander.translate(jsonParser, this.scalarType); + } +} diff --git a/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/ExtensionLoader.java b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/ExtensionLoader.java index abbf0c68d7..ebe42ee129 100644 --- a/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/ExtensionLoader.java +++ b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/ExtensionLoader.java @@ -49,8 +49,9 @@ private PluginArgumentsContext getConstructionContext(final Class extensionPl return new NoArgumentsArgumentsContext(); } else { final Class pluginConfigurationType = pluginAnnotation.modelType(); - final String rootKey = pluginAnnotation.rootKey(); + final String rootKey = pluginAnnotation.rootKeyJsonPath(); final Object configuration = extensionPluginConfigurationConverter.convert( + pluginAnnotation.allowInPipelineConfigurations(), pluginConfigurationType, rootKey); return new SingleConfigArgumentArgumentsContext(configuration); } diff --git a/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/ExtensionPluginConfigurationConverter.java b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/ExtensionPluginConfigurationConverter.java index e5cbb46134..df9573852b 100644 --- a/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/ExtensionPluginConfigurationConverter.java +++ b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/ExtensionPluginConfigurationConverter.java @@ -1,5 +1,8 @@ package org.opensearch.dataprepper.plugin; +import com.fasterxml.jackson.core.JsonPointer; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.validation.ConstraintViolation; import jakarta.validation.Validator; @@ -8,12 +11,14 @@ import javax.inject.Inject; import javax.inject.Named; import java.util.Collections; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @Named public class ExtensionPluginConfigurationConverter { + static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference<>() {}; private final ExtensionPluginConfigurationResolver extensionPluginConfigurationResolver; private final ObjectMapper objectMapper; private final Validator validator; @@ -22,19 +27,25 @@ public class ExtensionPluginConfigurationConverter { public ExtensionPluginConfigurationConverter( final ExtensionPluginConfigurationResolver extensionPluginConfigurationResolver, final Validator validator, - @Named("pluginConfigObjectMapper") + @Named("extensionPluginConfigObjectMapper") final ObjectMapper objectMapper) { this.extensionPluginConfigurationResolver = extensionPluginConfigurationResolver; this.objectMapper = objectMapper; this.validator = validator; } - public Object convert(final Class extensionPluginConfigurationType, final String rootKey) { + public Object convert(final boolean configAllowedInPipelineConfigurations, + final Class extensionPluginConfigurationType, final String rootKey) { Objects.requireNonNull(extensionPluginConfigurationType); Objects.requireNonNull(rootKey); - final Object configuration = convertSettings(extensionPluginConfigurationType, - extensionPluginConfigurationResolver.getExtensionMap().get(rootKey)); + final Object configuration = configAllowedInPipelineConfigurations ? + convertSettings(extensionPluginConfigurationType, + getExtensionPluginConfigMap( + extensionPluginConfigurationResolver.getCombinedExtensionMap(), rootKey)) : + convertSettings(extensionPluginConfigurationType, + getExtensionPluginConfigMap( + extensionPluginConfigurationResolver.getDataPrepperConfigExtensionMap(), rootKey)); final Set> constraintViolations = configuration == null ? Collections.emptySet() : validator.validate(configuration); @@ -54,4 +65,12 @@ public Object convert(final Class extensionPluginConfigurationType, final Str private Object convertSettings(final Class extensionPluginConfigurationType, final Object extensionPlugin) { return objectMapper.convertValue(extensionPlugin, extensionPluginConfigurationType); } + + private Map getExtensionPluginConfigMap( + final Map extensionMap, final String rootKey) { + final JsonNode jsonNode = objectMapper.valueToTree(extensionMap); + final JsonPointer jsonPointer = JsonPointer.compile(rootKey); + final JsonNode extensionPluginConfigNode = jsonNode.at(jsonPointer); + return objectMapper.convertValue(extensionPluginConfigNode, MAP_TYPE_REFERENCE); + } } diff --git a/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/ExtensionPluginConfigurationResolver.java b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/ExtensionPluginConfigurationResolver.java index c7688d5e4e..0b64ed10f1 100644 --- a/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/ExtensionPluginConfigurationResolver.java +++ b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/ExtensionPluginConfigurationResolver.java @@ -11,19 +11,26 @@ @Named public class ExtensionPluginConfigurationResolver { - private final Map extensionMap; + private final Map combinedExtensionMap; + + private final Map dataPrepperConfigExtensionMap; @Inject public ExtensionPluginConfigurationResolver(final DataPrepperConfiguration dataPrepperConfiguration, final PipelinesDataFlowModel pipelinesDataFlowModel) { - extensionMap = dataPrepperConfiguration.getPipelineExtensions() == null? + this.dataPrepperConfigExtensionMap = dataPrepperConfiguration.getPipelineExtensions() == null? new HashMap<>() : new HashMap<>(dataPrepperConfiguration.getPipelineExtensions().getExtensionMap()); + combinedExtensionMap = new HashMap<>(dataPrepperConfigExtensionMap); if (pipelinesDataFlowModel.getPipelineExtensions() != null) { - extensionMap.putAll(pipelinesDataFlowModel.getPipelineExtensions().getExtensionMap()); + combinedExtensionMap.putAll(pipelinesDataFlowModel.getPipelineExtensions().getExtensionMap()); } } - public Map getExtensionMap() { - return Collections.unmodifiableMap(extensionMap); + public Map getDataPrepperConfigExtensionMap() { + return Collections.unmodifiableMap(dataPrepperConfigExtensionMap); + } + + public Map getCombinedExtensionMap() { + return Collections.unmodifiableMap(combinedExtensionMap); } } diff --git a/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/ObjectMapperConfiguration.java b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/ObjectMapperConfiguration.java index fbc2765d38..0c5494591d 100644 --- a/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/ObjectMapperConfiguration.java +++ b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/ObjectMapperConfiguration.java @@ -8,14 +8,19 @@ import javax.inject.Named; import java.time.Duration; +import java.util.Set; /** * Application context for internal plugin framework beans. */ @Named public class ObjectMapperConfiguration { - @Bean(name = "pluginConfigObjectMapper") - ObjectMapper objectMapper() { + static final Set TRANSLATE_VALUE_SUPPORTED_JAVA_TYPES = Set.of( + String.class, Number.class, Long.class, Short.class, Integer.class, Double.class, Float.class, + Boolean.class, Duration.class, Enum.class, Character.class); + + @Bean(name = "extensionPluginConfigObjectMapper") + ObjectMapper extensionPluginConfigObjectMapper() { final SimpleModule simpleModule = new SimpleModule(); simpleModule.addDeserializer(Duration.class, new DataPrepperDurationDeserializer()); @@ -23,4 +28,15 @@ ObjectMapper objectMapper() { .setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE) .registerModule(simpleModule); } + + @Bean(name = "pluginConfigObjectMapper") + ObjectMapper pluginConfigObjectMapper(final VariableExpander variableExpander) { + final SimpleModule simpleModule = new SimpleModule(); + TRANSLATE_VALUE_SUPPORTED_JAVA_TYPES.stream().forEach(clazz -> simpleModule.addDeserializer( + clazz, new DataPrepperScalarTypeDeserializer<>(variableExpander, clazz))); + + return new ObjectMapper() + .setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE) + .registerModule(simpleModule); + } } diff --git a/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/PluginBeanFactoryProvider.java b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/PluginBeanFactoryProvider.java index a0e03cd16b..66a42eb36a 100644 --- a/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/PluginBeanFactoryProvider.java +++ b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/PluginBeanFactoryProvider.java @@ -27,11 +27,13 @@ @Named class PluginBeanFactoryProvider implements Provider { private final GenericApplicationContext sharedPluginApplicationContext; + private final GenericApplicationContext coreApplicationContext; @Inject - PluginBeanFactoryProvider(final ApplicationContext coreContext) { - final ApplicationContext publicContext = Objects.requireNonNull(coreContext.getParent()); + PluginBeanFactoryProvider(final GenericApplicationContext coreApplicationContext) { + final ApplicationContext publicContext = Objects.requireNonNull(coreApplicationContext.getParent()); sharedPluginApplicationContext = new GenericApplicationContext(publicContext); + this.coreApplicationContext = coreApplicationContext; } /** @@ -43,6 +45,10 @@ GenericApplicationContext getSharedPluginApplicationContext() { return sharedPluginApplicationContext; } + GenericApplicationContext getCoreApplicationContext() { + return coreApplicationContext; + } + /** * @since 1.3 * Creates a new isolated application context that inherits from diff --git a/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/PluginConfigurationConverter.java b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/PluginConfigurationConverter.java index 5f612993ce..d11b3f6e78 100644 --- a/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/PluginConfigurationConverter.java +++ b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/PluginConfigurationConverter.java @@ -5,12 +5,14 @@ package org.opensearch.dataprepper.plugin; +import com.fasterxml.jackson.core.type.TypeReference; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.validation.ConstraintViolation; import jakarta.validation.Validator; +import org.springframework.context.annotation.DependsOn; import javax.inject.Named; import java.util.Collections; @@ -24,7 +26,9 @@ * and converting it to the plugin model type which should be denoted by {@link DataPrepperPlugin#pluginConfigurationType()} */ @Named +@DependsOn({"extensionsApplier"}) class PluginConfigurationConverter { + static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference<>() {}; private final ObjectMapper objectMapper; private final Validator validator; @@ -48,8 +52,12 @@ public Object convert(final Class pluginConfigurationType, final PluginSettin Objects.requireNonNull(pluginConfigurationType); Objects.requireNonNull(pluginSetting); - if (pluginConfigurationType.equals(PluginSetting.class)) + if (pluginConfigurationType.equals(PluginSetting.class)) { + final Map settings = pluginSetting.getSettings(); + final Map convertedSettings = objectMapper.convertValue(settings, MAP_TYPE_REFERENCE); + pluginSetting.setSettings(convertedSettings); return pluginSetting; + } final Object configuration = convertSettings(pluginConfigurationType, pluginSetting); diff --git a/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/SupportSecretString.java b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/SupportSecretString.java new file mode 100644 index 0000000000..a009598f13 --- /dev/null +++ b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/SupportSecretString.java @@ -0,0 +1,17 @@ +package org.opensearch.dataprepper.plugin; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +/** + * This annotation can be added to any plugin configuration field that is supposed to support secret string value. + */ +@Documented +@Retention(RUNTIME) +@Target({ElementType.FIELD}) +public @interface SupportSecretString { +} diff --git a/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/VariableExpander.java b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/VariableExpander.java new file mode 100644 index 0000000000..c2dfe1c6b4 --- /dev/null +++ b/data-prepper-core/src/main/java/org/opensearch/dataprepper/plugin/VariableExpander.java @@ -0,0 +1,54 @@ +package org.opensearch.dataprepper.plugin; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.opensearch.dataprepper.model.plugin.PluginConfigValueTranslator; + +import javax.inject.Inject; +import javax.inject.Named; +import java.io.IOException; +import java.util.AbstractMap; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +@Named +public class VariableExpander { + static final String VALUE_REFERENCE_KEY = "valueReferenceKey"; + static final String SECRETS_REFERENCE_PATTERN_STRING = "^\\$\\{\\{%s\\:(?<%s>.*)\\}\\}$"; + private final Map patternPluginConfigValueTranslatorMap; + private final ObjectMapper objectMapper; + + @Inject + public VariableExpander( + @Named("extensionPluginConfigObjectMapper") + final ObjectMapper objectMapper, + final Set pluginConfigValueTranslators) { + this.objectMapper = objectMapper; + patternPluginConfigValueTranslatorMap = pluginConfigValueTranslators.stream().collect(Collectors.toMap( + pluginConfigValueTranslator -> Pattern.compile( + String.format(SECRETS_REFERENCE_PATTERN_STRING, + pluginConfigValueTranslator.getPrefix(), VALUE_REFERENCE_KEY)), + Function.identity())); + } + + public T translate(final JsonParser jsonParser, final Class destinationType) throws IOException { + if (JsonToken.VALUE_STRING.equals(jsonParser.currentToken())) { + final String rawValue = jsonParser.getValueAsString(); + return patternPluginConfigValueTranslatorMap.entrySet().stream() + .map(entry -> new AbstractMap.SimpleEntry<>(entry.getKey().matcher(rawValue), entry.getValue())) + .filter(entry -> entry.getKey().matches()) + .map(entry -> { + final String valueReferenceKey = entry.getKey().group(VALUE_REFERENCE_KEY); + return objectMapper.convertValue( + entry.getValue().translate(valueReferenceKey), destinationType); + }) + .findFirst() + .orElseGet(() -> objectMapper.convertValue(rawValue, destinationType)); + } + return objectMapper.readValue(jsonParser, destinationType); + } +} diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/DataPrepperExtensionPointsTest.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/DataPrepperExtensionPointsTest.java index 7647f8a0d1..d2a20cb07a 100644 --- a/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/DataPrepperExtensionPointsTest.java +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/DataPrepperExtensionPointsTest.java @@ -40,6 +40,9 @@ class DataPrepperExtensionPointsTest { @Mock(lenient = true) private GenericApplicationContext sharedApplicationContext; + @Mock(lenient = true) + private GenericApplicationContext coreApplicationContext; + @Mock(lenient = true) private ExtensionProvider extensionProvider; @@ -47,6 +50,8 @@ class DataPrepperExtensionPointsTest { @BeforeEach void setUp() { + when(pluginBeanFactoryProvider.getCoreApplicationContext()) + .thenReturn(coreApplicationContext); when(pluginBeanFactoryProvider.getSharedPluginApplicationContext()) .thenReturn(sharedApplicationContext); @@ -69,6 +74,17 @@ void constructor_throws_if_provider_is_null() { @Test void constructor_throws_if_provider_getSharedPluginApplicationContext_is_null() { reset(pluginBeanFactoryProvider); + when(pluginBeanFactoryProvider.getCoreApplicationContext()) + .thenReturn(coreApplicationContext); + + assertThrows(NullPointerException.class, this::createObjectUnderTest); + } + + @Test + void constructor_throws_if_provider_getCoreApplicationContext_is_null() { + reset(pluginBeanFactoryProvider); + when(pluginBeanFactoryProvider.getSharedPluginApplicationContext()) + .thenReturn(sharedApplicationContext); assertThrows(NullPointerException.class, this::createObjectUnderTest); } @@ -78,16 +94,31 @@ void addExtensionProvider_should_registerBean() { createObjectUnderTest().addExtensionProvider(extensionProvider); verify(sharedApplicationContext).registerBean(eq(extensionClass), any(Supplier.class), any(BeanDefinitionCustomizer.class)); + verify(coreApplicationContext).registerBean(eq(extensionClass), any(Supplier.class), any(BeanDefinitionCustomizer.class)); } @Test void addExtensionProvider_should_registerBean_which_calls_provideInstance() { createObjectUnderTest().addExtensionProvider(extensionProvider); + verifyRegisterBeanWithProvideInstance(sharedApplicationContext); + verifyRegisterBeanWithProvideInstanceOrElse(coreApplicationContext); + } + + @Test + void addExtensionProvider_should_registerBean_as_prototype() { + createObjectUnderTest().addExtensionProvider(extensionProvider); + + verifyRegisterBeanAsPrototype(sharedApplicationContext); + verifyRegisterBeanAsPrototype(coreApplicationContext); + } + + private void verifyRegisterBeanWithProvideInstance(final GenericApplicationContext applicationContext) { + reset(extensionProvider); final ArgumentCaptor> supplierArgumentCaptor = ArgumentCaptor.forClass(Supplier.class); - verify(sharedApplicationContext).registerBean(eq(extensionClass), supplierArgumentCaptor.capture(), any(BeanDefinitionCustomizer.class)); + verify(applicationContext).registerBean(eq(extensionClass), supplierArgumentCaptor.capture(), any(BeanDefinitionCustomizer.class)); final Supplier extensionProviderSupplier = supplierArgumentCaptor.getValue(); @@ -105,14 +136,30 @@ void addExtensionProvider_should_registerBean_which_calls_provideInstance() { verify(extensionProvider).provideInstance(any()); } - @Test - void addExtensionProvider_should_registerBean_as_prototype() { - createObjectUnderTest().addExtensionProvider(extensionProvider); + private void verifyRegisterBeanWithProvideInstanceOrElse(final GenericApplicationContext applicationContext) { + reset(extensionProvider); + final ArgumentCaptor> supplierArgumentCaptor = + ArgumentCaptor.forClass(Supplier.class); + + verify(applicationContext).registerBean(eq(extensionClass), supplierArgumentCaptor.capture(), any(BeanDefinitionCustomizer.class)); + + final Supplier extensionProviderSupplier = supplierArgumentCaptor.getValue(); + + verify(extensionProvider, never()).provideInstance(any()); + + Object providedInstance = mock(Object.class); + when(extensionProvider.provideInstance(any())).thenReturn(Optional.of(providedInstance)); + final Object actualValueFromSupplier = extensionProviderSupplier.get(); + + assertThat(actualValueFromSupplier, equalTo(providedInstance)); + verify(extensionProvider).provideInstance(any()); + } + private void verifyRegisterBeanAsPrototype(final GenericApplicationContext applicationContext) { final ArgumentCaptor beanDefinitionCustomizerArgumentCaptor = ArgumentCaptor.forClass(BeanDefinitionCustomizer.class); - verify(sharedApplicationContext).registerBean(eq(extensionClass), any(Supplier.class), beanDefinitionCustomizerArgumentCaptor.capture()); + verify(applicationContext).registerBean(eq(extensionClass), any(Supplier.class), beanDefinitionCustomizerArgumentCaptor.capture()); final BeanDefinitionCustomizer beanDefinitionCustomizer = beanDefinitionCustomizerArgumentCaptor.getValue(); diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/DataPrepperScalarTypeDeserializerTest.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/DataPrepperScalarTypeDeserializerTest.java new file mode 100644 index 0000000000..8714b77fdd --- /dev/null +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/DataPrepperScalarTypeDeserializerTest.java @@ -0,0 +1,53 @@ +package org.opensearch.dataprepper.plugin; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.IOException; +import java.time.Duration; +import java.util.stream.Stream; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class DataPrepperScalarTypeDeserializerTest { + @Mock + private VariableExpander variableExpander; + @Mock + private JsonParser jsonParser; + @Mock + private DeserializationContext ctxt; + + private DataPrepperScalarTypeDeserializer objectUnderTest; + + @ParameterizedTest + @MethodSource("getScalarTypeArguments") + void testDeserialize(final Class scalarType, final Object scalarValue) throws IOException { + when(variableExpander.translate(eq(jsonParser), eq(scalarType))).thenReturn(scalarValue); + objectUnderTest = new DataPrepperScalarTypeDeserializer<>(variableExpander, scalarType); + assertThat(objectUnderTest.deserialize(jsonParser, ctxt), equalTo(scalarValue)); + } + + private static Stream getScalarTypeArguments() { + return Stream.of( + Arguments.of(String.class, RandomStringUtils.randomAlphabetic(5)), + Arguments.of(Duration.class, Duration.parse("PT15M")), + Arguments.of(Boolean.class, true), + Arguments.of(Short.class, (short) 2), + Arguments.of(Integer.class, 10), + Arguments.of(Long.class, 200L), + Arguments.of(Double.class, 1.23d), + Arguments.of(Float.class, 2.15f), + Arguments.of(Character.class, 'c')); + } +} \ No newline at end of file diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/ExtensionLoaderTest.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/ExtensionLoaderTest.java index 4df0af1e33..956c343060 100644 --- a/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/ExtensionLoaderTest.java +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/ExtensionLoaderTest.java @@ -96,8 +96,8 @@ void loadExtensions_returns_single_extension_with_config_for_single_plugin_class final TestExtensionWithConfig expectedPlugin = mock(TestExtensionWithConfig.class); final String expectedPluginName = "test_extension_with_config"; - when(extensionPluginConfigurationConverter.convert(eq(TestExtensionConfig.class), - eq("test_extension"))).thenReturn(testExtensionConfig); + when(extensionPluginConfigurationConverter.convert(eq(true), eq(TestExtensionConfig.class), + eq("/test_extension"))).thenReturn(testExtensionConfig); when(pluginCreator.newPluginInstance( eq(TestExtensionWithConfig.class), any(PluginArgumentsContext.class), diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/ExtensionPluginConfigurationConverterTest.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/ExtensionPluginConfigurationConverterTest.java index 6d28bf04d7..3cb7d9b4b5 100644 --- a/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/ExtensionPluginConfigurationConverterTest.java +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/ExtensionPluginConfigurationConverterTest.java @@ -37,7 +37,7 @@ class ExtensionPluginConfigurationConverterTest { @Mock private ConstraintViolation constraintViolation; - private final ObjectMapper objectMapper = new ObjectMapperConfiguration().objectMapper(); + private final ObjectMapper objectMapper = new ObjectMapperConfiguration().extensionPluginConfigObjectMapper(); private ExtensionPluginConfigurationConverter objectUnderTest; @BeforeEach @@ -49,41 +49,56 @@ void setUp() { @Test void convert_with_null_extensionConfigurationType_should_throw() { assertThrows(NullPointerException.class, - () -> objectUnderTest.convert(null, "testKey")); + () -> objectUnderTest.convert(true, null, "testKey")); } @Test void convert_with_null_rootKey_should_throw() { assertThrows(NullPointerException.class, - () -> objectUnderTest.convert(TestExtension.class, null)); + () -> objectUnderTest.convert(true, TestExtension.class, null)); } @Test - void convert_with_test_extension_with_config() { + void convert_with_test_extension_with_config_allowed_in_pipeline_configurations() { when(validator.validate(any())).thenReturn(Collections.emptySet()); final String rootKey = "test_extension"; final String testValue = "test_value"; - when(extensionPluginConfigurationResolver.getExtensionMap()).thenReturn(Map.of( + when(extensionPluginConfigurationResolver.getCombinedExtensionMap()).thenReturn(Map.of( rootKey, Map.of("test_attribute", testValue) )); - final Object testExtensionConfig = objectUnderTest.convert(TestExtensionConfig.class, rootKey); + final Object testExtensionConfig = objectUnderTest.convert(true, TestExtensionConfig.class, "/" + rootKey); assertThat(testExtensionConfig, instanceOf(TestExtensionConfig.class)); assertThat(((TestExtensionConfig) testExtensionConfig).getTestAttribute(), equalTo(testValue)); } @Test - void convert_with_null_rootKey_value_should_return_null() { + void convert_with_test_extension_with_config_not_allowed_in_pipeline_configurations() { + when(validator.validate(any())).thenReturn(Collections.emptySet()); final String rootKey = "test_extension"; - when(extensionPluginConfigurationResolver.getExtensionMap()).thenReturn(Collections.emptyMap()); - final Object testExtensionConfig = objectUnderTest.convert(TestExtensionConfig.class, rootKey); + final String testValue = "test_value"; + when(extensionPluginConfigurationResolver.getDataPrepperConfigExtensionMap()).thenReturn(Map.of( + rootKey, Map.of("test_attribute", testValue) + )); + final Object testExtensionConfig = objectUnderTest.convert(false, TestExtensionConfig.class, "/" + rootKey); + assertThat(testExtensionConfig, instanceOf(TestExtensionConfig.class)); + assertThat(((TestExtensionConfig) testExtensionConfig).getTestAttribute(), equalTo(testValue)); + } + + @Test + void convert_with_null_rootKey_value_should_return_null() { + final String rootKeyPath = "/test_extension"; + when(extensionPluginConfigurationResolver.getCombinedExtensionMap()).thenReturn(Collections.emptyMap()); + final Object testExtensionConfig = objectUnderTest.convert(true, TestExtensionConfig.class, rootKeyPath); assertThat(testExtensionConfig, nullValue()); } @Test void convert_should_throw_exception_when_there_are_constraint_violations() { - final String rootKey = UUID.randomUUID().toString(); - when(extensionPluginConfigurationResolver.getExtensionMap()).thenReturn( - Map.of(rootKey, Collections.emptyMap())); + final String firstKey = "first"; + final String secondKey = "second"; + final String jsonPointer = String.format("/%s/%s", firstKey, secondKey); + when(extensionPluginConfigurationResolver.getCombinedExtensionMap()).thenReturn( + Map.of(firstKey, Map.of(secondKey, Collections.emptyMap()))); final String errorMessage = UUID.randomUUID().toString(); given(constraintViolation.getMessage()).willReturn(errorMessage); final String propertyPathString = UUID.randomUUID().toString(); @@ -95,9 +110,9 @@ void convert_should_throw_exception_when_there_are_constraint_violations() { .willReturn(Collections.singleton(constraintViolation)); final InvalidPluginConfigurationException actualException = assertThrows(InvalidPluginConfigurationException.class, - () -> objectUnderTest.convert(TestExtensionConfig.class, rootKey)); + () -> objectUnderTest.convert(true, TestExtensionConfig.class, jsonPointer)); - assertThat(actualException.getMessage(), containsString(rootKey)); + assertThat(actualException.getMessage(), containsString(jsonPointer)); assertThat(actualException.getMessage(), containsString(propertyPathString)); assertThat(actualException.getMessage(), containsString(errorMessage)); } diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/ExtensionPluginConfigurationResolverTest.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/ExtensionPluginConfigurationResolverTest.java index f0e77112e7..9f324d0c51 100644 --- a/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/ExtensionPluginConfigurationResolverTest.java +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/ExtensionPluginConfigurationResolverTest.java @@ -33,7 +33,7 @@ void testGetExtensionMap_defined_in_dataPrepperConfiguration_only() { when(pipelineExtensions.getExtensionMap()).thenReturn(extensionMap); when(pipelinesDataFlowModel.getPipelineExtensions()).thenReturn(null); objectUnderTest = new ExtensionPluginConfigurationResolver(dataPrepperConfiguration, pipelinesDataFlowModel); - assertThat(objectUnderTest.getExtensionMap(), equalTo(extensionMap)); + assertThat(objectUnderTest.getCombinedExtensionMap(), equalTo(extensionMap)); } @Test @@ -43,7 +43,7 @@ void testGetExtensionMap_defined_in_pipelinesDataFlowModel_only() { final Map extensionMap = Map.of("test_extension", Map.of("test_key", "test_value")); when(pipelineExtensions.getExtensionMap()).thenReturn(extensionMap); objectUnderTest = new ExtensionPluginConfigurationResolver(dataPrepperConfiguration, pipelinesDataFlowModel); - assertThat(objectUnderTest.getExtensionMap(), equalTo(extensionMap)); + assertThat(objectUnderTest.getCombinedExtensionMap(), equalTo(extensionMap)); } @Test @@ -65,6 +65,6 @@ void testGetExtensionMap_defined_in_both() { "test_extension1", Map.of("test_key2", "test_value2"), "test_extension2", Map.of("test_key1", "test_value1") ); - assertThat(objectUnderTest.getExtensionMap(), equalTo(expectedExtensionMap)); + assertThat(objectUnderTest.getCombinedExtensionMap(), equalTo(expectedExtensionMap)); } } \ No newline at end of file diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/PluginBeanFactoryProviderTest.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/PluginBeanFactoryProviderTest.java index 8d1ee50853..14681b27eb 100644 --- a/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/PluginBeanFactoryProviderTest.java +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/PluginBeanFactoryProviderTest.java @@ -8,7 +8,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.BeanFactory; -import org.springframework.context.ApplicationContext; import org.springframework.context.support.GenericApplicationContext; import static org.hamcrest.CoreMatchers.equalTo; @@ -25,11 +24,11 @@ class PluginBeanFactoryProviderTest { - private ApplicationContext context; + private GenericApplicationContext context; @BeforeEach void setUp() { - context = mock(ApplicationContext.class); + context = mock(GenericApplicationContext.class); } private PluginBeanFactoryProvider createObjectUnderTest() { @@ -54,7 +53,7 @@ void testPluginBeanFactoryProviderRequiresContext() { @Test void testPluginBeanFactoryProviderRequiresParentContext() { - context = mock(ApplicationContext.class); + context = mock(GenericApplicationContext.class); assertThrows(NullPointerException.class, () -> createObjectUnderTest()); } diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/PluginConfigurationConverterTest.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/PluginConfigurationConverterTest.java index b70dafb09f..91df976a76 100644 --- a/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/PluginConfigurationConverterTest.java +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/PluginConfigurationConverterTest.java @@ -5,6 +5,7 @@ package org.opensearch.dataprepper.plugin; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; @@ -26,6 +27,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.BDDMockito.given; import static org.mockito.BDDMockito.then; import static org.mockito.Mockito.mock; @@ -33,10 +35,11 @@ class PluginConfigurationConverterTest { private PluginSetting pluginSetting; private Validator validator; - private final ObjectMapper objectMapper = new ObjectMapperConfiguration().objectMapper(); + private final ObjectMapper objectMapper = new ObjectMapper(); static class TestConfiguration { @SuppressWarnings("unused") + @JsonProperty("my_value") private String myValue; public String getMyValue() { @@ -76,7 +79,7 @@ void convert_with_PluginSetting_target_should_return_pluginSetting_object_direct assertThat(createObjectUnderTest().convert(PluginSetting.class, pluginSetting), sameInstance(pluginSetting)); - then(pluginSetting).shouldHaveNoInteractions(); + then(pluginSetting).should().setSettings(anyMap()); then(validator).shouldHaveNoInteractions(); } diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/VariableExpanderTest.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/VariableExpanderTest.java new file mode 100644 index 0000000000..7331cfe38b --- /dev/null +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugin/VariableExpanderTest.java @@ -0,0 +1,127 @@ +package org.opensearch.dataprepper.plugin; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.MappingJsonFactory; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.PluginConfigValueTranslator; + +import java.io.IOException; +import java.time.Duration; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class VariableExpanderTest { + static final ObjectMapper OBJECT_MAPPER = new ObjectMapperConfiguration().extensionPluginConfigObjectMapper(); + static final JsonFactory JSON_FACTORY = new MappingJsonFactory(); + + @Mock + private PluginConfigValueTranslator pluginConfigValueTranslator; + + private VariableExpander objectUnderTest; + + @BeforeEach + void setUp() { + objectUnderTest = new VariableExpander(OBJECT_MAPPER, Set.of(pluginConfigValueTranslator)); + } + + @ParameterizedTest + @MethodSource("getNonStringTypeArguments") + void testTranslateJsonParserWithNonStringValue(final Class clazz, final String value, final Object expectedResult) + throws IOException { + final JsonParser jsonParser = JSON_FACTORY.createParser(value); + jsonParser.nextToken(); + final Object actualResult = objectUnderTest.translate(jsonParser, clazz); + assertThat(actualResult, equalTo(expectedResult)); + } + + @ParameterizedTest + @MethodSource("getStringTypeArguments") + void testTranslateJsonParserWithStringValue_no_pattern_match( + final Class clazz, final String value, final Object expectedResult) throws IOException { + final JsonParser jsonParser = JSON_FACTORY.createParser(value); + jsonParser.nextToken(); + final Object actualResult = objectUnderTest.translate(jsonParser, clazz); + assertThat(actualResult, equalTo(expectedResult)); + } + + @ParameterizedTest + @MethodSource("getStringTypeArguments") + void testTranslateJsonParserWithStringValue_no_translator( + final Class clazz, final String value, final Object expectedResult) throws IOException { + final JsonParser jsonParser = JSON_FACTORY.createParser(value); + jsonParser.nextToken(); + objectUnderTest = new VariableExpander(OBJECT_MAPPER, Collections.emptySet()); + final Object actualResult = objectUnderTest.translate(jsonParser, clazz); + assertThat(actualResult, equalTo(expectedResult)); + } + + @Test + void testTranslateJsonParserWithStringValue_no_key_match() throws IOException { + final String testSecretKey = "testSecretKey"; + final String testSecretReference = String.format("${{unknown.%s}}", testSecretKey); + final JsonParser jsonParser = JSON_FACTORY.createParser(String.format("\"%s\"", testSecretReference)); + jsonParser.nextToken(); + when(pluginConfigValueTranslator.getPrefix()).thenReturn("test_prefix"); + objectUnderTest = new VariableExpander(OBJECT_MAPPER, Set.of(pluginConfigValueTranslator)); + final Object actualResult = objectUnderTest.translate(jsonParser, String.class); + assertThat(actualResult, equalTo(testSecretReference)); + } + + @ParameterizedTest + @MethodSource("getStringTypeArguments") + void testTranslateJsonParserWithStringValue_translate_success( + final Class clazz, final String value, final Object expectedResult) throws IOException { + final String testSecretKey = "testSecretKey"; + final String testTranslatorKey = "test_prefix"; + final String testSecretReference = String.format("${{%s:%s}}", testTranslatorKey, testSecretKey); + final JsonParser jsonParser = JSON_FACTORY.createParser(String.format("\"%s\"", testSecretReference)); + jsonParser.nextToken(); + when(pluginConfigValueTranslator.getPrefix()).thenReturn(testTranslatorKey); + when(pluginConfigValueTranslator.translate(eq(testSecretKey))).thenReturn(value.replace("\"", "")); + objectUnderTest = new VariableExpander(OBJECT_MAPPER, Set.of(pluginConfigValueTranslator)); + final Object actualResult = objectUnderTest.translate(jsonParser, clazz); + assertThat(actualResult, equalTo(expectedResult)); + } + + private static Stream getNonStringTypeArguments() { + return Stream.of(Arguments.of(Boolean.class, "true", true), + Arguments.of(Short.class, "2", (short) 2), + Arguments.of(Integer.class, "10", 10), + Arguments.of(Long.class, "200", 200L), + Arguments.of(Double.class, "1.23", 1.23d), + Arguments.of(Float.class, "2.15", 2.15f), + Arguments.of(Map.class, "{}", Collections.emptyMap())); + } + + private static Stream getStringTypeArguments() { + final String testRandomValue = "non-secret-prefix-" + RandomStringUtils.randomAlphabetic(5); + return Stream.of(Arguments.of(String.class, String.format("\"%s\"", testRandomValue), + testRandomValue), + Arguments.of(Duration.class, "\"PT15M\"", Duration.parse("PT15M")), + Arguments.of(Boolean.class, "\"true\"", true), + Arguments.of(Short.class, "\"2\"", (short) 2), + Arguments.of(Integer.class, "\"10\"", 10), + Arguments.of(Long.class, "\"200\"", 200L), + Arguments.of(Double.class, "\"1.23\"", 1.23d), + Arguments.of(Float.class, "\"2.15\"", 2.15f), + Arguments.of(Character.class, "\"c\"", 'c')); + } +} \ No newline at end of file diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugins/test/TestExtensionWithConfig.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugins/test/TestExtensionWithConfig.java index 768f42ad76..1ae3d70281 100644 --- a/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugins/test/TestExtensionWithConfig.java +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugins/test/TestExtensionWithConfig.java @@ -17,7 +17,8 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; -@DataPrepperExtensionPlugin(modelType = TestExtensionConfig.class, rootKey = "test_extension") +@DataPrepperExtensionPlugin(modelType = TestExtensionConfig.class, rootKeyJsonPath = "/test_extension", + allowInPipelineConfigurations = true) public class TestExtensionWithConfig implements ExtensionPlugin { private static final Logger LOG = LoggerFactory.getLogger(TestExtensionWithConfig.class); private static final AtomicInteger CONSTRUCTED_COUNT = new AtomicInteger(0); diff --git a/data-prepper-core/src/test/resources/multi-pipelines-distributed-pipeline-extensions/pipeline_configuration_with_test_extension1.yml b/data-prepper-core/src/test/resources/multi-pipelines-distributed-pipeline-extensions/pipeline_configuration_with_test_extension1.yml new file mode 100644 index 0000000000..4a9f9a59c2 --- /dev/null +++ b/data-prepper-core/src/test/resources/multi-pipelines-distributed-pipeline-extensions/pipeline_configuration_with_test_extension1.yml @@ -0,0 +1,13 @@ +# this configuration file is solely for testing formatting +pipeline_extensions: + test_extension_1: + test_attribute: test_string +test-pipeline-1: + source: + file: + path: "/tmp/file-source.tmp" + buffer: + bounded_blocking: #to check non object nodes for plugins + sink: + - pipeline: + name: "test-pipeline-2" diff --git a/data-prepper-core/src/test/resources/multi-pipelines-distributed-pipeline-extensions/pipeline_configuration_with_test_extension2.yml b/data-prepper-core/src/test/resources/multi-pipelines-distributed-pipeline-extensions/pipeline_configuration_with_test_extension2.yml new file mode 100644 index 0000000000..79fd195c8f --- /dev/null +++ b/data-prepper-core/src/test/resources/multi-pipelines-distributed-pipeline-extensions/pipeline_configuration_with_test_extension2.yml @@ -0,0 +1,11 @@ +# this configuration file is solely for testing formatting +pipeline_extensions: + test_extension_2: + test_attribute: test_string +test-pipeline-2: + source: + pipeline: + name: "test-pipeline-1" + sink: + - pipeline: + name: "test-pipeline-3" diff --git a/data-prepper-core/src/test/resources/multi-pipelines-single-pipeline-extensions/pipeline_configuration2.yml b/data-prepper-core/src/test/resources/multi-pipelines-single-pipeline-extensions/pipeline_configuration2.yml new file mode 100644 index 0000000000..22c0fe8c62 --- /dev/null +++ b/data-prepper-core/src/test/resources/multi-pipelines-single-pipeline-extensions/pipeline_configuration2.yml @@ -0,0 +1,8 @@ +# this configuration file is solely for testing formatting +test-pipeline-2: + source: + pipeline: + name: "test-pipeline-1" + sink: + - pipeline: + name: "test-pipeline-3" diff --git a/data-prepper-core/src/test/resources/multi-pipelines-single-pipeline-extensions/pipeline_configuration_with_test_extension.yml b/data-prepper-core/src/test/resources/multi-pipelines-single-pipeline-extensions/pipeline_configuration_with_test_extension.yml new file mode 100644 index 0000000000..fac47fa689 --- /dev/null +++ b/data-prepper-core/src/test/resources/multi-pipelines-single-pipeline-extensions/pipeline_configuration_with_test_extension.yml @@ -0,0 +1,13 @@ +# this configuration file is solely for testing formatting +pipeline_extensions: + test_extension: + test_attribute: test_string +test-pipeline-1: + source: + file: + path: "/tmp/file-source.tmp" + buffer: + bounded_blocking: #to check non object nodes for plugins + sink: + - pipeline: + name: "test-pipeline-2" diff --git a/data-prepper-plugins/aws-plugin/build.gradle b/data-prepper-plugins/aws-plugin/build.gradle index 70e386cf70..bb14699ee6 100644 --- a/data-prepper-plugins/aws-plugin/build.gradle +++ b/data-prepper-plugins/aws-plugin/build.gradle @@ -2,9 +2,13 @@ dependencies { implementation project(':data-prepper-api') implementation project(':data-prepper-plugins:aws-plugin-api') + implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'software.amazon.awssdk:auth' + implementation 'software.amazon.awssdk:secretsmanager' implementation 'software.amazon.awssdk:sts' implementation 'software.amazon.awssdk:arns' + testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' + testImplementation 'org.hibernate.validator:hibernate-validator:8.0.0.Final' } test { diff --git a/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretExtensionProvider.java b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretExtensionProvider.java new file mode 100644 index 0000000000..ff911f0211 --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretExtensionProvider.java @@ -0,0 +1,23 @@ +package org.opensearch.dataprepper.plugins.aws; + +import org.opensearch.dataprepper.model.plugin.ExtensionProvider; +import org.opensearch.dataprepper.model.plugin.PluginConfigValueTranslator; + +import java.util.Optional; + +public class AwsSecretExtensionProvider implements ExtensionProvider { + private final PluginConfigValueTranslator pluginConfigValueTranslator; + + AwsSecretExtensionProvider(final PluginConfigValueTranslator pluginConfigValueTranslator) { + this.pluginConfigValueTranslator = pluginConfigValueTranslator; + } + @Override + public Optional provideInstance(Context context) { + return Optional.ofNullable(pluginConfigValueTranslator); + } + + @Override + public Class supportedClass() { + return PluginConfigValueTranslator.class; + } +} diff --git a/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretManagerConfiguration.java b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretManagerConfiguration.java new file mode 100644 index 0000000000..2f44fc2033 --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretManagerConfiguration.java @@ -0,0 +1,104 @@ +package org.opensearch.dataprepper.plugins.aws; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Size; +import software.amazon.awssdk.arns.Arn; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; + +import java.util.Optional; +import java.util.UUID; + +public class AwsSecretManagerConfiguration { + private static final String DEFAULT_AWS_REGION = "us-east-1"; + private static final String AWS_IAM_ROLE = "role"; + private static final String AWS_IAM = "iam"; + + @JsonProperty("secret_id") + @NotNull + @Size(min = 1, max = 512, message = "awsSecretId length should be between 1 and 512 characters") + private String awsSecretId; + + @JsonProperty("region") + @Size(min = 1, message = "Region cannot be empty string") + private String awsRegion = DEFAULT_AWS_REGION; + + @JsonProperty("sts_role_arn") + @Size(min = 20, max = 2048, message = "awsStsRoleArn length should be between 1 and 2048 characters") + private String awsStsRoleArn; + + public String getAwsSecretId() { + return awsSecretId; + } + + public Region getAwsRegion() { + return Region.of(awsRegion); + } + + public SecretsManagerClient createSecretManagerClient() { + return SecretsManagerClient.builder() + .credentialsProvider(authenticateAwsConfiguration()) + .region(getAwsRegion()) + .build(); + } + + public GetSecretValueRequest createGetSecretValueRequest() { + return GetSecretValueRequest.builder() + .secretId(awsSecretId) + .build(); + } + + private AwsCredentialsProvider authenticateAwsConfiguration() { + + final AwsCredentialsProvider awsCredentialsProvider; + if (awsStsRoleArn != null && !awsStsRoleArn.isEmpty()) { + + validateStsRoleArn(); + + final StsClient stsClient = StsClient.builder() + .region(getAwsRegion()) + .build(); + + AssumeRoleRequest.Builder assumeRoleRequestBuilder = AssumeRoleRequest.builder() + .roleSessionName("aws-secret-" + UUID.randomUUID()) + .roleArn(awsStsRoleArn); + + awsCredentialsProvider = StsAssumeRoleCredentialsProvider.builder() + .stsClient(stsClient) + .refreshRequest(assumeRoleRequestBuilder.build()) + .build(); + + } else { + // use default credential provider + awsCredentialsProvider = DefaultCredentialsProvider.create(); + } + + return awsCredentialsProvider; + } + + private void validateStsRoleArn() { + final Arn arn = getArn(); + if (!AWS_IAM.equals(arn.service())) { + throw new IllegalArgumentException("sts_role_arn must be an IAM Role"); + } + final Optional resourceType = arn.resource().resourceType(); + if (resourceType.isEmpty() || !resourceType.get().equals(AWS_IAM_ROLE)) { + throw new IllegalArgumentException("sts_role_arn must be an IAM Role"); + } + } + + private Arn getArn() { + try { + return Arn.fromString(awsStsRoleArn); + } catch (final Exception e) { + throw new IllegalArgumentException(String.format("Invalid ARN format for sts_role_arn. Check the format of %s", awsStsRoleArn)); + } + } +} diff --git a/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretPlugin.java b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretPlugin.java new file mode 100644 index 0000000000..58dd819014 --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretPlugin.java @@ -0,0 +1,30 @@ +package org.opensearch.dataprepper.plugins.aws; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.opensearch.dataprepper.model.annotations.DataPrepperExtensionPlugin; +import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; +import org.opensearch.dataprepper.model.plugin.ExtensionPlugin; +import org.opensearch.dataprepper.model.plugin.ExtensionPoints; +import org.opensearch.dataprepper.model.plugin.PluginConfigValueTranslator; + +@DataPrepperExtensionPlugin(modelType = AwsSecretPluginConfig.class, rootKeyJsonPath = "/aws/secrets", + allowInPipelineConfigurations = true) +public class AwsSecretPlugin implements ExtensionPlugin { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private final PluginConfigValueTranslator pluginConfigValueTranslator; + + @DataPrepperPluginConstructor + public AwsSecretPlugin(final AwsSecretPluginConfig awsSecretPluginConfig) { + if (awsSecretPluginConfig != null) { + final SecretsSupplier secretsSupplier = new AwsSecretsSupplier(awsSecretPluginConfig, OBJECT_MAPPER); + pluginConfigValueTranslator = new AwsSecretsPluginConfigValueTranslator(secretsSupplier); + } else { + pluginConfigValueTranslator = null; + } + } + + @Override + public void apply(final ExtensionPoints extensionPoints) { + extensionPoints.addExtensionProvider(new AwsSecretExtensionProvider(pluginConfigValueTranslator)); + } +} diff --git a/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretPluginConfig.java b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretPluginConfig.java new file mode 100644 index 0000000000..db5962f0c9 --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretPluginConfig.java @@ -0,0 +1,16 @@ +package org.opensearch.dataprepper.plugins.aws; + +import com.fasterxml.jackson.annotation.JsonAnySetter; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class AwsSecretPluginConfig { + @JsonAnySetter + private Map awsSecretManagerConfigurationMap = new HashMap<>(); + + public Map getAwsSecretManagerConfigurationMap() { + return Collections.unmodifiableMap(awsSecretManagerConfigurationMap); + } +} diff --git a/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsPluginConfigValueTranslator.java b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsPluginConfigValueTranslator.java new file mode 100644 index 0000000000..71ae2ea8a3 --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsPluginConfigValueTranslator.java @@ -0,0 +1,41 @@ +package org.opensearch.dataprepper.plugins.aws; + +import org.opensearch.dataprepper.model.plugin.PluginConfigValueTranslator; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class AwsSecretsPluginConfigValueTranslator implements PluginConfigValueTranslator { + static final String AWS_SECRETS_PREFIX = "aws_secrets"; + static final String SECRET_CONFIGURATION_ID_GROUP = "secretConfigurationId"; + static final String SECRET_KEY_GROUP = "secretKey"; + static final Pattern SECRETS_REF_PATTERN = Pattern.compile( + String.format("^(?<%s>[a-zA-Z0-9\\/_+.=@-]+)(:(?<%s>.+))?$", + SECRET_CONFIGURATION_ID_GROUP, SECRET_KEY_GROUP)); + + private final SecretsSupplier secretsSupplier; + + public AwsSecretsPluginConfigValueTranslator(final SecretsSupplier secretsSupplier) { + this.secretsSupplier = secretsSupplier; + } + + @Override + public Object translate(final String value) { + final Matcher matcher = SECRETS_REF_PATTERN.matcher(value); + if (matcher.matches()) { + final String secretId = matcher.group(SECRET_CONFIGURATION_ID_GROUP); + final String secretKey = matcher.group(SECRET_KEY_GROUP); + return secretKey != null ? secretsSupplier.retrieveValue(secretId, secretKey) : + secretsSupplier.retrieveValue(secretId); + } else { + throw new IllegalArgumentException(String.format( + "Unable to parse %s or %s according to pattern %s", + SECRET_CONFIGURATION_ID_GROUP, SECRET_KEY_GROUP, SECRETS_REF_PATTERN.pattern())); + } + } + + @Override + public String getPrefix() { + return AWS_SECRETS_PREFIX; + } +} diff --git a/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsSupplier.java b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsSupplier.java new file mode 100644 index 0000000000..7919324a6e --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsSupplier.java @@ -0,0 +1,96 @@ +package org.opensearch.dataprepper.plugins.aws; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse; + +import java.util.Map; +import java.util.stream.Collectors; + +public class AwsSecretsSupplier implements SecretsSupplier { + static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference<>() { + }; + + private final ObjectMapper objectMapper; + private final Map secretIdToValue; + + public AwsSecretsSupplier(final AwsSecretPluginConfig awsSecretPluginConfig, final ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + secretIdToValue = toSecretMap(awsSecretPluginConfig); + } + + private Map toSecretMap(final AwsSecretPluginConfig awsSecretPluginConfig) { + final Map secretsManagerClientMap = toSecretsManagerClientMap( + awsSecretPluginConfig); + final Map awsSecretManagerConfigurationMap = awsSecretPluginConfig + .getAwsSecretManagerConfigurationMap(); + return secretsManagerClientMap.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> { + final String secretConfigurationId = entry.getKey(); + final AwsSecretManagerConfiguration awsSecretManagerConfiguration = + awsSecretManagerConfigurationMap.get(secretConfigurationId); + final SecretsManagerClient secretsManagerClient = entry.getValue(); + final GetSecretValueRequest getSecretValueRequest = awsSecretManagerConfiguration + .createGetSecretValueRequest(); + final GetSecretValueResponse getSecretValueResponse; + try { + getSecretValueResponse = secretsManagerClient.getSecretValue(getSecretValueRequest); + } catch (Exception e) { + throw new RuntimeException( + String.format("Unable to retrieve secret: %s", + awsSecretManagerConfiguration.getAwsSecretId()), e); + } + + try { + return objectMapper.readValue(getSecretValueResponse.secretString(), MAP_TYPE_REFERENCE); + } catch (JsonProcessingException e) { + return getSecretValueResponse.secretString(); + } + })); + } + + private Map toSecretsManagerClientMap( + final AwsSecretPluginConfig awsSecretPluginConfig) { + return awsSecretPluginConfig.getAwsSecretManagerConfigurationMap().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> { + final AwsSecretManagerConfiguration awsSecretManagerConfiguration = entry.getValue(); + return awsSecretManagerConfiguration.createSecretManagerClient(); + })); + } + + @Override + public Object retrieveValue(String secretId, String key) { + if (!secretIdToValue.containsKey(secretId)) { + throw new IllegalArgumentException(String.format("Unable to find secretId: %s", secretId)); + } + final Object keyValuePairs = secretIdToValue.get(secretId); + if (!(keyValuePairs instanceof Map)) { + throw new IllegalArgumentException(String.format("The value under secretId: %s is not a valid json.", + secretId)); + } + final Map keyValueMap = (Map) keyValuePairs; + if (!keyValueMap.containsKey(key)) { + throw new IllegalArgumentException(String.format("Unable to find the value of key: %s under secretId: %s", + key, secretId)); + } + return keyValueMap.get(key); + } + + @Override + public Object retrieveValue(String secretId) { + if (!secretIdToValue.containsKey(secretId)) { + throw new IllegalArgumentException(String.format("Unable to find secretId: %s", secretId)); + } + try { + final Object secretValue = secretIdToValue.get(secretId); + return secretValue instanceof Map ? objectMapper.writeValueAsString(secretValue) : + secretValue; + } catch (JsonProcessingException e) { + throw new IllegalArgumentException(String.format("Unable to read the value under secretId: %s as string.", + secretId)); + } + } +} diff --git a/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/SecretsSupplier.java b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/SecretsSupplier.java new file mode 100644 index 0000000000..467649775b --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/SecretsSupplier.java @@ -0,0 +1,7 @@ +package org.opensearch.dataprepper.plugins.aws; + +public interface SecretsSupplier { + Object retrieveValue(String secretId, String key); + + Object retrieveValue(String secretId); +} diff --git a/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretExtensionProviderTest.java b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretExtensionProviderTest.java new file mode 100644 index 0000000000..c1e84e9c7b --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretExtensionProviderTest.java @@ -0,0 +1,51 @@ +package org.opensearch.dataprepper.plugins.aws; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.ExtensionProvider; +import org.opensearch.dataprepper.model.plugin.PluginConfigValueTranslator; + +import java.util.Optional; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.sameInstance; +import static org.hamcrest.MatcherAssert.assertThat; + +@ExtendWith(MockitoExtension.class) +class AwsSecretExtensionProviderTest { + @Mock + private PluginConfigValueTranslator pluginConfigValueTranslator; + + @Mock + private ExtensionProvider.Context context; + + private AwsSecretExtensionProvider createObjectUnderTest() { + return new AwsSecretExtensionProvider(pluginConfigValueTranslator); + } + + @Test + void supportedClass_returns_PluginConfigValueTranslator() { + assertThat(createObjectUnderTest().supportedClass(), equalTo(PluginConfigValueTranslator.class)); + } + + @Test + void provideInstance_returns_the_PluginConfigValueTranslator_from_the_constructor() { + final AwsSecretExtensionProvider objectUnderTest = createObjectUnderTest(); + + final Optional optionalPluginConfigValueTranslator = + objectUnderTest.provideInstance(context); + assertThat(optionalPluginConfigValueTranslator, notNullValue()); + assertThat(optionalPluginConfigValueTranslator.isPresent(), equalTo(true)); + assertThat(optionalPluginConfigValueTranslator.get(), equalTo(pluginConfigValueTranslator)); + + final Optional anotherOptionalPluginConfigValueTranslator = + objectUnderTest.provideInstance(context); + assertThat(anotherOptionalPluginConfigValueTranslator, notNullValue()); + assertThat(anotherOptionalPluginConfigValueTranslator.isPresent(), equalTo(true)); + assertThat(anotherOptionalPluginConfigValueTranslator.get(), sameInstance( + anotherOptionalPluginConfigValueTranslator.get())); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretManagerConfigurationTest.java b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretManagerConfigurationTest.java new file mode 100644 index 0000000000..7daed30307 --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretManagerConfigurationTest.java @@ -0,0 +1,151 @@ +package org.opensearch.dataprepper.plugins.aws; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import jakarta.validation.ConstraintViolation; +import jakarta.validation.Validation; +import jakarta.validation.Validator; +import org.hibernate.validator.messageinterpolation.ParameterMessageInterpolator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.junit.jupiter.MockitoExtension; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClientBuilder; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Set; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class AwsSecretManagerConfigurationTest { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory()); + private static final Validator VALIDATOR = Validation.byDefaultProvider() + .configure() + .messageInterpolator(new ParameterMessageInterpolator()) + .buildValidatorFactory().getValidator(); + + @Mock + private GetSecretValueRequest.Builder getSecretValueRequestBuilder; + + @Mock + private GetSecretValueRequest getSecretValueRequest; + + @Mock + private SecretsManagerClientBuilder secretsManagerClientBuilder; + + @Mock + private SecretsManagerClient secretsManagerClient; + + @Captor + private ArgumentCaptor awsCredentialsProviderArgumentCaptor; + + @Test + void testCreateGetSecretValueRequest() throws IOException { + when(getSecretValueRequestBuilder.secretId(anyString())).thenReturn(getSecretValueRequestBuilder); + when(getSecretValueRequestBuilder.build()).thenReturn(getSecretValueRequest); + final InputStream inputStream = AwsSecretPluginConfigTest.class.getResourceAsStream( + "/test-aws-secret-manager-configuration-default.yaml"); + final AwsSecretManagerConfiguration awsSecretManagerConfiguration = OBJECT_MAPPER.readValue( + inputStream, AwsSecretManagerConfiguration.class); + try (final MockedStatic getSecretValueRequestMockedStatic = + mockStatic(GetSecretValueRequest.class)) { + getSecretValueRequestMockedStatic.when(GetSecretValueRequest::builder).thenReturn( + getSecretValueRequestBuilder); + assertThat(awsSecretManagerConfiguration.createGetSecretValueRequest(), is(getSecretValueRequest)); + } + verify(getSecretValueRequestBuilder).secretId("test-secret"); + } + + @Test + void testCreateSecretManagerClientWithDefaultCredential() throws IOException { + final InputStream inputStream = AwsSecretPluginConfigTest.class.getResourceAsStream( + "/test-aws-secret-manager-configuration-default.yaml"); + final AwsSecretManagerConfiguration awsSecretManagerConfiguration = OBJECT_MAPPER.readValue( + inputStream, AwsSecretManagerConfiguration.class); + assertThat(awsSecretManagerConfiguration.getAwsSecretId(), equalTo("test-secret")); + when(secretsManagerClientBuilder.region(any(Region.class))).thenReturn(secretsManagerClientBuilder); + when(secretsManagerClientBuilder.credentialsProvider(any(AwsCredentialsProvider.class))) + .thenReturn(secretsManagerClientBuilder); + when(secretsManagerClientBuilder.build()).thenReturn(secretsManagerClient); + try (final MockedStatic secretsManagerClientMockedStatic = mockStatic( + SecretsManagerClient.class)) { + secretsManagerClientMockedStatic.when(SecretsManagerClient::builder).thenReturn(secretsManagerClientBuilder); + assertThat(awsSecretManagerConfiguration.createSecretManagerClient(), is(secretsManagerClient)); + } + verify(secretsManagerClientBuilder).credentialsProvider(awsCredentialsProviderArgumentCaptor.capture()); + final AwsCredentialsProvider awsCredentialsProvider = awsCredentialsProviderArgumentCaptor.getValue(); + assertThat(awsCredentialsProvider, instanceOf(DefaultCredentialsProvider.class)); + } + + @Test + void testCreateSecretManagerClientWithStsCredential() throws IOException { + final InputStream inputStream = AwsSecretPluginConfigTest.class.getResourceAsStream( + "/test-aws-secret-manager-configuration-with-sts.yaml"); + final AwsSecretManagerConfiguration awsSecretManagerConfiguration = OBJECT_MAPPER.readValue( + inputStream, AwsSecretManagerConfiguration.class); + assertThat(awsSecretManagerConfiguration.getAwsSecretId(), equalTo("test-secret")); + when(secretsManagerClientBuilder.region(any(Region.class))).thenReturn(secretsManagerClientBuilder); + when(secretsManagerClientBuilder.credentialsProvider(any(AwsCredentialsProvider.class))) + .thenReturn(secretsManagerClientBuilder); + when(secretsManagerClientBuilder.build()).thenReturn(secretsManagerClient); + try (final MockedStatic secretsManagerClientMockedStatic = mockStatic( + SecretsManagerClient.class)) { + secretsManagerClientMockedStatic.when(SecretsManagerClient::builder).thenReturn(secretsManagerClientBuilder); + assertThat(awsSecretManagerConfiguration.createSecretManagerClient(), is(secretsManagerClient)); + } + verify(secretsManagerClientBuilder).credentialsProvider(awsCredentialsProviderArgumentCaptor.capture()); + final AwsCredentialsProvider awsCredentialsProvider = awsCredentialsProviderArgumentCaptor.getValue(); + assertThat(awsCredentialsProvider, instanceOf(StsAssumeRoleCredentialsProvider.class)); + } + + @ParameterizedTest + @ValueSource(strings = { + "/test-aws-secret-manager-configuration-invalid-sts-1.yaml", + "/test-aws-secret-manager-configuration-invalid-sts-2.yaml", + "/test-aws-secret-manager-configuration-invalid-sts-3.yaml" + }) + void testCreateSecretManagerClientWithInvalidStsRoleArn(final String testFileName) throws IOException { + final InputStream inputStream = AwsSecretPluginConfigTest.class.getResourceAsStream(testFileName); + final AwsSecretManagerConfiguration awsSecretManagerConfiguration = OBJECT_MAPPER.readValue( + inputStream, AwsSecretManagerConfiguration.class); + try (final MockedStatic secretsManagerClientMockedStatic = mockStatic( + SecretsManagerClient.class)) { + secretsManagerClientMockedStatic.when(SecretsManagerClient::builder).thenReturn(secretsManagerClientBuilder); + assertThrows(IllegalArgumentException.class, + () -> awsSecretManagerConfiguration.createSecretManagerClient()); + } + } + + @Test + void testDeserializationMissingName() throws IOException { + final InputStream inputStream = AwsSecretPluginConfigTest.class.getResourceAsStream( + "/test-aws-secret-manager-configuration-missing-secret-id.yaml"); + final AwsSecretManagerConfiguration awsSecretManagerConfiguration = OBJECT_MAPPER.readValue( + inputStream, AwsSecretManagerConfiguration.class); + final Set> violations = VALIDATOR.validate( + awsSecretManagerConfiguration); + assertThat(violations.size(), equalTo(1)); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretPluginConfigTest.java b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretPluginConfigTest.java new file mode 100644 index 0000000000..3c8348bc74 --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretPluginConfigTest.java @@ -0,0 +1,36 @@ +package org.opensearch.dataprepper.plugins.aws; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.MatcherAssert.assertThat; + +class AwsSecretPluginConfigTest { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory()); + @Test + void testDefault() { + final AwsSecretPluginConfig awsSecretPluginConfig = new AwsSecretPluginConfig(); + assertThat(awsSecretPluginConfig.getAwsSecretManagerConfigurationMap(), equalTo(Collections.emptyMap())); + } + + @Test + void testDeserialization() throws IOException { + final InputStream inputStream = AwsSecretPluginConfigTest.class.getResourceAsStream( + "/test-aws-secret-plugin-config.yaml"); + final AwsSecretPluginConfig awsSecretPluginConfig = OBJECT_MAPPER.readValue( + inputStream, AwsSecretPluginConfig.class); + final Map awsSecretManagerConfigurationMap = + awsSecretPluginConfig.getAwsSecretManagerConfigurationMap(); + assertThat(awsSecretManagerConfigurationMap.size(), equalTo(1)); + assertThat(awsSecretManagerConfigurationMap.get("test_config"), + instanceOf(AwsSecretManagerConfiguration.class)); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretPluginIT.java b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretPluginIT.java new file mode 100644 index 0000000000..b0cefab287 --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretPluginIT.java @@ -0,0 +1,63 @@ +package org.opensearch.dataprepper.plugins.aws; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.ExtensionPoints; +import org.opensearch.dataprepper.model.plugin.ExtensionProvider; +import org.opensearch.dataprepper.model.plugin.PluginConfigValueTranslator; + +import java.util.Collections; +import java.util.Optional; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class AwsSecretPluginIT { + @Mock + private AwsSecretPluginConfig awsSecretPluginConfig; + + @Mock + private ExtensionPoints extensionPoints; + + @Mock + private ExtensionProvider.Context context; + + private AwsSecretPlugin objectUnderTest; + + @Test + void testInitializationWithNonNullConfig() { + when(awsSecretPluginConfig.getAwsSecretManagerConfigurationMap()).thenReturn(Collections.emptyMap()); + objectUnderTest = new AwsSecretPlugin(awsSecretPluginConfig); + objectUnderTest.apply(extensionPoints); + final ArgumentCaptor extensionProviderArgumentCaptor = + ArgumentCaptor.forClass(ExtensionProvider.class); + verify(extensionPoints).addExtensionProvider(extensionProviderArgumentCaptor.capture()); + final ExtensionProvider actualExtensionProvider = extensionProviderArgumentCaptor.getValue(); + assertThat(actualExtensionProvider, instanceOf(AwsSecretExtensionProvider.class)); + final Optional optionalPluginConfigValueTranslator = + actualExtensionProvider.provideInstance(context); + assertThat(optionalPluginConfigValueTranslator.isPresent(), is(true)); + assertThat(optionalPluginConfigValueTranslator.get(), instanceOf(AwsSecretsPluginConfigValueTranslator.class)); + } + + @Test + void testInitializationWithNullConfig() { + objectUnderTest = new AwsSecretPlugin(null); + objectUnderTest.apply(extensionPoints); + final ArgumentCaptor extensionProviderArgumentCaptor = + ArgumentCaptor.forClass(ExtensionProvider.class); + verify(extensionPoints).addExtensionProvider(extensionProviderArgumentCaptor.capture()); + final ExtensionProvider actualExtensionProvider = extensionProviderArgumentCaptor.getValue(); + assertThat(actualExtensionProvider, instanceOf(AwsSecretExtensionProvider.class)); + final Optional optionalPluginConfigValueTranslator = + actualExtensionProvider.provideInstance(context); + assertThat(optionalPluginConfigValueTranslator.isEmpty(), is(true)); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsPluginConfigValueTranslatorTest.java b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsPluginConfigValueTranslatorTest.java new file mode 100644 index 0000000000..85d247cd73 --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsPluginConfigValueTranslatorTest.java @@ -0,0 +1,63 @@ +package org.opensearch.dataprepper.plugins.aws; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.aws.AwsSecretsPluginConfigValueTranslator.AWS_SECRETS_PREFIX; + +@ExtendWith(MockitoExtension.class) +class AwsSecretsPluginConfigValueTranslatorTest { + @Mock + private SecretsSupplier secretsSupplier; + + private AwsSecretsPluginConfigValueTranslator objectUnderTest; + + @BeforeEach + void setUp() { + objectUnderTest = new AwsSecretsPluginConfigValueTranslator(secretsSupplier); + } + + @Test + void testGetPrefix() { + assertThat(objectUnderTest.getPrefix(), equalTo(AWS_SECRETS_PREFIX)); + } + + @ParameterizedTest + @ValueSource(strings = { + "", + "invalid secret id with space:secret_key" + }) + void testTranslateInputNoMatch(final String input) { + assertThrows(IllegalArgumentException.class, () -> objectUnderTest.translate(input)); + } + + @Test + void testTranslateSecretIdWithKeyMatch() { + final String testSecretName = "valid@secret-manager_name"; + final String testSecretKey = UUID.randomUUID().toString(); + final String testSecretValue = UUID.randomUUID().toString(); + final String input = String.format("%s:%s", testSecretName, testSecretKey); + when(secretsSupplier.retrieveValue(eq(testSecretName), eq(testSecretKey))).thenReturn(testSecretValue); + assertThat(objectUnderTest.translate(input), equalTo(testSecretValue)); + } + + @Test + void testTranslateSecretIdWithoutKeyMatch() { + final String testSecretName = "valid@secret-manager_name"; + final String testSecretValue = UUID.randomUUID().toString(); + when(secretsSupplier.retrieveValue(eq(testSecretName))).thenReturn(testSecretValue); + assertThat(objectUnderTest.translate(testSecretName), equalTo(testSecretValue)); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsSupplierTest.java b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsSupplierTest.java new file mode 100644 index 0000000000..f9047086f6 --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsSupplierTest.java @@ -0,0 +1,130 @@ +package org.opensearch.dataprepper.plugins.aws; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse; +import software.amazon.awssdk.services.secretsmanager.model.SecretsManagerException; + +import java.util.Map; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.aws.AwsSecretsSupplier.MAP_TYPE_REFERENCE; + +@ExtendWith(MockitoExtension.class) +class AwsSecretsSupplierTest { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final String TEST_AWS_SECRET_CONFIGURATION_NAME = "test-secret-config"; + private static final String TEST_KEY = "test-key"; + private static final String TEST_VALUE = "test-value"; + + @Mock + private AwsSecretManagerConfiguration awsSecretManagerConfiguration; + + @Mock + private AwsSecretPluginConfig awsSecretPluginConfig; + + @Mock + private SecretsManagerClient secretsManagerClient; + + @Mock + private GetSecretValueRequest getSecretValueRequest; + + @Mock + private GetSecretValueResponse getSecretValueResponse; + + @Mock + private SecretsManagerException secretsManagerException; + + private AwsSecretsSupplier objectUnderTest; + + @BeforeEach + void setUp() throws JsonProcessingException { + when(awsSecretManagerConfiguration.createGetSecretValueRequest()).thenReturn(getSecretValueRequest); + when(awsSecretPluginConfig.getAwsSecretManagerConfigurationMap()).thenReturn( + Map.of(TEST_AWS_SECRET_CONFIGURATION_NAME, awsSecretManagerConfiguration) + ); + when(awsSecretManagerConfiguration.createSecretManagerClient()).thenReturn(secretsManagerClient); + when(getSecretValueResponse.secretString()).thenReturn(OBJECT_MAPPER.writeValueAsString( + Map.of(TEST_KEY, TEST_VALUE) + )); + when(secretsManagerClient.getSecretValue(eq(getSecretValueRequest))).thenReturn(getSecretValueResponse); + objectUnderTest = new AwsSecretsSupplier(awsSecretPluginConfig, OBJECT_MAPPER); + } + + @Test + void testRetrieveValueExists() { + assertThat(objectUnderTest.retrieveValue(TEST_AWS_SECRET_CONFIGURATION_NAME, TEST_KEY), equalTo(TEST_VALUE)); + } + + @Test + void testRetrieveValueMissingSecretConfigName() { + assertThrows(IllegalArgumentException.class, + () -> objectUnderTest.retrieveValue("missing-config-id", TEST_KEY)); + } + + @Test + void testRetrieveValueMissingKey() { + assertThrows(IllegalArgumentException.class, + () -> objectUnderTest.retrieveValue(TEST_AWS_SECRET_CONFIGURATION_NAME, "missing-key")); + } + + @Test + void testRetrieveValueInvalidKeyValuePair() { + when(getSecretValueResponse.secretString()).thenReturn(TEST_VALUE); + objectUnderTest = new AwsSecretsSupplier(awsSecretPluginConfig, OBJECT_MAPPER); + final Exception exception = assertThrows(IllegalArgumentException.class, + () -> objectUnderTest.retrieveValue(TEST_AWS_SECRET_CONFIGURATION_NAME, TEST_KEY)); + assertThat(exception.getMessage(), equalTo(String.format("The value under secretId: %s is not a valid json.", + TEST_AWS_SECRET_CONFIGURATION_NAME))); + } + + @Test + void testRetrieveValueBySecretIdOnlyDoesNotExist() { + assertThrows(IllegalArgumentException.class, + () -> objectUnderTest.retrieveValue("missing-config-id")); + } + + @Test + void testRetrieveValueBySecretIdOnlyNotSerializable() throws JsonProcessingException { + final ObjectMapper mockedObjectMapper = mock(ObjectMapper.class); + final JsonProcessingException mockedJsonProcessingException = mock(JsonProcessingException.class); + final String testValue = "{\"a\":\"b\"}"; + when(mockedObjectMapper.readValue(eq(testValue), eq(MAP_TYPE_REFERENCE))).thenReturn(Map.of("a", "b")); + when(mockedObjectMapper.writeValueAsString(ArgumentMatchers.any())).thenThrow(mockedJsonProcessingException); + when(getSecretValueResponse.secretString()).thenReturn(testValue); + objectUnderTest = new AwsSecretsSupplier(awsSecretPluginConfig, mockedObjectMapper); + final Exception exception = assertThrows(IllegalArgumentException.class, + () -> objectUnderTest.retrieveValue(TEST_AWS_SECRET_CONFIGURATION_NAME)); + assertThat(exception.getMessage(), equalTo(String.format("Unable to read the value under secretId: %s as string.", + TEST_AWS_SECRET_CONFIGURATION_NAME))); + } + + @ParameterizedTest + @ValueSource(strings = {TEST_VALUE, "{\"a\":\"b\"}"}) + void testRetrieveValueWithoutKey(String testValue) { + when(getSecretValueResponse.secretString()).thenReturn(testValue); + objectUnderTest = new AwsSecretsSupplier(awsSecretPluginConfig, OBJECT_MAPPER); + assertThat(objectUnderTest.retrieveValue(TEST_AWS_SECRET_CONFIGURATION_NAME), equalTo(testValue)); + } + + @Test + void testConstructorWithGetSecretValueFailure() { + when(secretsManagerClient.getSecretValue(eq(getSecretValueRequest))).thenThrow(secretsManagerException); + assertThrows(RuntimeException.class, () -> new AwsSecretsSupplier(awsSecretPluginConfig, OBJECT_MAPPER)); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-default.yaml b/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-default.yaml new file mode 100644 index 0000000000..4125645a24 --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-default.yaml @@ -0,0 +1 @@ +secret_id: test-secret \ No newline at end of file diff --git a/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-invalid-sts-1.yaml b/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-invalid-sts-1.yaml new file mode 100644 index 0000000000..6b99bc300e --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-invalid-sts-1.yaml @@ -0,0 +1,3 @@ +secret_id: test-secret +region: us-east-1 +sts_role_arn: invalid_arn \ No newline at end of file diff --git a/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-invalid-sts-2.yaml b/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-invalid-sts-2.yaml new file mode 100644 index 0000000000..6cbf5294fe --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-invalid-sts-2.yaml @@ -0,0 +1,3 @@ +secret_id: test-secret +region: us-east-1 +sts_role_arn: arn:aws:unknown-service::123456789012:role/test-role \ No newline at end of file diff --git a/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-invalid-sts-3.yaml b/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-invalid-sts-3.yaml new file mode 100644 index 0000000000..a5610dd0cc --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-invalid-sts-3.yaml @@ -0,0 +1,3 @@ +secret_id: test-secret +region: us-east-1 +sts_role_arn: arn:aws:iam::123456789012:unknown-resource-type/test-role \ No newline at end of file diff --git a/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-missing-secret-id.yaml b/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-missing-secret-id.yaml new file mode 100644 index 0000000000..a3494dd04c --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-missing-secret-id.yaml @@ -0,0 +1,2 @@ +region: us-east-1 +sts_role_arn: arn:aws:iam::123456789012:role/test-role \ No newline at end of file diff --git a/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-with-sts.yaml b/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-with-sts.yaml new file mode 100644 index 0000000000..8bf929a56b --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-with-sts.yaml @@ -0,0 +1,3 @@ +secret_id: test-secret +region: us-east-1 +sts_role_arn: arn:aws:iam::123456789012:role/test-role \ No newline at end of file diff --git a/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-plugin-config.yaml b/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-plugin-config.yaml new file mode 100644 index 0000000000..d29cca0dcc --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-plugin-config.yaml @@ -0,0 +1,4 @@ +test_config: + secret_id: test-secret + region: us-east-1 + sts_role_arn: arn:aws:iam::123456789012:role/test-role \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java index 87c94a5b91..678d4fb00d 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java @@ -5,6 +5,7 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.http.HttpHost; import org.apache.http.HttpRequestInterceptor; import org.apache.http.auth.AuthScope; @@ -62,6 +63,7 @@ import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DISTRIBUTION_VERSION; public class ConnectionConfiguration { + static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final Logger LOG = LoggerFactory.getLogger(OpenSearchSink.class); private static final String AWS_IAM_ROLE = "role"; private static final String AWS_IAM = "iam"; @@ -189,11 +191,11 @@ public static ConnectionConfiguration readConnectionConfiguration(final PluginSe if (password != null) { builder = builder.withPassword(password); } - final Integer socketTimeout = (Integer) pluginSetting.getAttributeFromSettings(SOCKET_TIMEOUT); + final Integer socketTimeout = pluginSetting.getIntegerOrDefault(SOCKET_TIMEOUT, null); if (socketTimeout != null) { builder = builder.withSocketTimeout(socketTimeout); } - final Integer connectTimeout = (Integer) pluginSetting.getAttributeFromSettings(CONNECT_TIMEOUT); + final Integer connectTimeout = pluginSetting.getIntegerOrDefault(CONNECT_TIMEOUT, null); if (connectTimeout != null) { builder = builder.withConnectTimeout(connectTimeout); } @@ -208,7 +210,8 @@ public static ConnectionConfiguration readConnectionConfiguration(final PluginSe builder.withAWSStsRoleArn((String)(awsOption.getOrDefault(AWS_STS_ROLE_ARN.substring(4), null))); builder.withAWSStsExternalId((String)(awsOption.getOrDefault(AWS_STS_EXTERNAL_ID.substring(4), null))); builder.withAwsStsHeaderOverrides((Map)awsOption.get(AWS_STS_HEADER_OVERRIDES.substring(4))); - builder.withServerless((Boolean)awsOption.getOrDefault(SERVERLESS, false)); + builder.withServerless(OBJECT_MAPPER.convertValue( + awsOption.getOrDefault(SERVERLESS, false), Boolean.class)); } else { builder.withServerless(false); } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java index f6d790048d..fa397e9271 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java @@ -78,7 +78,7 @@ public static RetryConfiguration readRetryConfig(final PluginSetting pluginSetti if (dlqFile != null) { builder = builder.withDlqFile(dlqFile); } - final Integer maxRetries = (Integer) pluginSetting.getAttributeFromSettings(MAX_RETRIES); + final Integer maxRetries = pluginSetting.getIntegerOrDefault(MAX_RETRIES, null); if (maxRetries != null) { builder = builder.withMaxRetries(maxRetries); } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java index e23fdd4e26..d346c9f61c 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java @@ -34,6 +34,7 @@ public class IndexConfiguration { private static final Logger LOG = LoggerFactory.getLogger(IndexConfiguration.class); + static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); public static final String SETTINGS = "settings"; public static final String INDEX_ALIAS = "index"; @@ -228,7 +229,8 @@ public static IndexConfiguration readIndexConfig(final PluginSetting pluginSetti Map awsOption = pluginSetting.getTypedMap(AWS_OPTION, String.class, Object.class); if (awsOption != null && !awsOption.isEmpty()) { - builder.withServerless((Boolean)awsOption.getOrDefault(SERVERLESS, false)); + builder.withServerless(OBJECT_MAPPER.convertValue( + awsOption.getOrDefault(SERVERLESS, false), Boolean.class)); } else { builder.withServerless(false); }