From 9620bc920addd97f438b5783b769dd79cdec89ad Mon Sep 17 00:00:00 2001 From: George Chen Date: Sat, 21 Sep 2024 20:23:42 -0500 Subject: [PATCH 01/11] ADD: initial impl on resolving target type Signed-off-by: George Chen --- data-prepper-api/build.gradle | 3 +- .../annotations/UsesDataPrepperPlugin.java | 20 ++++++++ .../model/configuration/PluginModel.java | 50 ++++++++++++++++++- .../DataPrepperPluginSchemaExecute.java | 2 +- .../schemas/JsonSchemaConverter.java | 29 ++++++++++- .../schemas/JsonSchemaConverterTest.java | 39 +++++++++++++-- .../PluginConfigsJsonSchemaConverterIT.java | 2 +- 7 files changed, 135 insertions(+), 10 deletions(-) create mode 100644 data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/UsesDataPrepperPlugin.java diff --git a/data-prepper-api/build.gradle b/data-prepper-api/build.gradle index 045d331704..5ff379f10c 100644 --- a/data-prepper-api/build.gradle +++ b/data-prepper-api/build.gradle @@ -12,9 +12,10 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8' + implementation 'org.reflections:reflections:0.10.2' implementation libs.parquet.common - testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' implementation libs.commons.lang3 + testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' testImplementation project(':data-prepper-test-common') testImplementation 'org.skyscreamer:jsonassert:1.5.3' testImplementation libs.commons.io diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/UsesDataPrepperPlugin.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/UsesDataPrepperPlugin.java new file mode 100644 index 0000000000..8d81290df3 --- /dev/null +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/UsesDataPrepperPlugin.java @@ -0,0 +1,20 @@ +package org.opensearch.dataprepper.model.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD}) +public @interface UsesDataPrepperPlugin { + /** + * The class type for this plugin. + * + * @return The Java class + * @since 1.2 + */ + Class pluginType(); +} diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginModel.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginModel.java index d8b2b41054..030ed9dcff 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginModel.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginModel.java @@ -11,22 +11,33 @@ import com.fasterxml.jackson.core.JacksonException; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.BeanProperty; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.deser.ContextualDeserializer; import com.fasterxml.jackson.databind.deser.std.StdDeserializer; import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; +import org.reflections.Reflections; +import org.reflections.scanners.Scanners; +import org.reflections.util.ClasspathHelper; +import org.reflections.util.ConfigurationBuilder; import java.io.IOException; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Model class for a Plugin in Configuration YAML containing name of the Plugin and its associated settings @@ -36,7 +47,7 @@ @JsonSerialize(using = PluginModel.PluginModelSerializer.class) @JsonDeserialize(using = PluginModel.PluginModelDeserializer.class) public class PluginModel { - + static final String DEFAULT_PLUGINS_CLASSPATH = "org.opensearch.dataprepper.plugins"; private static final ObjectMapper SERIALIZER_OBJECT_MAPPER = new ObjectMapper(); private final String pluginName; @@ -148,11 +159,15 @@ public PluginModelDeserializer() { * * @see SinkModel.SinkModelDeserializer */ - abstract static class AbstractPluginModelDeserializer extends StdDeserializer { + abstract static class AbstractPluginModelDeserializer + extends StdDeserializer implements ContextualDeserializer { private final Class innerModelClass; private final BiFunction constructorFunction; private final Supplier emptyInnerModelConstructor; + private final Reflections reflections; + + private UsesDataPrepperPlugin usesDataPrepperPlugin; protected AbstractPluginModelDeserializer( final Class valueClass, @@ -163,6 +178,16 @@ protected AbstractPluginModelDeserializer( this.innerModelClass = innerModelClass; this.constructorFunction = constructorFunction; this.emptyInnerModelConstructor = emptyInnerModelConstructor; + this.reflections = new Reflections(new ConfigurationBuilder() + .setUrls(ClasspathHelper.forPackage(DEFAULT_PLUGINS_CLASSPATH)) + .setScanners(Scanners.TypesAnnotated, Scanners.SubTypes)); + } + + @Override + public AbstractPluginModelDeserializer createContextual( + final DeserializationContext context, final BeanProperty property) { + usesDataPrepperPlugin = property.getAnnotation(UsesDataPrepperPlugin.class); + return this; } @Override @@ -175,12 +200,33 @@ public PluginModel deserialize(final JsonParser jsonParser, final Deserializatio final String pluginName = onlyField.getKey(); final JsonNode value = onlyField.getValue(); + if (usesDataPrepperPlugin != null) { + final Set pluginNames = scanForPluginNames(usesDataPrepperPlugin.pluginType()); + if (!pluginNames.contains(pluginName)) { + throw new IOException(String.format("%s is not found on %s.", + pluginName, usesDataPrepperPlugin.pluginType())); + } + } + M innerModel = SERIALIZER_OBJECT_MAPPER.convertValue(value, innerModelClass); if(innerModel == null) innerModel = emptyInnerModelConstructor.get(); return constructorFunction.apply(pluginName, innerModel); } + + private Set scanForPluginNames(final Class pluginType) { + return reflections.getTypesAnnotatedWith(DataPrepperPlugin.class).stream() + .map(clazz -> clazz.getAnnotation(DataPrepperPlugin.class)) + .filter(dataPrepperPlugin -> pluginType.equals(dataPrepperPlugin.pluginType())) + .flatMap(dataPrepperPlugin -> { + if (!dataPrepperPlugin.deprecatedName().isEmpty()) { + return Stream.of(dataPrepperPlugin.deprecatedName(), dataPrepperPlugin.name()); + } + return Stream.of(dataPrepperPlugin.name()); + }) + .collect(Collectors.toSet()); + } } } diff --git a/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/DataPrepperPluginSchemaExecute.java b/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/DataPrepperPluginSchemaExecute.java index 75115eb7e6..d8cd0f12b2 100644 --- a/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/DataPrepperPluginSchemaExecute.java +++ b/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/DataPrepperPluginSchemaExecute.java @@ -62,7 +62,7 @@ public void run() { .setUrls(ClasspathHelper.forPackage(DEFAULT_PLUGINS_CLASSPATH)) .setScanners(Scanners.TypesAnnotated, Scanners.SubTypes)); final PluginConfigsJsonSchemaConverter pluginConfigsJsonSchemaConverter = new PluginConfigsJsonSchemaConverter( - reflections, new JsonSchemaConverter(modules), siteUrl, siteBaseUrl); + reflections, new JsonSchemaConverter(modules, reflections), siteUrl, siteBaseUrl); final Class pluginType = pluginConfigsJsonSchemaConverter.pluginTypeNameToPluginType(pluginTypeName); final Map pluginNameToJsonSchemaMap = pluginConfigsJsonSchemaConverter.convertPluginConfigsIntoJsonSchemas( SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, pluginType); diff --git a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java index 0985485952..9410a5e45b 100644 --- a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java +++ b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java @@ -11,15 +11,24 @@ import com.github.victools.jsonschema.generator.SchemaGeneratorConfigBuilder; import com.github.victools.jsonschema.generator.SchemaGeneratorConfigPart; import com.github.victools.jsonschema.generator.SchemaVersion; +import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; +import org.reflections.Reflections; import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class JsonSchemaConverter { static final String DEPRECATED_SINCE_KEY = "deprecated"; + static final String PLUGIN_NAME_KEY = "name"; private final List jsonSchemaGeneratorModules; + private final Reflections reflections; - public JsonSchemaConverter(final List jsonSchemaGeneratorModules) { + public JsonSchemaConverter(final List jsonSchemaGeneratorModules, final Reflections reflections) { this.jsonSchemaGeneratorModules = jsonSchemaGeneratorModules; + this.reflections = reflections; } public ObjectNode convertIntoJsonSchema( @@ -30,6 +39,7 @@ public ObjectNode convertIntoJsonSchema( loadJsonSchemaGeneratorModules(configBuilder); final SchemaGeneratorConfigPart scopeSchemaGeneratorConfigPart = configBuilder.forFields(); overrideInstanceAttributeWithDeprecated(scopeSchemaGeneratorConfigPart); + overrideTargetTypeWithUsesDataPrepperPlugin(scopeSchemaGeneratorConfigPart); resolveDefaultValueFromJsonProperty(scopeSchemaGeneratorConfigPart); final SchemaGeneratorConfig config = configBuilder.build(); @@ -52,6 +62,16 @@ private void overrideInstanceAttributeWithDeprecated( }); } + private void overrideTargetTypeWithUsesDataPrepperPlugin( + final SchemaGeneratorConfigPart scopeSchemaGeneratorConfigPart) { + scopeSchemaGeneratorConfigPart.withTargetTypeOverridesResolver(field -> Optional + .ofNullable(field.getAnnotationConsideringFieldAndGetterIfSupported(UsesDataPrepperPlugin.class)) + .map(usesDataPrepperPlugin -> scanForPluginConfigs(usesDataPrepperPlugin.pluginType())) + .map(stream -> stream.map(specificSubtype -> field.getContext().resolve(specificSubtype))) + .map(stream -> stream.collect(Collectors.toList())) + .orElse(null)); + } + private void resolveDefaultValueFromJsonProperty( final SchemaGeneratorConfigPart scopeSchemaGeneratorConfigPart) { scopeSchemaGeneratorConfigPart.withDefaultResolver(field -> { @@ -59,4 +79,11 @@ private void resolveDefaultValueFromJsonProperty( return annotation == null || annotation.defaultValue().isEmpty() ? null : annotation.defaultValue(); }); } + + private Stream> scanForPluginConfigs(final Class pluginType) { + final Stream result = reflections.getTypesAnnotatedWith(DataPrepperPlugin.class).stream() + .map(clazz -> clazz.getAnnotation(DataPrepperPlugin.class)) + .filter(dataPrepperPlugin -> pluginType.equals(dataPrepperPlugin.pluginType())); + return result.map(DataPrepperPlugin::pluginConfigurationType); + } } diff --git a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java index 67cf0ac527..bfb1dba807 100644 --- a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java +++ b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java @@ -9,26 +9,42 @@ import com.github.victools.jsonschema.generator.Module; import com.github.victools.jsonschema.generator.OptionPreset; import com.github.victools.jsonschema.generator.SchemaVersion; +import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationModule; +import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationOption; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.schemas.module.CustomJacksonModule; +import org.reflections.Reflections; +import org.reflections.scanners.Scanners; +import org.reflections.util.ClasspathHelper; +import org.reflections.util.ConfigurationBuilder; import java.util.Collections; import java.util.List; +import static com.github.victools.jsonschema.module.jackson.JacksonOption.RESPECT_JSONPROPERTY_ORDER; +import static com.github.victools.jsonschema.module.jackson.JacksonOption.RESPECT_JSONPROPERTY_REQUIRED; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.opensearch.dataprepper.schemas.PluginConfigsJsonSchemaConverterIT.DEFAULT_PLUGINS_CLASSPATH; +@ExtendWith(MockitoExtension.class) class JsonSchemaConverterTest { + @Mock + private Reflections reflections; - public JsonSchemaConverter createObjectUnderTest(final List modules) { - return new JsonSchemaConverter(modules); + public JsonSchemaConverter createObjectUnderTest(final List modules, final Reflections reflections) { + return new JsonSchemaConverter(modules, reflections); } @Test void testConvertIntoJsonSchemaWithDefaultModules() throws JsonProcessingException { - final JsonSchemaConverter jsonSchemaConverter = createObjectUnderTest(Collections.emptyList()); + final JsonSchemaConverter jsonSchemaConverter = createObjectUnderTest( + Collections.emptyList(), reflections); final ObjectNode jsonSchemaNode = jsonSchemaConverter.convertIntoJsonSchema( SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfig.class); assertThat(jsonSchemaNode, instanceOf(ObjectNode.class)); @@ -44,7 +60,8 @@ void testConvertIntoJsonSchemaWithDefaultModules() throws JsonProcessingExceptio @Test void testConvertIntoJsonSchemaWithCustomJacksonModule() throws JsonProcessingException { final JsonSchemaConverter jsonSchemaConverter = createObjectUnderTest( - Collections.singletonList(new CustomJacksonModule())); + Collections.singletonList(new CustomJacksonModule()), + reflections); final ObjectNode jsonSchemaNode = jsonSchemaConverter.convertIntoJsonSchema( SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfig.class); assertThat(jsonSchemaNode, instanceOf(ObjectNode.class)); @@ -69,4 +86,18 @@ public String getTestAttributeWithGetter() { return testAttributeWithGetter; } } + + public static void main(String[] args) throws JsonProcessingException { + final List modules = List.of( + new CustomJacksonModule(RESPECT_JSONPROPERTY_REQUIRED, RESPECT_JSONPROPERTY_ORDER), + new JakartaValidationModule(JakartaValidationOption.NOT_NULLABLE_FIELD_IS_REQUIRED, + JakartaValidationOption.INCLUDE_PATTERN_EXPRESSIONS) + ); + final Reflections reflections = new Reflections(new ConfigurationBuilder() + .setUrls(ClasspathHelper.forPackage(DEFAULT_PLUGINS_CLASSPATH)) + .setScanners(Scanners.TypesAnnotated, Scanners.SubTypes)); + JsonSchemaConverter jsonSchemaConverter = new JsonSchemaConverter(modules, reflections); + System.out.println(jsonSchemaConverter.convertIntoJsonSchema( + SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestPluginConfig.class).toPrettyString()); + } } \ No newline at end of file diff --git a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterIT.java b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterIT.java index 71e9bf5faa..0449c844e6 100644 --- a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterIT.java +++ b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterIT.java @@ -50,7 +50,7 @@ void setUp() { .setUrls(ClasspathHelper.forPackage(DEFAULT_PLUGINS_CLASSPATH)) .setScanners(Scanners.TypesAnnotated, Scanners.SubTypes)); objectUnderTest = new PluginConfigsJsonSchemaConverter( - reflections, new JsonSchemaConverter(modules), TEST_URL, TEST_BASE_URL); + reflections, new JsonSchemaConverter(modules, reflections), TEST_URL, TEST_BASE_URL); } @ParameterizedTest From 35e0c9c6f71a83f168f99cf7659289ab30adda19 Mon Sep 17 00:00:00 2001 From: George Chen Date: Tue, 24 Sep 2024 00:15:15 -0500 Subject: [PATCH 02/11] ENH: validate embedded pluginModel and generated embedded schemas Signed-off-by: George Chen --- .../model/configuration/PluginModel.java | 4 +- .../model/configuration/PluginModelTests.java | 51 ++++++++++++ .../dataprepper/plugins/TestPluginConfig.java | 8 ++ .../TestPluginConfigWithDeprecatedName.java | 9 +++ ...n_model_attribute_and_existing_plugin.yaml | 4 + ...del_attribute_but_non_existing_plugin.yaml | 2 + .../schemas/JsonSchemaConverter.java | 35 +++++++-- .../PluginConfigsJsonSchemaConverter.java | 4 +- .../schemas/JsonSchemaConverterIT.java | 78 +++++++++++++++++++ .../schemas/JsonSchemaConverterTest.java | 22 ------ 10 files changed, 187 insertions(+), 30 deletions(-) create mode 100644 data-prepper-api/src/test/java/org/opensearch/dataprepper/plugins/TestPluginConfig.java create mode 100644 data-prepper-api/src/test/java/org/opensearch/dataprepper/plugins/TestPluginConfigWithDeprecatedName.java create mode 100644 data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/test_config_with_plugin_model_attribute_and_existing_plugin.yaml create mode 100644 data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/test_config_with_plugin_model_attribute_but_non_existing_plugin.yaml create mode 100644 data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginModel.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginModel.java index 030ed9dcff..385aa4ecde 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginModel.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginModel.java @@ -186,7 +186,9 @@ protected AbstractPluginModelDeserializer( @Override public AbstractPluginModelDeserializer createContextual( final DeserializationContext context, final BeanProperty property) { - usesDataPrepperPlugin = property.getAnnotation(UsesDataPrepperPlugin.class); + if (Objects.nonNull(property)) { + usesDataPrepperPlugin = property.getAnnotation(UsesDataPrepperPlugin.class); + } return this; } diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/PluginModelTests.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/PluginModelTests.java index 26f8b0758f..47cdf0c62e 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/PluginModelTests.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/PluginModelTests.java @@ -5,6 +5,7 @@ package org.opensearch.dataprepper.model.configuration; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonGenerationException; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; @@ -14,6 +15,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; +import org.opensearch.dataprepper.model.processor.Processor; import java.io.BufferedReader; import java.io.IOException; @@ -27,9 +30,11 @@ import java.util.Map; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasKey; +import static org.junit.jupiter.api.Assertions.assertThrows; class PluginModelTests { @@ -90,6 +95,35 @@ final void deserialize_with_empty_inner(final String resourceName) throws IOExce assertThat(pluginModel.getPluginSettings().size(), equalTo(0)); } + @Test + final void deserialize_PluginModel_attribute_matches_UsesDataPrepperPlugin_annotation() throws IOException { + final InputStream inputStream = PluginModelTests.class.getResourceAsStream( + "test_config_with_plugin_model_attribute_and_existing_plugin.yaml"); + + final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + final TestConfig testConfig = mapper.readValue(inputStream, TestConfig.class); + assertThat(testConfig.getPluginModel(), instanceOf(PluginModel.class)); + assertThat(testConfig.getPluginModel().getPluginName(), equalTo("test_plugin")); + assertThat(testConfig.getPluginModel().getPluginSettings(), notNullValue()); + assertThat(testConfig.getPluginModel().getPluginSettings().size(), equalTo(0)); + assertThat(testConfig.getPluginModelDeprecated(), instanceOf(PluginModel.class)); + assertThat(testConfig.getPluginModelDeprecated().getPluginName(), equalTo("test_plugin_deprecated")); + assertThat(testConfig.getPluginModelDeprecated().getPluginSettings(), notNullValue()); + assertThat(testConfig.getPluginModelDeprecated().getPluginSettings().size(), equalTo(0)); + } + + @Test + final void deserialize_PluginModel_attribute_does_not_match_UsesDataPrepperPlugin_annotation() { + final InputStream inputStream = PluginModelTests.class.getResourceAsStream( + "test_config_with_plugin_model_attribute_but_non_existing_plugin.yaml"); + + final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + final IOException exception = assertThrows( + IOException.class, () -> mapper.readValue(inputStream, TestConfig.class)); + assertThat(exception.getMessage(), equalTo( + "non-existing is not found on interface org.opensearch.dataprepper.model.processor.Processor.")); + } + @Test final void testUsingCustomSerializerWithPluginSettings_noExceptions() throws JsonGenerationException, JsonMappingException, IOException { final PluginModel pluginModel = new PluginModel("customPlugin", validPluginSettings()); @@ -156,4 +190,21 @@ static String convertInputStreamToString(InputStream inputStream) throws IOExcep return stringBuilder.toString(); } + static class TestConfig { + @JsonProperty("plugin_model") + @UsesDataPrepperPlugin(pluginType = Processor.class) + private PluginModel pluginModel; + + @JsonProperty("plugin_model_deprecated") + @UsesDataPrepperPlugin(pluginType = Processor.class) + private PluginModel pluginModelDeprecated; + + public PluginModel getPluginModel() { + return pluginModel; + } + + public PluginModel getPluginModelDeprecated() { + return pluginModelDeprecated; + } + } } diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/plugins/TestPluginConfig.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/plugins/TestPluginConfig.java new file mode 100644 index 0000000000..c945941312 --- /dev/null +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/plugins/TestPluginConfig.java @@ -0,0 +1,8 @@ +package org.opensearch.dataprepper.plugins; + +import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.opensearch.dataprepper.model.processor.Processor; + +@DataPrepperPlugin(name = "test_plugin", pluginType = Processor.class) +public class TestPluginConfig { +} diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/plugins/TestPluginConfigWithDeprecatedName.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/plugins/TestPluginConfigWithDeprecatedName.java new file mode 100644 index 0000000000..43fb711d9b --- /dev/null +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/plugins/TestPluginConfigWithDeprecatedName.java @@ -0,0 +1,9 @@ +package org.opensearch.dataprepper.plugins; + +import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.opensearch.dataprepper.model.processor.Processor; + +@DataPrepperPlugin(name = "test_plugin_config_with_deprecated_name", + deprecatedName = "test_plugin_deprecated", pluginType = Processor.class) +public class TestPluginConfigWithDeprecatedName { +} diff --git a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/test_config_with_plugin_model_attribute_and_existing_plugin.yaml b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/test_config_with_plugin_model_attribute_and_existing_plugin.yaml new file mode 100644 index 0000000000..715b175b14 --- /dev/null +++ b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/test_config_with_plugin_model_attribute_and_existing_plugin.yaml @@ -0,0 +1,4 @@ +plugin_model: + test_plugin: {} +plugin_model_deprecated: + test_plugin_deprecated: {} \ No newline at end of file diff --git a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/test_config_with_plugin_model_attribute_but_non_existing_plugin.yaml b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/test_config_with_plugin_model_attribute_but_non_existing_plugin.yaml new file mode 100644 index 0000000000..1e1475ec2a --- /dev/null +++ b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/test_config_with_plugin_model_attribute_but_non_existing_plugin.yaml @@ -0,0 +1,2 @@ +plugin_model: + non-existing: {} \ No newline at end of file diff --git a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java index 9410a5e45b..22a84186e0 100644 --- a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java +++ b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java @@ -10,10 +10,13 @@ import com.github.victools.jsonschema.generator.SchemaGeneratorConfig; import com.github.victools.jsonschema.generator.SchemaGeneratorConfigBuilder; import com.github.victools.jsonschema.generator.SchemaGeneratorConfigPart; +import com.github.victools.jsonschema.generator.SchemaGeneratorGeneralConfigPart; import com.github.victools.jsonschema.generator.SchemaVersion; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; import org.reflections.Reflections; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; import java.util.Optional; @@ -21,8 +24,8 @@ import java.util.stream.Stream; public class JsonSchemaConverter { + private static final Logger LOG = LoggerFactory.getLogger(JsonSchemaConverter.class); static final String DEPRECATED_SINCE_KEY = "deprecated"; - static final String PLUGIN_NAME_KEY = "name"; private final List jsonSchemaGeneratorModules; private final Reflections reflections; @@ -41,6 +44,7 @@ public ObjectNode convertIntoJsonSchema( overrideInstanceAttributeWithDeprecated(scopeSchemaGeneratorConfigPart); overrideTargetTypeWithUsesDataPrepperPlugin(scopeSchemaGeneratorConfigPart); resolveDefaultValueFromJsonProperty(scopeSchemaGeneratorConfigPart); + overrideDataPrepperPluginTypeAttribute(configBuilder.forTypesInGeneral(), schemaVersion, optionPreset); final SchemaGeneratorConfig config = configBuilder.build(); final SchemaGenerator generator = new SchemaGenerator(config); @@ -72,6 +76,26 @@ private void overrideTargetTypeWithUsesDataPrepperPlugin( .orElse(null)); } + private void overrideDataPrepperPluginTypeAttribute( + final SchemaGeneratorGeneralConfigPart schemaGeneratorGeneralConfigPart, + final SchemaVersion schemaVersion, final OptionPreset optionPreset) { + schemaGeneratorGeneralConfigPart.withTypeAttributeOverride((node, scope, context) -> { + final DataPrepperPlugin dataPrepperPlugin = scope.getType().getErasedType() + .getAnnotation(DataPrepperPlugin.class); + if (dataPrepperPlugin != null) { + final ObjectNode propertiesNode = node.putObject("properties"); + try { + final ObjectNode schemaNode = this.convertIntoJsonSchema( + schemaVersion, optionPreset, dataPrepperPlugin.pluginConfigurationType()); + propertiesNode.set(dataPrepperPlugin.name(), schemaNode); + } catch (JsonProcessingException e) { + LOG.error("Encountered error retrieving JSON schema for {}", dataPrepperPlugin.name(), e); + throw new RuntimeException(e); + } + } + }); + } + private void resolveDefaultValueFromJsonProperty( final SchemaGeneratorConfigPart scopeSchemaGeneratorConfigPart) { scopeSchemaGeneratorConfigPart.withDefaultResolver(field -> { @@ -81,9 +105,10 @@ private void resolveDefaultValueFromJsonProperty( } private Stream> scanForPluginConfigs(final Class pluginType) { - final Stream result = reflections.getTypesAnnotatedWith(DataPrepperPlugin.class).stream() - .map(clazz -> clazz.getAnnotation(DataPrepperPlugin.class)) - .filter(dataPrepperPlugin -> pluginType.equals(dataPrepperPlugin.pluginType())); - return result.map(DataPrepperPlugin::pluginConfigurationType); + return reflections.getTypesAnnotatedWith(DataPrepperPlugin.class).stream() + .filter(clazz -> { + final DataPrepperPlugin dataPrepperPlugin = clazz.getAnnotation(DataPrepperPlugin.class); + return pluginType.equals(dataPrepperPlugin.pluginType()); + }); } } diff --git a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java index b7f4c1a531..8f8cf8cf47 100644 --- a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java +++ b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java @@ -90,8 +90,8 @@ public Map convertPluginConfigsIntoJsonSchemas( addPluginName(jsonSchemaNode, pluginName); addDocumentationLink(jsonSchemaNode, pluginName, pluginType); value = jsonSchemaNode.toPrettyString(); - } catch (JsonProcessingException e) { - LOG.error("Encountered error retrieving JSON schema for {}", pluginName); + } catch (final Exception e) { + LOG.error("Encountered error retrieving JSON schema for {}", pluginName, e); return Stream.empty(); } return Stream.of(Map.entry(entry.getKey(), value)); diff --git a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java new file mode 100644 index 0000000000..a99c30e47a --- /dev/null +++ b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java @@ -0,0 +1,78 @@ +package org.opensearch.dataprepper.schemas; + +import com.fasterxml.jackson.annotation.JsonClassDescription; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.github.victools.jsonschema.generator.Module; +import com.github.victools.jsonschema.generator.OptionPreset; +import com.github.victools.jsonschema.generator.SchemaVersion; +import com.github.victools.jsonschema.module.jackson.JacksonModule; +import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationModule; +import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationOption; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; +import org.opensearch.dataprepper.model.configuration.PluginModel; +import org.opensearch.dataprepper.plugins.processor.aggregate.AggregateAction; +import org.reflections.Reflections; +import org.reflections.scanners.Scanners; +import org.reflections.util.ClasspathHelper; +import org.reflections.util.ConfigurationBuilder; + +import java.util.List; + +import static com.github.victools.jsonschema.module.jackson.JacksonOption.RESPECT_JSONPROPERTY_REQUIRED; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +public class JsonSchemaConverterIT { + static final String DEFAULT_PLUGINS_CLASSPATH = "org.opensearch.dataprepper.plugins"; + static final String PROPERTIES_KEY = "properties"; + static final String ANY_OF_KEY = "anyOf"; + + private JsonSchemaConverter objectUnderTest; + + @BeforeEach + void setUp() { + final List modules = List.of( + new JacksonModule(RESPECT_JSONPROPERTY_REQUIRED), + new JakartaValidationModule(JakartaValidationOption.NOT_NULLABLE_FIELD_IS_REQUIRED, + JakartaValidationOption.INCLUDE_PATTERN_EXPRESSIONS) + ); + final Reflections reflections = new Reflections(new ConfigurationBuilder() + .setUrls(ClasspathHelper.forPackage(DEFAULT_PLUGINS_CLASSPATH)) + .setScanners(Scanners.TypesAnnotated, Scanners.SubTypes)); + objectUnderTest = new JsonSchemaConverter(modules, reflections); + } + + @Test + void testSubTypes() throws JsonProcessingException { + final ObjectNode jsonSchemaNode = objectUnderTest.convertIntoJsonSchema( + SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfig.class); + assertThat(jsonSchemaNode, instanceOf(ObjectNode.class)); + final JsonNode propertiesNode = jsonSchemaNode.at("/" + PROPERTIES_KEY); + assertThat(propertiesNode, instanceOf(ObjectNode.class)); + assertThat(propertiesNode.has("action"), is(true)); + final JsonNode actionNode = propertiesNode.at("/action"); + assertThat(actionNode.has(ANY_OF_KEY), is(true)); + final JsonNode anyOfNode = actionNode.at("/" + ANY_OF_KEY); + assertThat(anyOfNode, instanceOf(ArrayNode.class)); + anyOfNode.forEach(aggregateActionNode -> assertThat(aggregateActionNode.has(PROPERTIES_KEY), is(true))); + } + + @JsonClassDescription("test config") + static class TestConfig { + @JsonPropertyDescription("The aggregate action description") + @UsesDataPrepperPlugin(pluginType = AggregateAction.class) + private PluginModel action; + + public PluginModel getAction() { + return action; + } + } +} diff --git a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java index bfb1dba807..c719016b32 100644 --- a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java +++ b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java @@ -9,28 +9,20 @@ import com.github.victools.jsonschema.generator.Module; import com.github.victools.jsonschema.generator.OptionPreset; import com.github.victools.jsonschema.generator.SchemaVersion; -import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationModule; -import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationOption; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.schemas.module.CustomJacksonModule; import org.reflections.Reflections; -import org.reflections.scanners.Scanners; -import org.reflections.util.ClasspathHelper; -import org.reflections.util.ConfigurationBuilder; import java.util.Collections; import java.util.List; -import static com.github.victools.jsonschema.module.jackson.JacksonOption.RESPECT_JSONPROPERTY_ORDER; -import static com.github.victools.jsonschema.module.jackson.JacksonOption.RESPECT_JSONPROPERTY_REQUIRED; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import static org.opensearch.dataprepper.schemas.PluginConfigsJsonSchemaConverterIT.DEFAULT_PLUGINS_CLASSPATH; @ExtendWith(MockitoExtension.class) class JsonSchemaConverterTest { @@ -86,18 +78,4 @@ public String getTestAttributeWithGetter() { return testAttributeWithGetter; } } - - public static void main(String[] args) throws JsonProcessingException { - final List modules = List.of( - new CustomJacksonModule(RESPECT_JSONPROPERTY_REQUIRED, RESPECT_JSONPROPERTY_ORDER), - new JakartaValidationModule(JakartaValidationOption.NOT_NULLABLE_FIELD_IS_REQUIRED, - JakartaValidationOption.INCLUDE_PATTERN_EXPRESSIONS) - ); - final Reflections reflections = new Reflections(new ConfigurationBuilder() - .setUrls(ClasspathHelper.forPackage(DEFAULT_PLUGINS_CLASSPATH)) - .setScanners(Scanners.TypesAnnotated, Scanners.SubTypes)); - JsonSchemaConverter jsonSchemaConverter = new JsonSchemaConverter(modules, reflections); - System.out.println(jsonSchemaConverter.convertIntoJsonSchema( - SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestPluginConfig.class).toPrettyString()); - } } \ No newline at end of file From 4030ee1d7462a3afbb252c13ac844594b926be0d Mon Sep 17 00:00:00 2001 From: George Chen Date: Tue, 24 Sep 2024 00:26:57 -0500 Subject: [PATCH 03/11] MAINT: backfill annotations Signed-off-by: George Chen --- .../dataprepper/model/annotations/UsesDataPrepperPlugin.java | 3 +++ .../plugins/processor/aggregate/AggregateProcessorConfig.java | 2 ++ .../anomalydetector/AnomalyDetectorProcessorConfig.java | 2 ++ .../processor/obfuscation/ObfuscationProcessorConfig.java | 3 +++ 4 files changed, 10 insertions(+) diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/UsesDataPrepperPlugin.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/UsesDataPrepperPlugin.java index 8d81290df3..e94e2cca4c 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/UsesDataPrepperPlugin.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/UsesDataPrepperPlugin.java @@ -6,6 +6,9 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +/** + * Annotates a field that uses Data Prepper plugin config as its value. + */ @Documented @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.FIELD}) diff --git a/data-prepper-plugins/aggregate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/aggregate/AggregateProcessorConfig.java b/data-prepper-plugins/aggregate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/aggregate/AggregateProcessorConfig.java index 40ed0f14d7..f1c0124a01 100644 --- a/data-prepper-plugins/aggregate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/aggregate/AggregateProcessorConfig.java +++ b/data-prepper-plugins/aggregate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/aggregate/AggregateProcessorConfig.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.annotation.JsonClassDescription; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; import org.opensearch.dataprepper.model.configuration.PluginModel; import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.fasterxml.jackson.annotation.JsonProperty; @@ -37,6 +38,7 @@ public class AggregateProcessorConfig { @JsonPropertyDescription("The action to be performed on each group. One of the available aggregate actions must be provided.") @JsonProperty("action") @NotNull + @UsesDataPrepperPlugin(pluginType = AggregateAction.class) private PluginModel aggregateAction; @JsonPropertyDescription("When local_mode is set to true, the aggregation is performed locally on each Data Prepper node instead of forwarding events to a specific node based on the identification_keys using a hash function. Default is false.") diff --git a/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/AnomalyDetectorProcessorConfig.java b/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/AnomalyDetectorProcessorConfig.java index 0eb59edc58..2122a0f318 100644 --- a/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/AnomalyDetectorProcessorConfig.java +++ b/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/AnomalyDetectorProcessorConfig.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.annotation.JsonClassDescription; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; import org.opensearch.dataprepper.model.configuration.PluginModel; import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.fasterxml.jackson.annotation.JsonProperty; @@ -23,6 +24,7 @@ public class AnomalyDetectorProcessorConfig { @JsonPropertyDescription("The ML algorithm (or model) used to detect anomalies. You must provide a mode. See random_cut_forest mode.") @JsonProperty("mode") @NotNull + @UsesDataPrepperPlugin(pluginType = AnomalyDetectorMode.class) private PluginModel detectorMode; @JsonPropertyDescription("A non-ordered List that is used as input to the ML algorithm to detect anomalies in the values of the keys in the list. At least one key is required.") diff --git a/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java b/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java index fa4df27ce1..5b9ef682f5 100644 --- a/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java +++ b/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java @@ -12,8 +12,10 @@ import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; +import org.opensearch.dataprepper.plugins.processor.obfuscation.action.ObfuscationAction; import java.util.List; @@ -40,6 +42,7 @@ public class ObfuscationProcessorConfig { @JsonProperty("action") @JsonPropertyDescription("The obfuscation action. Available actions include 'hash' and 'mask'.") + @UsesDataPrepperPlugin(pluginType = ObfuscationAction.class) private PluginModel action; @JsonProperty("obfuscate_when") From 80ef6700c9b624ca29fc88bb2e5778feac5b0faf Mon Sep 17 00:00:00 2001 From: George Chen Date: Tue, 24 Sep 2024 00:31:17 -0500 Subject: [PATCH 04/11] STY: fix errors Signed-off-by: George Chen --- .../dataprepper/schemas/PluginConfigsJsonSchemaConverter.java | 1 - .../opensearch/dataprepper/schemas/JsonSchemaConverterIT.java | 1 - 2 files changed, 2 deletions(-) diff --git a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java index 8f8cf8cf47..8995aa32bc 100644 --- a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java +++ b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java @@ -1,6 +1,5 @@ package org.opensearch.dataprepper.schemas; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.node.ObjectNode; import com.github.victools.jsonschema.generator.OptionPreset; import com.github.victools.jsonschema.generator.SchemaVersion; diff --git a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java index a99c30e47a..709089cead 100644 --- a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java +++ b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java @@ -1,7 +1,6 @@ package org.opensearch.dataprepper.schemas; import com.fasterxml.jackson.annotation.JsonClassDescription; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; From 1eaa56de7a54158d973c31ef78504d9c497bbb28 Mon Sep 17 00:00:00 2001 From: George Chen Date: Tue, 24 Sep 2024 11:22:57 -0500 Subject: [PATCH 05/11] MAINT: fix dependency Signed-off-by: George Chen --- data-prepper-api/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-api/build.gradle b/data-prepper-api/build.gradle index 5ff379f10c..d47c3e9a84 100644 --- a/data-prepper-api/build.gradle +++ b/data-prepper-api/build.gradle @@ -12,7 +12,7 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8' - implementation 'org.reflections:reflections:0.10.2' + implementation libs.reflections.core implementation libs.parquet.common implementation libs.commons.lang3 testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' From 254b737e16374082f05fa353395c8d9250160b7d Mon Sep 17 00:00:00 2001 From: George Chen Date: Tue, 24 Sep 2024 11:32:55 -0500 Subject: [PATCH 06/11] FIX: bump reflection version Signed-off-by: George Chen --- settings.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle b/settings.gradle index 4328fa9aac..ec5a86f255 100644 --- a/settings.gradle +++ b/settings.gradle @@ -54,7 +54,7 @@ dependencyResolutionManagement { library('bouncycastle-bcpkix', 'org.bouncycastle', 'bcpkix-jdk18on').versionRef('bouncycastle') version('guava', '32.1.2-jre') library('guava-core', 'com.google.guava', 'guava').versionRef('guava') - version('reflections', '0.9.12') + version('reflections', '0.10.2') library('reflections-core', 'org.reflections', 'reflections').versionRef('reflections') library('commons-lang3', 'org.apache.commons', 'commons-lang3').version('3.14.0') library('commons-io', 'commons-io', 'commons-io').version('2.15.1') From 07b69ba1ab767eafc9da760dfd874528b5e0ea97 Mon Sep 17 00:00:00 2001 From: George Chen Date: Tue, 1 Oct 2024 17:23:52 -0500 Subject: [PATCH 07/11] REF: use scan for plugins in plugin-framework Signed-off-by: George Chen --- .../model/configuration/PluginModel.java | 52 +----------------- .../model/configuration/PluginModelTests.java | 51 ----------------- .../plugin/ClasspathPluginProvider.java | 17 ++++++ .../dataprepper/plugin/PluginProvider.java | 12 ++++ .../plugin/ClasspathPluginProviderTest.java | 55 +++++++++++++++++++ data-prepper-plugin-schema-cli/build.gradle | 1 + .../DataPrepperPluginSchemaExecute.java | 12 ++-- .../schemas/JsonSchemaConverter.java | 18 ++---- .../PluginConfigsJsonSchemaConverter.java | 10 ++-- .../schemas/JsonSchemaConverterIT.java | 13 ++--- .../schemas/JsonSchemaConverterTest.java | 12 ++-- .../PluginConfigsJsonSchemaConverterIT.java | 12 ++-- 12 files changed, 116 insertions(+), 149 deletions(-) diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginModel.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginModel.java index 385aa4ecde..d8b2b41054 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginModel.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginModel.java @@ -11,33 +11,22 @@ import com.fasterxml.jackson.core.JacksonException; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.BeanProperty; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.fasterxml.jackson.databind.deser.ContextualDeserializer; import com.fasterxml.jackson.databind.deser.std.StdDeserializer; import com.fasterxml.jackson.databind.ser.std.StdSerializer; -import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; -import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; -import org.reflections.Reflections; -import org.reflections.scanners.Scanners; -import org.reflections.util.ClasspathHelper; -import org.reflections.util.ConfigurationBuilder; import java.io.IOException; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.function.BiFunction; import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.Stream; /** * Model class for a Plugin in Configuration YAML containing name of the Plugin and its associated settings @@ -47,7 +36,7 @@ @JsonSerialize(using = PluginModel.PluginModelSerializer.class) @JsonDeserialize(using = PluginModel.PluginModelDeserializer.class) public class PluginModel { - static final String DEFAULT_PLUGINS_CLASSPATH = "org.opensearch.dataprepper.plugins"; + private static final ObjectMapper SERIALIZER_OBJECT_MAPPER = new ObjectMapper(); private final String pluginName; @@ -159,15 +148,11 @@ public PluginModelDeserializer() { * * @see SinkModel.SinkModelDeserializer */ - abstract static class AbstractPluginModelDeserializer - extends StdDeserializer implements ContextualDeserializer { + abstract static class AbstractPluginModelDeserializer extends StdDeserializer { private final Class innerModelClass; private final BiFunction constructorFunction; private final Supplier emptyInnerModelConstructor; - private final Reflections reflections; - - private UsesDataPrepperPlugin usesDataPrepperPlugin; protected AbstractPluginModelDeserializer( final Class valueClass, @@ -178,18 +163,6 @@ protected AbstractPluginModelDeserializer( this.innerModelClass = innerModelClass; this.constructorFunction = constructorFunction; this.emptyInnerModelConstructor = emptyInnerModelConstructor; - this.reflections = new Reflections(new ConfigurationBuilder() - .setUrls(ClasspathHelper.forPackage(DEFAULT_PLUGINS_CLASSPATH)) - .setScanners(Scanners.TypesAnnotated, Scanners.SubTypes)); - } - - @Override - public AbstractPluginModelDeserializer createContextual( - final DeserializationContext context, final BeanProperty property) { - if (Objects.nonNull(property)) { - usesDataPrepperPlugin = property.getAnnotation(UsesDataPrepperPlugin.class); - } - return this; } @Override @@ -202,33 +175,12 @@ public PluginModel deserialize(final JsonParser jsonParser, final Deserializatio final String pluginName = onlyField.getKey(); final JsonNode value = onlyField.getValue(); - if (usesDataPrepperPlugin != null) { - final Set pluginNames = scanForPluginNames(usesDataPrepperPlugin.pluginType()); - if (!pluginNames.contains(pluginName)) { - throw new IOException(String.format("%s is not found on %s.", - pluginName, usesDataPrepperPlugin.pluginType())); - } - } - M innerModel = SERIALIZER_OBJECT_MAPPER.convertValue(value, innerModelClass); if(innerModel == null) innerModel = emptyInnerModelConstructor.get(); return constructorFunction.apply(pluginName, innerModel); } - - private Set scanForPluginNames(final Class pluginType) { - return reflections.getTypesAnnotatedWith(DataPrepperPlugin.class).stream() - .map(clazz -> clazz.getAnnotation(DataPrepperPlugin.class)) - .filter(dataPrepperPlugin -> pluginType.equals(dataPrepperPlugin.pluginType())) - .flatMap(dataPrepperPlugin -> { - if (!dataPrepperPlugin.deprecatedName().isEmpty()) { - return Stream.of(dataPrepperPlugin.deprecatedName(), dataPrepperPlugin.name()); - } - return Stream.of(dataPrepperPlugin.name()); - }) - .collect(Collectors.toSet()); - } } } diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/PluginModelTests.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/PluginModelTests.java index 47cdf0c62e..26f8b0758f 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/PluginModelTests.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/PluginModelTests.java @@ -5,7 +5,6 @@ package org.opensearch.dataprepper.model.configuration; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonGenerationException; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; @@ -15,8 +14,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; -import org.opensearch.dataprepper.model.processor.Processor; import java.io.BufferedReader; import java.io.IOException; @@ -30,11 +27,9 @@ import java.util.Map; import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasKey; -import static org.junit.jupiter.api.Assertions.assertThrows; class PluginModelTests { @@ -95,35 +90,6 @@ final void deserialize_with_empty_inner(final String resourceName) throws IOExce assertThat(pluginModel.getPluginSettings().size(), equalTo(0)); } - @Test - final void deserialize_PluginModel_attribute_matches_UsesDataPrepperPlugin_annotation() throws IOException { - final InputStream inputStream = PluginModelTests.class.getResourceAsStream( - "test_config_with_plugin_model_attribute_and_existing_plugin.yaml"); - - final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); - final TestConfig testConfig = mapper.readValue(inputStream, TestConfig.class); - assertThat(testConfig.getPluginModel(), instanceOf(PluginModel.class)); - assertThat(testConfig.getPluginModel().getPluginName(), equalTo("test_plugin")); - assertThat(testConfig.getPluginModel().getPluginSettings(), notNullValue()); - assertThat(testConfig.getPluginModel().getPluginSettings().size(), equalTo(0)); - assertThat(testConfig.getPluginModelDeprecated(), instanceOf(PluginModel.class)); - assertThat(testConfig.getPluginModelDeprecated().getPluginName(), equalTo("test_plugin_deprecated")); - assertThat(testConfig.getPluginModelDeprecated().getPluginSettings(), notNullValue()); - assertThat(testConfig.getPluginModelDeprecated().getPluginSettings().size(), equalTo(0)); - } - - @Test - final void deserialize_PluginModel_attribute_does_not_match_UsesDataPrepperPlugin_annotation() { - final InputStream inputStream = PluginModelTests.class.getResourceAsStream( - "test_config_with_plugin_model_attribute_but_non_existing_plugin.yaml"); - - final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); - final IOException exception = assertThrows( - IOException.class, () -> mapper.readValue(inputStream, TestConfig.class)); - assertThat(exception.getMessage(), equalTo( - "non-existing is not found on interface org.opensearch.dataprepper.model.processor.Processor.")); - } - @Test final void testUsingCustomSerializerWithPluginSettings_noExceptions() throws JsonGenerationException, JsonMappingException, IOException { final PluginModel pluginModel = new PluginModel("customPlugin", validPluginSettings()); @@ -190,21 +156,4 @@ static String convertInputStreamToString(InputStream inputStream) throws IOExcep return stringBuilder.toString(); } - static class TestConfig { - @JsonProperty("plugin_model") - @UsesDataPrepperPlugin(pluginType = Processor.class) - private PluginModel pluginModel; - - @JsonProperty("plugin_model_deprecated") - @UsesDataPrepperPlugin(pluginType = Processor.class) - private PluginModel pluginModelDeprecated; - - public PluginModel getPluginModel() { - return pluginModel; - } - - public PluginModel getPluginModelDeprecated() { - return pluginModelDeprecated; - } - } } diff --git a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ClasspathPluginProvider.java b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ClasspathPluginProvider.java index 764c83f4db..83ceca9317 100644 --- a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ClasspathPluginProvider.java +++ b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ClasspathPluginProvider.java @@ -11,13 +11,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.opensearch.dataprepper.model.annotations.DataPrepperPlugin.DEFAULT_ALTERNATE_NAME; import static org.opensearch.dataprepper.model.annotations.DataPrepperPlugin.DEFAULT_DEPRECATED_NAME; @@ -60,6 +63,20 @@ public Optional> findPluginClass(final Class pluginTyp return Optional.ofNullable((Class) supportedTypesMap.get(pluginType)); } + @Override + public Set> findPluginClasses(Class pluginType) { + if (nameToSupportedTypeToPluginType == null) { + nameToSupportedTypeToPluginType = scanForPlugins(); + } + + return nameToSupportedTypeToPluginType.values().stream() + .flatMap(supportedTypeToPluginType -> + supportedTypeToPluginType.entrySet().stream() + .filter(entry -> pluginType.equals(entry.getKey())) + .flatMap(entry -> Stream.of((Class) entry.getValue()))) + .collect(Collectors.toSet()); + } + private Map, Class>> scanForPlugins() { final Set> dataPrepperPluginClasses = reflections.getTypesAnnotatedWith(DataPrepperPlugin.class); diff --git a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/PluginProvider.java b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/PluginProvider.java index dd15176569..3aff8b0e3c 100644 --- a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/PluginProvider.java +++ b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/PluginProvider.java @@ -5,6 +5,7 @@ package org.opensearch.dataprepper.plugin; +import java.util.Collection; import java.util.Optional; /** @@ -27,4 +28,15 @@ public interface PluginProvider { * @since 1.2 */ Optional> findPluginClass(Class pluginType, String pluginName); + + /** + * Finds the Java classes for a specific pluginType. + * + * @param pluginType The type of plugin which is being supported. + * e.g. {@link org.opensearch.dataprepper.model.sink.Sink}. + * @param The type + * @return An {@link Collection} of Java classes for plugins + * @since 1.2 + */ + Collection> findPluginClasses(Class pluginType); } diff --git a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ClasspathPluginProviderTest.java b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ClasspathPluginProviderTest.java index 6cda169636..d4e977bf63 100644 --- a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ClasspathPluginProviderTest.java +++ b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ClasspathPluginProviderTest.java @@ -11,6 +11,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.sink.Sink; import org.opensearch.dataprepper.model.source.Source; import org.opensearch.dataprepper.plugins.test.TestSink; @@ -18,13 +19,17 @@ import org.reflections.Reflections; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.UUID; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.mockito.BDDMockito.given; @@ -61,6 +66,22 @@ void findPlugin_should_scan_for_plugins() { .getTypesAnnotatedWith(DataPrepperPlugin.class); } + @Test + void findPlugins_should_scan_for_plugins() { + final ClasspathPluginProvider objectUnderTest = createObjectUnderTest(); + + then(reflections).shouldHaveNoInteractions(); + + given(reflections.getTypesAnnotatedWith(DataPrepperPlugin.class)) + .willReturn(Collections.emptySet()); + + objectUnderTest.findPluginClasses(Sink.class); + + then(reflections) + .should() + .getTypesAnnotatedWith(DataPrepperPlugin.class); + } + @Test void findPlugin_should_scan_for_plugins_only_once() { final ClasspathPluginProvider objectUnderTest = createObjectUnderTest(); @@ -76,6 +97,21 @@ void findPlugin_should_scan_for_plugins_only_once() { .getTypesAnnotatedWith(DataPrepperPlugin.class); } + @Test + void findPlugins_should_scan_for_plugins_only_once() { + final ClasspathPluginProvider objectUnderTest = createObjectUnderTest(); + + given(reflections.getTypesAnnotatedWith(DataPrepperPlugin.class)) + .willReturn(Collections.emptySet()); + + for (int i = 0; i < 10; i++) + objectUnderTest.findPluginClasses(Sink.class); + + then(reflections) + .should() + .getTypesAnnotatedWith(DataPrepperPlugin.class); + } + @Test void findPlugin_should_return_empty_if_no_plugins_found() { given(reflections.getTypesAnnotatedWith(DataPrepperPlugin.class)) @@ -130,6 +166,17 @@ void findPlugin_should_return_plugin_if_found_for_alternate_name_and_type_using_ assertThat(optionalPlugin.get(), equalTo(TestSource.class)); } + @Test + void findPlugins_should_return_empty_if_no_plugins_found() { + given(reflections.getTypesAnnotatedWith(DataPrepperPlugin.class)) + .willReturn(Collections.emptySet()); + + final Collection> foundPlugins = createObjectUnderTest().findPluginClasses( + PluginSetting.class); + assertThat(foundPlugins, notNullValue()); + assertThat(foundPlugins.isEmpty(), is(true)); + } + @Nested class WithPredefinedPlugins { @@ -161,5 +208,13 @@ void findPlugin_should_return_plugin_if_found_for_name_and_type_using_pluginType assertThat(optionalPlugin.isPresent(), equalTo(true)); assertThat(optionalPlugin.get(), equalTo(TestSink.class)); } + + @Test + void findPlugins_should_return_plugins_if_plugin_found_for_specified_type() { + final Set> foundPlugins = createObjectUnderTest().findPluginClasses(Source.class); + assertThat(foundPlugins, notNullValue()); + assertThat(foundPlugins.size(), equalTo(1)); + assertThat(foundPlugins.stream().iterator().next(), equalTo(TestSource.class)); + } } } \ No newline at end of file diff --git a/data-prepper-plugin-schema-cli/build.gradle b/data-prepper-plugin-schema-cli/build.gradle index fdcf46cf92..830a401ae5 100644 --- a/data-prepper-plugin-schema-cli/build.gradle +++ b/data-prepper-plugin-schema-cli/build.gradle @@ -9,6 +9,7 @@ application { dependencies { implementation project(':data-prepper-plugins') + implementation project(':data-prepper-plugin-framework') implementation project(':data-prepper-plugin-schema') implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'org.reflections:reflections:0.10.2' diff --git a/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/DataPrepperPluginSchemaExecute.java b/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/DataPrepperPluginSchemaExecute.java index d8cd0f12b2..fd817631a6 100644 --- a/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/DataPrepperPluginSchemaExecute.java +++ b/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/DataPrepperPluginSchemaExecute.java @@ -5,11 +5,9 @@ import com.github.victools.jsonschema.generator.SchemaVersion; import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationModule; import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationOption; +import org.opensearch.dataprepper.plugin.ClasspathPluginProvider; +import org.opensearch.dataprepper.plugin.PluginProvider; import org.opensearch.dataprepper.schemas.module.CustomJacksonModule; -import org.reflections.Reflections; -import org.reflections.scanners.Scanners; -import org.reflections.util.ClasspathHelper; -import org.reflections.util.ConfigurationBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine; @@ -58,11 +56,9 @@ public void run() { new JakartaValidationModule(JakartaValidationOption.NOT_NULLABLE_FIELD_IS_REQUIRED, JakartaValidationOption.INCLUDE_PATTERN_EXPRESSIONS) ); - final Reflections reflections = new Reflections(new ConfigurationBuilder() - .setUrls(ClasspathHelper.forPackage(DEFAULT_PLUGINS_CLASSPATH)) - .setScanners(Scanners.TypesAnnotated, Scanners.SubTypes)); + final PluginProvider pluginProvider = new ClasspathPluginProvider(); final PluginConfigsJsonSchemaConverter pluginConfigsJsonSchemaConverter = new PluginConfigsJsonSchemaConverter( - reflections, new JsonSchemaConverter(modules, reflections), siteUrl, siteBaseUrl); + pluginProvider, new JsonSchemaConverter(modules, pluginProvider), siteUrl, siteBaseUrl); final Class pluginType = pluginConfigsJsonSchemaConverter.pluginTypeNameToPluginType(pluginTypeName); final Map pluginNameToJsonSchemaMap = pluginConfigsJsonSchemaConverter.convertPluginConfigsIntoJsonSchemas( SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, pluginType); diff --git a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java index 22a84186e0..03edb58b85 100644 --- a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java +++ b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java @@ -14,6 +14,7 @@ import com.github.victools.jsonschema.generator.SchemaVersion; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; +import org.opensearch.dataprepper.plugin.PluginProvider; import org.reflections.Reflections; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,11 +28,11 @@ public class JsonSchemaConverter { private static final Logger LOG = LoggerFactory.getLogger(JsonSchemaConverter.class); static final String DEPRECATED_SINCE_KEY = "deprecated"; private final List jsonSchemaGeneratorModules; - private final Reflections reflections; + private final PluginProvider pluginProvider; - public JsonSchemaConverter(final List jsonSchemaGeneratorModules, final Reflections reflections) { + public JsonSchemaConverter(final List jsonSchemaGeneratorModules, final PluginProvider pluginProvider) { this.jsonSchemaGeneratorModules = jsonSchemaGeneratorModules; - this.reflections = reflections; + this.pluginProvider = pluginProvider; } public ObjectNode convertIntoJsonSchema( @@ -70,7 +71,8 @@ private void overrideTargetTypeWithUsesDataPrepperPlugin( final SchemaGeneratorConfigPart scopeSchemaGeneratorConfigPart) { scopeSchemaGeneratorConfigPart.withTargetTypeOverridesResolver(field -> Optional .ofNullable(field.getAnnotationConsideringFieldAndGetterIfSupported(UsesDataPrepperPlugin.class)) - .map(usesDataPrepperPlugin -> scanForPluginConfigs(usesDataPrepperPlugin.pluginType())) + .map(usesDataPrepperPlugin -> + pluginProvider.findPluginClasses(usesDataPrepperPlugin.pluginType()).stream()) .map(stream -> stream.map(specificSubtype -> field.getContext().resolve(specificSubtype))) .map(stream -> stream.collect(Collectors.toList())) .orElse(null)); @@ -103,12 +105,4 @@ private void resolveDefaultValueFromJsonProperty( return annotation == null || annotation.defaultValue().isEmpty() ? null : annotation.defaultValue(); }); } - - private Stream> scanForPluginConfigs(final Class pluginType) { - return reflections.getTypesAnnotatedWith(DataPrepperPlugin.class).stream() - .filter(clazz -> { - final DataPrepperPlugin dataPrepperPlugin = clazz.getAnnotation(DataPrepperPlugin.class); - return pluginType.equals(dataPrepperPlugin.pluginType()); - }); - } } diff --git a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java index 8995aa32bc..eb9c29dd63 100644 --- a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java +++ b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java @@ -9,7 +9,7 @@ import org.opensearch.dataprepper.model.processor.Processor; import org.opensearch.dataprepper.model.sink.Sink; import org.opensearch.dataprepper.model.source.Source; -import org.reflections.Reflections; +import org.opensearch.dataprepper.plugin.PluginProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,15 +50,15 @@ public class PluginConfigsJsonSchemaConverter { private final String siteUrl; private final String siteBaseUrl; - private final Reflections reflections; + private final PluginProvider pluginProvider; private final JsonSchemaConverter jsonSchemaConverter; public PluginConfigsJsonSchemaConverter( - final Reflections reflections, + final PluginProvider pluginProvider, final JsonSchemaConverter jsonSchemaConverter, final String siteUrl, final String siteBaseUrl) { - this.reflections = reflections; + this.pluginProvider = pluginProvider; this.jsonSchemaConverter = jsonSchemaConverter; this.siteUrl = siteUrl == null ? SITE_URL_PLACEHOLDER : siteUrl; this.siteBaseUrl = siteBaseUrl == null ? SITE_BASE_URL_PLACEHOLDER : siteBaseUrl; @@ -106,7 +106,7 @@ private Map> scanForPluginConfigs(final Class pluginType) { if (ConditionalRoute.class.equals(pluginType)) { return Map.of(CONDITIONAL_ROUTE_PROCESSOR_NAME, ConditionalRoute.class); } - return reflections.getTypesAnnotatedWith(DataPrepperPlugin.class).stream() + return pluginProvider.findPluginClasses(pluginType).stream() .map(clazz -> clazz.getAnnotation(DataPrepperPlugin.class)) .filter(dataPrepperPlugin -> pluginType.equals(dataPrepperPlugin.pluginType())) .collect(Collectors.toMap( diff --git a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java index 709089cead..7a3dca5991 100644 --- a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java +++ b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java @@ -16,11 +16,9 @@ import org.junit.jupiter.api.Test; import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; import org.opensearch.dataprepper.model.configuration.PluginModel; +import org.opensearch.dataprepper.plugin.ClasspathPluginProvider; +import org.opensearch.dataprepper.plugin.PluginProvider; import org.opensearch.dataprepper.plugins.processor.aggregate.AggregateAction; -import org.reflections.Reflections; -import org.reflections.scanners.Scanners; -import org.reflections.util.ClasspathHelper; -import org.reflections.util.ConfigurationBuilder; import java.util.List; @@ -30,7 +28,6 @@ import static org.hamcrest.MatcherAssert.assertThat; public class JsonSchemaConverterIT { - static final String DEFAULT_PLUGINS_CLASSPATH = "org.opensearch.dataprepper.plugins"; static final String PROPERTIES_KEY = "properties"; static final String ANY_OF_KEY = "anyOf"; @@ -43,10 +40,8 @@ void setUp() { new JakartaValidationModule(JakartaValidationOption.NOT_NULLABLE_FIELD_IS_REQUIRED, JakartaValidationOption.INCLUDE_PATTERN_EXPRESSIONS) ); - final Reflections reflections = new Reflections(new ConfigurationBuilder() - .setUrls(ClasspathHelper.forPackage(DEFAULT_PLUGINS_CLASSPATH)) - .setScanners(Scanners.TypesAnnotated, Scanners.SubTypes)); - objectUnderTest = new JsonSchemaConverter(modules, reflections); + final PluginProvider pluginProvider = new ClasspathPluginProvider(); + objectUnderTest = new JsonSchemaConverter(modules, pluginProvider); } @Test diff --git a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java index c719016b32..3100370685 100644 --- a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java +++ b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java @@ -13,8 +13,8 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugin.PluginProvider; import org.opensearch.dataprepper.schemas.module.CustomJacksonModule; -import org.reflections.Reflections; import java.util.Collections; import java.util.List; @@ -27,16 +27,16 @@ @ExtendWith(MockitoExtension.class) class JsonSchemaConverterTest { @Mock - private Reflections reflections; + private PluginProvider pluginProvider; - public JsonSchemaConverter createObjectUnderTest(final List modules, final Reflections reflections) { - return new JsonSchemaConverter(modules, reflections); + public JsonSchemaConverter createObjectUnderTest(final List modules, final PluginProvider pluginProvider) { + return new JsonSchemaConverter(modules, pluginProvider); } @Test void testConvertIntoJsonSchemaWithDefaultModules() throws JsonProcessingException { final JsonSchemaConverter jsonSchemaConverter = createObjectUnderTest( - Collections.emptyList(), reflections); + Collections.emptyList(), pluginProvider); final ObjectNode jsonSchemaNode = jsonSchemaConverter.convertIntoJsonSchema( SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfig.class); assertThat(jsonSchemaNode, instanceOf(ObjectNode.class)); @@ -53,7 +53,7 @@ void testConvertIntoJsonSchemaWithDefaultModules() throws JsonProcessingExceptio void testConvertIntoJsonSchemaWithCustomJacksonModule() throws JsonProcessingException { final JsonSchemaConverter jsonSchemaConverter = createObjectUnderTest( Collections.singletonList(new CustomJacksonModule()), - reflections); + pluginProvider); final ObjectNode jsonSchemaNode = jsonSchemaConverter.convertIntoJsonSchema( SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfig.class); assertThat(jsonSchemaNode, instanceOf(ObjectNode.class)); diff --git a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterIT.java b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterIT.java index 0449c844e6..d825a3472f 100644 --- a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterIT.java +++ b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterIT.java @@ -13,10 +13,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.reflections.Reflections; -import org.reflections.scanners.Scanners; -import org.reflections.util.ClasspathHelper; -import org.reflections.util.ConfigurationBuilder; +import org.opensearch.dataprepper.plugin.ClasspathPluginProvider; +import org.opensearch.dataprepper.plugin.PluginProvider; import java.util.List; import java.util.Map; @@ -46,11 +44,9 @@ void setUp() { new JakartaValidationModule(JakartaValidationOption.NOT_NULLABLE_FIELD_IS_REQUIRED, JakartaValidationOption.INCLUDE_PATTERN_EXPRESSIONS) ); - final Reflections reflections = new Reflections(new ConfigurationBuilder() - .setUrls(ClasspathHelper.forPackage(DEFAULT_PLUGINS_CLASSPATH)) - .setScanners(Scanners.TypesAnnotated, Scanners.SubTypes)); + final PluginProvider pluginProvider = new ClasspathPluginProvider(); objectUnderTest = new PluginConfigsJsonSchemaConverter( - reflections, new JsonSchemaConverter(modules, reflections), TEST_URL, TEST_BASE_URL); + pluginProvider, new JsonSchemaConverter(modules, pluginProvider), TEST_URL, TEST_BASE_URL); } @ParameterizedTest From 323773fa12dbdc4fd29503142429625d2599cd14 Mon Sep 17 00:00:00 2001 From: George Chen Date: Tue, 1 Oct 2024 17:25:51 -0500 Subject: [PATCH 08/11] RMV: unused files Signed-off-by: George Chen --- .../opensearch/dataprepper/plugins/TestPluginConfig.java | 8 -------- .../plugins/TestPluginConfigWithDeprecatedName.java | 9 --------- ..._with_plugin_model_attribute_and_existing_plugin.yaml | 4 ---- ...h_plugin_model_attribute_but_non_existing_plugin.yaml | 2 -- 4 files changed, 23 deletions(-) delete mode 100644 data-prepper-api/src/test/java/org/opensearch/dataprepper/plugins/TestPluginConfig.java delete mode 100644 data-prepper-api/src/test/java/org/opensearch/dataprepper/plugins/TestPluginConfigWithDeprecatedName.java delete mode 100644 data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/test_config_with_plugin_model_attribute_and_existing_plugin.yaml delete mode 100644 data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/test_config_with_plugin_model_attribute_but_non_existing_plugin.yaml diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/plugins/TestPluginConfig.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/plugins/TestPluginConfig.java deleted file mode 100644 index c945941312..0000000000 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/plugins/TestPluginConfig.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.opensearch.dataprepper.plugins; - -import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; -import org.opensearch.dataprepper.model.processor.Processor; - -@DataPrepperPlugin(name = "test_plugin", pluginType = Processor.class) -public class TestPluginConfig { -} diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/plugins/TestPluginConfigWithDeprecatedName.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/plugins/TestPluginConfigWithDeprecatedName.java deleted file mode 100644 index 43fb711d9b..0000000000 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/plugins/TestPluginConfigWithDeprecatedName.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.opensearch.dataprepper.plugins; - -import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; -import org.opensearch.dataprepper.model.processor.Processor; - -@DataPrepperPlugin(name = "test_plugin_config_with_deprecated_name", - deprecatedName = "test_plugin_deprecated", pluginType = Processor.class) -public class TestPluginConfigWithDeprecatedName { -} diff --git a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/test_config_with_plugin_model_attribute_and_existing_plugin.yaml b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/test_config_with_plugin_model_attribute_and_existing_plugin.yaml deleted file mode 100644 index 715b175b14..0000000000 --- a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/test_config_with_plugin_model_attribute_and_existing_plugin.yaml +++ /dev/null @@ -1,4 +0,0 @@ -plugin_model: - test_plugin: {} -plugin_model_deprecated: - test_plugin_deprecated: {} \ No newline at end of file diff --git a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/test_config_with_plugin_model_attribute_but_non_existing_plugin.yaml b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/test_config_with_plugin_model_attribute_but_non_existing_plugin.yaml deleted file mode 100644 index 1e1475ec2a..0000000000 --- a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/test_config_with_plugin_model_attribute_but_non_existing_plugin.yaml +++ /dev/null @@ -1,2 +0,0 @@ -plugin_model: - non-existing: {} \ No newline at end of file From 817d76403f31c237e4477b3234d0108268752185 Mon Sep 17 00:00:00 2001 From: George Chen Date: Tue, 1 Oct 2024 17:35:17 -0500 Subject: [PATCH 09/11] FIX: sty and broken tests Signed-off-by: George Chen --- .../dataprepper/schemas/JsonSchemaConverter.java | 2 -- .../schemas/PluginConfigsJsonSchemaConverterTest.java | 10 +++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java index 03edb58b85..7172bbbd02 100644 --- a/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java +++ b/data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java @@ -15,14 +15,12 @@ import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin; import org.opensearch.dataprepper.plugin.PluginProvider; -import org.reflections.Reflections; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; -import java.util.stream.Stream; public class JsonSchemaConverter { private static final Logger LOG = LoggerFactory.getLogger(JsonSchemaConverter.class); diff --git a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterTest.java b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterTest.java index 3d1c1b585a..39c5629e0c 100644 --- a/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterTest.java +++ b/data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterTest.java @@ -12,7 +12,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; -import org.reflections.Reflections; +import org.opensearch.dataprepper.plugin.PluginProvider; import java.util.Map; import java.util.Set; @@ -40,7 +40,7 @@ class PluginConfigsJsonSchemaConverterTest { private JsonSchemaConverter jsonSchemaConverter; @Mock - private Reflections reflections; + private PluginProvider pluginProvider; @InjectMocks private PluginConfigsJsonSchemaConverter objectUnderTest; @@ -67,7 +67,7 @@ void testPluginTypeNameToPluginTypeWithInValidInput() { @Test void testConvertPluginConfigsIntoJsonSchemasHappyPath() throws JsonProcessingException { - when(reflections.getTypesAnnotatedWith(eq(DataPrepperPlugin.class))).thenReturn(Set.of(TestPlugin.class)); + when(pluginProvider.findPluginClasses(eq(TestPluginType.class))).thenReturn(Set.of(TestPlugin.class)); final ObjectNode objectNode = OBJECT_MAPPER.createObjectNode(); when(jsonSchemaConverter.convertIntoJsonSchema( any(SchemaVersion.class), any(OptionPreset.class), eq(TestPluginConfig.class))).thenReturn(objectNode); @@ -84,7 +84,7 @@ void testConvertPluginConfigsIntoJsonSchemasHappyPath() throws JsonProcessingExc @Test void testConvertPluginConfigsIntoJsonSchemasWithError() throws JsonProcessingException { - when(reflections.getTypesAnnotatedWith(eq(DataPrepperPlugin.class))).thenReturn(Set.of(TestPlugin.class)); + when(pluginProvider.findPluginClasses(eq(TestPluginType.class))).thenReturn(Set.of(TestPlugin.class)); final JsonProcessingException jsonProcessingException = mock(JsonProcessingException.class); when(jsonSchemaConverter.convertIntoJsonSchema( any(SchemaVersion.class), any(OptionPreset.class), eq(TestPluginConfig.class))).thenThrow( @@ -96,7 +96,7 @@ void testConvertPluginConfigsIntoJsonSchemasWithError() throws JsonProcessingExc @DataPrepperPlugin( name = "test_plugin", pluginType = TestPluginType.class, pluginConfigurationType = TestPluginConfig.class) - static class TestPlugin { + static class TestPlugin extends TestPluginType { } From 60ad37d6be4b5b65af084ce4d41b5d48e30a9660 Mon Sep 17 00:00:00 2001 From: George Chen Date: Tue, 1 Oct 2024 17:36:49 -0500 Subject: [PATCH 10/11] FIX: sty Signed-off-by: George Chen --- .../opensearch/dataprepper/plugin/ClasspathPluginProvider.java | 2 -- .../dataprepper/plugin/ClasspathPluginProviderTest.java | 1 - 2 files changed, 3 deletions(-) diff --git a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ClasspathPluginProvider.java b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ClasspathPluginProvider.java index 83ceca9317..df29915cd1 100644 --- a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ClasspathPluginProvider.java +++ b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ClasspathPluginProvider.java @@ -11,9 +11,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collection; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; diff --git a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ClasspathPluginProviderTest.java b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ClasspathPluginProviderTest.java index d4e977bf63..56ec0f4167 100644 --- a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ClasspathPluginProviderTest.java +++ b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ClasspathPluginProviderTest.java @@ -28,7 +28,6 @@ import java.util.UUID; import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; From 3c9aa7f7a6dbdc893134f608e20990f177267ace Mon Sep 17 00:00:00 2001 From: George Chen Date: Wed, 2 Oct 2024 11:56:45 -0500 Subject: [PATCH 11/11] MAINT: remove unused dependency Signed-off-by: George Chen --- data-prepper-api/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/data-prepper-api/build.gradle b/data-prepper-api/build.gradle index d47c3e9a84..bf0f0aebd6 100644 --- a/data-prepper-api/build.gradle +++ b/data-prepper-api/build.gradle @@ -12,7 +12,6 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8' - implementation libs.reflections.core implementation libs.parquet.common implementation libs.commons.lang3 testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'