diff --git a/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/PipelineTransformationConfiguration.java b/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/PipelineTransformationConfiguration.java index 9cf83b11d7..07a476641e 100644 --- a/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/PipelineTransformationConfiguration.java +++ b/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/PipelineTransformationConfiguration.java @@ -10,15 +10,11 @@ import org.springframework.context.annotation.Configuration; import javax.inject.Named; -import java.nio.file.Path; -import java.nio.file.Paths; @Configuration public class PipelineTransformationConfiguration { public static final String TEMPLATES_DIRECTORY_PATH = "TEMPLATES_DIRECTORY_PATH"; public static final String RULES_DIRECTORY_PATH = "RULES_DIRECTORY_PATH"; - private static final Path currentDir = Paths.get(System.getProperty("user.dir")); -// private static final String parserRelativePath = "/data-prepper-pipeline-parser/src"; @Bean @Named(RULES_DIRECTORY_PATH) diff --git a/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluator.java b/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluator.java index e546d5fd10..9278416379 100644 --- a/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluator.java +++ b/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluator.java @@ -5,15 +5,14 @@ package org.opensearch.dataprepper.pipeline.parser.rule; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.Option; -import com.jayway.jsonpath.ParseContext; import com.jayway.jsonpath.PathNotFoundException; -import com.jayway.jsonpath.ReadContext; -import com.jayway.jsonpath.spi.json.JacksonJsonProvider; +import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; import org.opensearch.dataprepper.model.configuration.PipelineModel; import org.opensearch.dataprepper.model.configuration.PipelinesDataFlowModel; @@ -25,6 +24,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Path; import java.util.List; import java.util.Map; @@ -34,40 +34,28 @@ public class RuleEvaluator { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory()); private final TransformersFactory transformersFactory; - private String PLUGIN_NAME = null; public RuleEvaluator(TransformersFactory transformersFactory) { this.transformersFactory = transformersFactory; } public RuleEvaluatorResult isTransformationNeeded(PipelinesDataFlowModel pipelineModel) { - //TODO - Dynamically scan the rules folder and get the corresponding template. - return isDocDBSource(pipelineModel); - } - /** - * Evaluates model based on pre defined rules and - * result contains the name of the pipeline that will need transformation, - * evaluated boolean result and the corresponding template model - * Assumption: only one pipeline can have transformation. - * - * @param pipelinesModel - * @return RuleEvaluatorResult - */ - private RuleEvaluatorResult isDocDBSource(PipelinesDataFlowModel pipelinesModel) { - PLUGIN_NAME = "documentdb"; - - Map pipelines = pipelinesModel.getPipelines(); + Map pipelines = pipelineModel.getPipelines(); for (Map.Entry entry : pipelines.entrySet()) { try { String pipelineJson = OBJECT_MAPPER.writeValueAsString(entry); - if (evaluate(pipelineJson, PLUGIN_NAME)) { - LOG.info("Rule for {} is evaluated true for pipelineJson {}", PLUGIN_NAME, pipelineJson); + RuleFileEvaluation ruleFileEvaluation = evaluate(pipelineJson); + + if (ruleFileEvaluation.result) { + String pluginName = ruleFileEvaluation.pluginName; + LOG.info("Applying rule {}",ruleFileEvaluation.ruleFileName.toString()); + LOG.info("Rule for {} is evaluated true for pipelineJson {}", pluginName, pipelineJson); - InputStream templateStream = transformersFactory.getPluginTemplateFileStream(PLUGIN_NAME); + InputStream templateStream = transformersFactory.getPluginTemplateFileStream(pluginName); PipelineTemplateModel templateModel = yamlMapper.readValue(templateStream, PipelineTemplateModel.class); - LOG.info("Template is chosen for {}", PLUGIN_NAME); + LOG.info("Template is chosen for {}", pluginName); return RuleEvaluatorResult.builder() .withEvaluatedResult(true) @@ -75,11 +63,10 @@ private RuleEvaluatorResult isDocDBSource(PipelinesDataFlowModel pipelinesModel) .withPipelineName(entry.getKey()) .build(); } - } catch (FileNotFoundException e){ - LOG.error("Template File Not Found for {}", PLUGIN_NAME); + } catch (FileNotFoundException e) { + LOG.error("Template File Not Found"); throw new RuntimeException(e); - } - catch (JsonProcessingException e) { + } catch (JsonProcessingException e) { LOG.error("Error processing json"); throw new RuntimeException(e); } catch (IOException e) { @@ -94,38 +81,63 @@ private RuleEvaluatorResult isDocDBSource(PipelinesDataFlowModel pipelinesModel) .build(); } - private Boolean evaluate(String pipelinesJson, - String pluginName) { + private RuleFileEvaluation evaluate(String pipelinesJson) { Configuration parseConfig = Configuration.builder() - .jsonProvider(new JacksonJsonProvider()) + .jsonProvider(new JacksonJsonNodeJsonProvider()) .mappingProvider(new JacksonMappingProvider()) - .options(Option.AS_PATH_LIST) + .options(Option.SUPPRESS_EXCEPTIONS) .build(); - ParseContext parseContext = JsonPath.using(parseConfig); - ReadContext readPathContext = parseContext.parse(pipelinesJson); RuleTransformerModel rulesModel = null; InputStream ruleStream = null; try { - ruleStream = transformersFactory.getPluginRuleFileStream(pluginName); + List ruleFiles = transformersFactory.getRuleFiles(); - rulesModel = yamlMapper.readValue(ruleStream, RuleTransformerModel.class); - List rules = rulesModel.getApplyWhen(); - for (String rule : rules) { - try { - Object result = readPathContext.read(rule); - } catch (PathNotFoundException e) { - LOG.warn("Json Path not found for {}", pluginName); - return false; + for (Path ruleFile : ruleFiles) { + + ruleStream = transformersFactory.readRuleFile(ruleFile); + if(ruleStream == null){ + continue; + } + rulesModel = yamlMapper.readValue(ruleStream, RuleTransformerModel.class); + List rules = rulesModel.getApplyWhen(); + String pluginName = rulesModel.getPluginName(); + boolean allRulesValid = true; + + for (String rule : rules) { + try { + JsonNode result = JsonPath.using(parseConfig).parse(pipelinesJson).read(rule); + if (result == null || result.size()==0) { + allRulesValid = false; + break; + } + } catch (PathNotFoundException e) { + LOG.debug("Json Path not found for {}", ruleFile.getFileName().toString()); + allRulesValid = false; + break; + } + } + + if (allRulesValid) { + return RuleFileEvaluation.builder() + .withPluginName(pluginName) + .withRuleFileName(ruleFile.getFileName().toString()) + .withResult(true) + .build(); } } - } catch (FileNotFoundException e){ - LOG.warn("Rule File Not Found for {}", pluginName); - return false; - } catch(IOException e){ + + } catch (FileNotFoundException e) { + LOG.debug("Rule File Not Found"); + return RuleFileEvaluation.builder() + .withPluginName(null) + .withRuleFileName(null) + .withResult(false) + .build(); + } catch (IOException e) { throw new RuntimeException(e); - }finally { + } finally { if (ruleStream != null) { try { ruleStream.close(); @@ -134,7 +146,10 @@ private Boolean evaluate(String pipelinesJson, } } } - return true; + return RuleFileEvaluation.builder() + .withPluginName(null) + .withRuleFileName(null) + .withResult(false) + .build(); } -} - +} \ No newline at end of file diff --git a/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleFileEvaluation.java b/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleFileEvaluation.java new file mode 100644 index 0000000000..e819afe878 --- /dev/null +++ b/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleFileEvaluation.java @@ -0,0 +1,18 @@ +package org.opensearch.dataprepper.pipeline.parser.rule; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; + +@Builder(setterPrefix = "with") +@AllArgsConstructor +@Data +public class RuleFileEvaluation { + Boolean result; + String ruleFileName; + String pluginName; + + public RuleFileEvaluation() { + + } +} diff --git a/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleTransformerModel.java b/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleTransformerModel.java index 0ad9e45b72..aaf757cedb 100644 --- a/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleTransformerModel.java +++ b/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleTransformerModel.java @@ -4,36 +4,31 @@ */ package org.opensearch.dataprepper.pipeline.parser.rule; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; import lombok.Data; import java.util.List; @Data +@AllArgsConstructor public class RuleTransformerModel { + @JsonInclude(JsonInclude.Include.NON_EMPTY) @JsonProperty("apply_when") private List applyWhen; - public RuleTransformerModel() { - } - - public RuleTransformerModel(List applyWhen) { - this.applyWhen = applyWhen; - } + @JsonProperty("plugin_name") + private String pluginName; - public List getApplyWhen() { - return applyWhen; - } - - public void setApplyWhen(List applyWhen) { - this.applyWhen = applyWhen; + public RuleTransformerModel() { } @Override public String toString() { return "RuleConfiguration{" + "applyWhen=" + applyWhen + - '}'; + "\npluginName="+ pluginName +'}'; } } \ No newline at end of file diff --git a/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/transformer/TransformersFactory.java b/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/transformer/TransformersFactory.java index 95741e9cf6..8e12f96317 100644 --- a/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/transformer/TransformersFactory.java +++ b/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/transformer/TransformersFactory.java @@ -14,6 +14,18 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class TransformersFactory implements PipelineTransformationPathProvider { @@ -41,35 +53,63 @@ public String getTransformationRulesDirectoryLocation() { } public String getPluginTemplateFileLocation(String pluginName) { - if(pluginName == null || pluginName.isEmpty()){ - throw new RuntimeException("Transformation plugin not found"); + if (pluginName == null || pluginName.isEmpty()) { + throw new RuntimeException("Transformation plugin not found"); } return templatesDirectoryPath + "/" + pluginName + TEMPLATE_FILE_NAME_PATTERN; } - public String getPluginRuleFileLocation(String pluginName) { - if(pluginName == null || pluginName.isEmpty()){ - throw new RuntimeException("Transformation plugin not found"); + public InputStream getPluginTemplateFileStream(String pluginName) { + if (pluginName == null || pluginName.isEmpty()) { + throw new RuntimeException("Transformation plugin not found"); } - return rulesDirectoryPath + "/" + pluginName + RULE_FILE_NAME_PATTERN; + ClassLoader classLoader = TransformersFactory.class.getClassLoader(); + InputStream filestream = classLoader.getResourceAsStream("templates" + "/" + pluginName + TEMPLATE_FILE_NAME_PATTERN); + return filestream; } - public InputStream getPluginRuleFileStream(String pluginName) { - if(pluginName == null || pluginName.isEmpty()){ - throw new RuntimeException("Transformation plugin not found"); + public List getRuleFiles() { + // Get the URI of the rules folder + URI uri = null; + try { + uri = getClass().getClassLoader().getResource("rules").toURI(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + + Path rulesFolderPath; + + if ("jar".equals(uri.getScheme())) { + // File is inside a JAR, create a filesystem for it + try (FileSystem fileSystem = FileSystems.newFileSystem(uri, Collections.emptyMap())) { + rulesFolderPath = fileSystem.getPath("rules"); + return scanFolder(rulesFolderPath); + } catch (IOException e) { + throw new RuntimeException(e); + } + } else { + // File is not inside a JAR + rulesFolderPath = Paths.get(uri); + return scanFolder(rulesFolderPath); } - ClassLoader classLoader = TransformersFactory.class.getClassLoader(); - InputStream filestream = classLoader.getResourceAsStream("rules" + "/" + pluginName + RULE_FILE_NAME_PATTERN); - return filestream; } - public InputStream getPluginTemplateFileStream(String pluginName) { - if(pluginName == null || pluginName.isEmpty()){ - throw new RuntimeException("Transformation plugin not found"); + private List scanFolder(Path folderPath) { + List pathsList = new ArrayList<>(); + try (Stream paths = Files.walk(folderPath)) { + pathsList = paths + .filter(Files::isRegularFile) // Filter to include only regular files + .collect(Collectors.toList()); // Collect paths into the list + } catch (IOException e) { + throw new RuntimeException(e); } + return pathsList; + } + + public InputStream readRuleFile(Path ruleFile) throws IOException { ClassLoader classLoader = TransformersFactory.class.getClassLoader(); - InputStream filestream = classLoader.getResourceAsStream("templates" + "/" + pluginName + TEMPLATE_FILE_NAME_PATTERN); - return filestream; + InputStream ruleStream = classLoader.getResourceAsStream("rules" + "/" + ruleFile.getFileName().toString()); + return ruleStream; } public PipelineTemplateModel getTemplateModel(String pluginName) { diff --git a/data-prepper-pipeline-parser/src/main/resources/rules/documentdb-rule.yaml b/data-prepper-pipeline-parser/src/main/resources/rules/documentdb-rule.yaml index e7279ee733..60aa428d8a 100644 --- a/data-prepper-pipeline-parser/src/main/resources/rules/documentdb-rule.yaml +++ b/data-prepper-pipeline-parser/src/main/resources/rules/documentdb-rule.yaml @@ -1,3 +1,4 @@ +plugin_name: "documentdb" apply_when: - "$..source.documentdb" - "$..source.documentdb.s3_bucket" \ No newline at end of file diff --git a/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluatorTest.java b/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluatorTest.java index 361feb3558..df00ddab07 100644 --- a/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluatorTest.java +++ b/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluatorTest.java @@ -5,10 +5,11 @@ package org.opensearch.dataprepper.pipeline.parser.rule; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import org.opensearch.dataprepper.model.configuration.PipelineExtensions; @@ -20,8 +21,10 @@ import org.opensearch.dataprepper.pipeline.parser.transformer.TransformersFactory; import java.io.FileInputStream; -import java.io.FileNotFoundException; +import java.io.IOException; import java.io.InputStream; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -31,16 +34,16 @@ class RuleEvaluatorTest { @Test - void test_isTransformationNeeded_ForDocDBSource_ShouldReturn_True() throws FileNotFoundException { + void test_isTransformationNeeded_ForDocDBSource_ShouldReturn_True() throws IOException { - // Set up String ruleDocDBFilePath = TestConfigurationProvider.RULES_TRANSFORMATION_DOCUMENTDB_CONFIG_FILE; + String ruleDocDBTemplatePath = TestConfigurationProvider.TEMPLATE_TRANSFORMATION_DOCDB1_CONFIG_FILE; String pluginName = "documentdb"; String pipelineName = "test-pipeline"; - Map sourceOptions = new HashMap(); - Map s3_bucket = new HashMap<>(); + Map sourceOptions = new HashMap<>(); + Map s3_bucket = new HashMap<>(); s3_bucket.put("s3_bucket", "bucket-name"); - List collections = new ArrayList(); + List> collections = new ArrayList<>(); collections.add(s3_bucket); sourceOptions.put("collections", collections); final PluginModel source = new PluginModel(pluginName, sourceOptions); @@ -51,54 +54,60 @@ void test_isTransformationNeeded_ForDocDBSource_ShouldReturn_True() throws FileN final PipelinesDataFlowModel pipelinesDataFlowModel = new PipelinesDataFlowModel( (PipelineExtensions) null, Collections.singletonMap(pipelineName, pipelineModel)); - TransformersFactory transformersFactory = Mockito.spy(new TransformersFactory( - TestConfigurationProvider.RULES_TRANSFORMATION_DIRECTORY, - TestConfigurationProvider.TEMPLATES_SOURCE_TRANSFORMATION_DIRECTORY - )); - RuleEvaluator ruleEvaluator = new RuleEvaluator(transformersFactory); - when(transformersFactory.getPluginRuleFileLocation(pluginName)).thenReturn(ruleDocDBFilePath); + TransformersFactory transformersFactory = mock(TransformersFactory.class); + Path ruleFile = mock(Path.class); + List ruleFiles = Collections.singletonList(ruleFile); InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); - when(transformersFactory.getPluginRuleFileStream(pluginName)).thenReturn(ruleStream); + InputStream templateStream = new FileInputStream(ruleDocDBTemplatePath); + when(ruleFile.getFileName()).thenReturn(Paths.get("documentdb-rule.yaml").getFileName()); + when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles); + when(transformersFactory.readRuleFile(eq(ruleFile))).thenReturn(ruleStream); + when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); + + RuleEvaluator ruleEvaluator = new RuleEvaluator(transformersFactory); RuleEvaluatorResult result = ruleEvaluator.isTransformationNeeded(pipelinesDataFlowModel); - // Assert assertTrue(result.isEvaluatedResult()); assertEquals(result.getPipelineName(), pipelineName); } @Test - void test_isTransformationNeeded_ForOtherSource_ShouldReturn_False() { - // Set up + void test_isTransformationNeeded_ForOtherSource_ShouldReturn_False() throws IOException { + + String pluginName = "http"; String pipelineName = "test-pipeline"; - Map sourceOptions = new HashMap(); - sourceOptions.put("option1", "1"); - sourceOptions.put("option2", null); - final PluginModel source = new PluginModel("http", sourceOptions); + Map sourceOptions = new HashMap<>(); + Map s3_bucket = new HashMap<>(); + s3_bucket.put("s3_bucket", "bucket-name"); + List> collections = new ArrayList<>(); + collections.add(s3_bucket); + sourceOptions.put("collections", collections); + final PluginModel source = new PluginModel(pluginName, sourceOptions); final List processors = Collections.singletonList(new PluginModel("testProcessor", null)); final List sinks = Collections.singletonList(new SinkModel("testSink", Collections.emptyList(), null, Collections.emptyList(), Collections.emptyList(), null)); final PipelineModel pipelineModel = new PipelineModel(source, null, processors, null, sinks, 8, 50); + TransformersFactory transformersFactory = mock(TransformersFactory.class); + + when(transformersFactory.getRuleFiles()).thenReturn(List.of()); + + final PipelinesDataFlowModel pipelinesDataFlowModel = new PipelinesDataFlowModel( (PipelineExtensions) null, Collections.singletonMap(pipelineName, pipelineModel)); - TransformersFactory transformersFactory = Mockito.spy(new TransformersFactory( - TestConfigurationProvider.RULES_TRANSFORMATION_DIRECTORY, - TestConfigurationProvider.TEMPLATES_SOURCE_TRANSFORMATION_DIRECTORY - )); RuleEvaluator ruleEvaluator = new RuleEvaluator(transformersFactory); RuleEvaluatorResult result = ruleEvaluator.isTransformationNeeded(pipelinesDataFlowModel); - // Assert - assertEquals(result.isEvaluatedResult(), false); + assertFalse(result.isEvaluatedResult()); } @Test void testThrowsExceptionOnFileError() { TransformersFactory transformersFactory = mock(TransformersFactory.class); String pipelineName = "test-pipeline"; - Map sourceOptions = new HashMap(); + Map sourceOptions = new HashMap<>(); sourceOptions.put("option1", "1"); sourceOptions.put("option2", null); final PluginModel source = new PluginModel("http", sourceOptions); @@ -109,16 +118,14 @@ void testThrowsExceptionOnFileError() { final PipelinesDataFlowModel pipelinesDataFlowModel = new PipelinesDataFlowModel( (PipelineExtensions) null, Collections.singletonMap(pipelineName, pipelineModel)); - // Setup mock to throw an exception when file path is incorrect - when(transformersFactory.getPluginRuleFileLocation("documentdb")).thenThrow(new RuntimeException("File not found")); - when(transformersFactory.getPluginRuleFileStream("documentdb")).thenThrow(new RuntimeException("File not found")); + when(transformersFactory.getRuleFiles()).thenThrow(new RuntimeException("File not found")); RuleEvaluator ruleEvaluator = new RuleEvaluator(transformersFactory); - // Execute and Assert Exception exception = assertThrows(RuntimeException.class, () -> { ruleEvaluator.isTransformationNeeded(pipelinesDataFlowModel); }); + assertEquals("File not found", exception.getMessage()); } } diff --git a/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleTransformerModelTest.java b/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleTransformerModelTest.java index 2cd6f0fad4..02a6370229 100644 --- a/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleTransformerModelTest.java +++ b/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleTransformerModelTest.java @@ -19,7 +19,8 @@ class RuleTransformerModelTest { @Test void testSerialization() throws Exception { List applyWhen = Arrays.asList("condition1", "condition2"); - RuleTransformerModel model = new RuleTransformerModel(applyWhen); + String pluginName = "testPlugin"; + RuleTransformerModel model = new RuleTransformerModel(applyWhen, pluginName); String json = objectMapper.writeValueAsString(model); assertNotNull(json, "Serialized JSON should not be null"); @@ -27,12 +28,13 @@ void testSerialization() throws Exception { @Test void testDeserialization() throws Exception { - String json = "{\"apply_when\":[\"condition1\",\"condition2\"]}"; + String json = "{\"plugin_name\": \"testPlugin\", \"apply_when\": [\"condition1\", \"condition2\"]}"; RuleTransformerModel model = objectMapper.readValue(json, RuleTransformerModel.class); assertNotNull(model, "Deserialized model should not be null"); assertEquals(2, model.getApplyWhen().size(), "ApplyWhen should contain two conditions"); assertEquals("condition1", model.getApplyWhen().get(0), "The first condition should be 'condition1'"); assertEquals("condition2", model.getApplyWhen().get(1), "The second condition should be 'condition2'"); + assertEquals("testPlugin", model.getPluginName(), "plugin Name"); } } diff --git a/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/transformer/DynamicConfigTransformerTest.java b/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/transformer/DynamicConfigTransformerTest.java index c5e0aea18d..86ac0a6b68 100644 --- a/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/transformer/DynamicConfigTransformerTest.java +++ b/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/transformer/DynamicConfigTransformerTest.java @@ -11,7 +11,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import org.opensearch.dataprepper.model.configuration.PipelinesDataFlowModel; import org.opensearch.dataprepper.pipeline.parser.PipelineConfigurationFileReader; @@ -24,6 +24,10 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.List; import java.util.Map; class DynamicConfigTransformerTest { @@ -48,15 +52,19 @@ void test_successful_transformation_with_only_source_and_sink() throws IOExcepti final PipelinesDataflowModelParser pipelinesDataflowModelParser = new PipelinesDataflowModelParser(pipelineConfigurationReader); - transformersFactory = Mockito.spy(new TransformersFactory(RULES_DIRECTORY_PATH, - TEMPLATES_DIRECTORY_PATH)); - when(transformersFactory.getPluginRuleFileLocation(pluginName)).thenReturn(ruleDocDBFilePath); - when(transformersFactory.getPluginTemplateFileLocation(pluginName)).thenReturn(templateDocDBFilePath); + TransformersFactory transformersFactory = mock(TransformersFactory.class); + + Path ruleFile = mock(Path.class); + when(ruleFile.getFileName()).thenReturn(Paths.get(ruleDocDBFilePath).getFileName()); InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); InputStream templateStream = new FileInputStream(templateDocDBFilePath); - when(transformersFactory.getPluginRuleFileStream(pluginName)).thenReturn(ruleStream); + when(transformersFactory.readRuleFile(ruleFile)).thenReturn(ruleStream); + List ruleFiles = Collections.singletonList(ruleFile); + when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles); when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); - ruleEvaluator = new RuleEvaluator(transformersFactory); + + RuleEvaluator ruleEvaluator = new RuleEvaluator(transformersFactory); + // Load the original and template YAML files from the test resources directory PipelinesDataFlowModel pipelinesDataFlowModel = pipelinesDataflowModelParser.parseConfiguration(); @@ -81,13 +89,14 @@ void test_successful_transformation_with_documentdb() throws IOException { final PipelinesDataflowModelParser pipelinesDataflowModelParser = new PipelinesDataflowModelParser(pipelineConfigurationReader); - transformersFactory = Mockito.spy(new TransformersFactory(RULES_DIRECTORY_PATH, - TEMPLATES_DIRECTORY_PATH)); - when(transformersFactory.getPluginRuleFileLocation(pluginName)).thenReturn(ruleDocDBFilePath); - when(transformersFactory.getPluginTemplateFileLocation(pluginName)).thenReturn(templateDocDBFilePath); + TransformersFactory transformersFactory = mock(TransformersFactory.class); + Path ruleFile = mock(Path.class); + when(ruleFile.getFileName()).thenReturn(Paths.get(ruleDocDBFilePath).getFileName()); InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); InputStream templateStream = new FileInputStream(templateDocDBFilePath); - when(transformersFactory.getPluginRuleFileStream(pluginName)).thenReturn(ruleStream); + when(transformersFactory.readRuleFile(ruleFile)).thenReturn(ruleStream); + List ruleFiles = Collections.singletonList(ruleFile); + when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles); when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); ruleEvaluator = new RuleEvaluator(transformersFactory); @@ -115,14 +124,16 @@ void test_successful_transformation_with_subpipelines() throws IOException { final PipelinesDataflowModelParser pipelinesDataflowModelParser = new PipelinesDataflowModelParser(pipelineConfigurationReader); - transformersFactory = Mockito.spy(new TransformersFactory(RULES_DIRECTORY_PATH, - TEMPLATES_DIRECTORY_PATH)); - when(transformersFactory.getPluginRuleFileLocation(pluginName)).thenReturn(ruleDocDBFilePath); - when(transformersFactory.getPluginTemplateFileLocation(pluginName)).thenReturn(templateDocDBFilePath); + TransformersFactory transformersFactory = mock(TransformersFactory.class); + Path ruleFile = mock(Path.class); + when(ruleFile.getFileName()).thenReturn(Paths.get(ruleDocDBFilePath).getFileName()); + InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); InputStream ruleStream1 = new FileInputStream(ruleDocDBFilePath); InputStream ruleStream2 = new FileInputStream(ruleDocDBFilePath); InputStream templateStream = new FileInputStream(templateDocDBFilePath); - when(transformersFactory.getPluginRuleFileStream(pluginName)).thenReturn(ruleStream1).thenReturn(ruleStream2); + when(transformersFactory.readRuleFile(ruleFile)).thenReturn(ruleStream).thenReturn(ruleStream1).thenReturn(ruleStream2); + List ruleFiles = Collections.singletonList(ruleFile); + when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles); when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); ruleEvaluator = new RuleEvaluator(transformersFactory); @@ -149,13 +160,14 @@ void test_successful_transformation_with_functionPlaceholder() throws IOExceptio final PipelinesDataflowModelParser pipelinesDataflowModelParser = new PipelinesDataflowModelParser(pipelineConfigurationReader); - transformersFactory = Mockito.spy(new TransformersFactory(RULES_DIRECTORY_PATH, - TEMPLATES_DIRECTORY_PATH)); - when(transformersFactory.getPluginRuleFileLocation(pluginName)).thenReturn(ruleDocDBFilePath); - when(transformersFactory.getPluginTemplateFileLocation(pluginName)).thenReturn(templateDocDBFilePath); + TransformersFactory transformersFactory = mock(TransformersFactory.class); + Path ruleFile = mock(Path.class); + when(ruleFile.getFileName()).thenReturn(Paths.get(ruleDocDBFilePath).getFileName()); InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); InputStream templateStream = new FileInputStream(templateDocDBFilePath); - when(transformersFactory.getPluginRuleFileStream(pluginName)).thenReturn(ruleStream); + when(transformersFactory.readRuleFile(ruleFile)).thenReturn(ruleStream); + List ruleFiles = Collections.singletonList(ruleFile); + when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles); when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); ruleEvaluator = new RuleEvaluator(transformersFactory); @@ -183,13 +195,14 @@ void test_successful_transformation_with_complete_template() throws IOException final PipelinesDataflowModelParser pipelinesDataflowModelParser = new PipelinesDataflowModelParser(pipelineConfigurationReader); - transformersFactory = Mockito.spy(new TransformersFactory(RULES_DIRECTORY_PATH, - TEMPLATES_DIRECTORY_PATH)); - when(transformersFactory.getPluginRuleFileLocation(pluginName)).thenReturn(ruleDocDBFilePath); - when(transformersFactory.getPluginTemplateFileLocation(pluginName)).thenReturn(templateDocDBFilePath); + TransformersFactory transformersFactory = mock(TransformersFactory.class); + Path ruleFile = mock(Path.class); + when(ruleFile.getFileName()).thenReturn(Paths.get(ruleDocDBFilePath).getFileName()); InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); InputStream templateStream = new FileInputStream(templateDocDBFilePath); - when(transformersFactory.getPluginRuleFileStream(pluginName)).thenReturn(ruleStream); + when(transformersFactory.readRuleFile(ruleFile)).thenReturn(ruleStream); + List ruleFiles = Collections.singletonList(ruleFile); + when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles); when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); ruleEvaluator = new RuleEvaluator(transformersFactory); @@ -217,13 +230,14 @@ void test_successful_transformation_with_routes_keyword() throws IOException { final PipelinesDataflowModelParser pipelinesDataflowModelParser = new PipelinesDataflowModelParser(pipelineConfigurationReader); - transformersFactory = Mockito.spy(new TransformersFactory(RULES_DIRECTORY_PATH, - TEMPLATES_DIRECTORY_PATH)); - when(transformersFactory.getPluginRuleFileLocation(pluginName)).thenReturn(ruleDocDBFilePath); - when(transformersFactory.getPluginTemplateFileLocation(pluginName)).thenReturn(templateDocDBFilePath); + TransformersFactory transformersFactory = mock(TransformersFactory.class); + Path ruleFile = mock(Path.class); + when(ruleFile.getFileName()).thenReturn(Paths.get(ruleDocDBFilePath).getFileName()); InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); InputStream templateStream = new FileInputStream(templateDocDBFilePath); - when(transformersFactory.getPluginRuleFileStream(pluginName)).thenReturn(ruleStream); + when(transformersFactory.readRuleFile(ruleFile)).thenReturn(ruleStream); + List ruleFiles = Collections.singletonList(ruleFile); + when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles); when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); ruleEvaluator = new RuleEvaluator(transformersFactory); @@ -240,7 +254,6 @@ void test_successful_transformation_with_routes_keyword() throws IOException { } - @Test void test_successful_transformation_with_route_keyword() throws IOException { @@ -254,13 +267,14 @@ void test_successful_transformation_with_route_keyword() throws IOException { final PipelinesDataflowModelParser pipelinesDataflowModelParser = new PipelinesDataflowModelParser(pipelineConfigurationReader); - transformersFactory = Mockito.spy(new TransformersFactory(RULES_DIRECTORY_PATH, - TEMPLATES_DIRECTORY_PATH)); - when(transformersFactory.getPluginRuleFileLocation(pluginName)).thenReturn(ruleDocDBFilePath); - when(transformersFactory.getPluginTemplateFileLocation(pluginName)).thenReturn(templateDocDBFilePath); + TransformersFactory transformersFactory = mock(TransformersFactory.class); + Path ruleFile = mock(Path.class); + when(ruleFile.getFileName()).thenReturn(Paths.get(ruleDocDBFilePath).getFileName()); InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); InputStream templateStream = new FileInputStream(templateDocDBFilePath); - when(transformersFactory.getPluginRuleFileStream(pluginName)).thenReturn(ruleStream); + when(transformersFactory.readRuleFile(ruleFile)).thenReturn(ruleStream); + List ruleFiles = Collections.singletonList(ruleFile); + when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles); when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); ruleEvaluator = new RuleEvaluator(transformersFactory); @@ -289,16 +303,16 @@ void test_successful_transformation_with_routes_and_subpipelines() throws IOExce final PipelinesDataflowModelParser pipelinesDataflowModelParser = new PipelinesDataflowModelParser(pipelineConfigurationReader); - transformersFactory = Mockito.spy(new TransformersFactory(RULES_DIRECTORY_PATH, - TEMPLATES_DIRECTORY_PATH)); - when(transformersFactory.getPluginRuleFileLocation(pluginName)).thenReturn(ruleDocDBFilePath); - when(transformersFactory.getPluginTemplateFileLocation(pluginName)).thenReturn(templateDocDBFilePath); + TransformersFactory transformersFactory = mock(TransformersFactory.class); + Path ruleFile = mock(Path.class); + when(ruleFile.getFileName()).thenReturn(Paths.get(ruleDocDBFilePath).getFileName()); InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); InputStream ruleStream2 = new FileInputStream(ruleDocDBFilePath); InputStream ruleStream3 = new FileInputStream(ruleDocDBFilePath); InputStream templateStream = new FileInputStream(templateDocDBFilePath); - - when(transformersFactory.getPluginRuleFileStream(pluginName)).thenReturn(ruleStream).thenReturn(ruleStream2).thenReturn(ruleStream3); + when(transformersFactory.readRuleFile(ruleFile)).thenReturn(ruleStream).thenReturn(ruleStream2).thenReturn(ruleStream3); + List ruleFiles = Collections.singletonList(ruleFile); + when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles).thenReturn(ruleFiles); when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); ruleEvaluator = new RuleEvaluator(transformersFactory); @@ -324,13 +338,14 @@ void testInvalidJsonPathThrowsException() throws IOException { final PipelinesDataflowModelParser pipelinesDataflowModelParser = new PipelinesDataflowModelParser(pipelineConfigurationReader); - transformersFactory = Mockito.spy(new TransformersFactory(RULES_DIRECTORY_PATH, - TEMPLATES_DIRECTORY_PATH)); - when(transformersFactory.getPluginRuleFileLocation(pluginName)).thenReturn(ruleDocDBFilePath); - when(transformersFactory.getPluginTemplateFileLocation(pluginName)).thenReturn(templateDocDBFilePath); + TransformersFactory transformersFactory = mock(TransformersFactory.class); + Path ruleFile = mock(Path.class); + when(ruleFile.getFileName()).thenReturn(Paths.get(ruleDocDBFilePath).getFileName()); InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); InputStream templateStream = new FileInputStream(templateDocDBFilePath); - when(transformersFactory.getPluginRuleFileStream(pluginName)).thenReturn(ruleStream); + when(transformersFactory.readRuleFile(ruleFile)).thenReturn(ruleStream); + List ruleFiles = Collections.singletonList(ruleFile); + when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles); when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); ruleEvaluator = new RuleEvaluator(transformersFactory); diff --git a/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/transformer/TransformersFactoryTest.java b/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/transformer/TransformersFactoryTest.java index 4b3b8df0e4..6233295ad3 100644 --- a/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/transformer/TransformersFactoryTest.java +++ b/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/transformer/TransformersFactoryTest.java @@ -7,24 +7,31 @@ import com.fasterxml.jackson.databind.ObjectMapper; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; import java.io.File; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; public class TransformersFactoryTest { - private TransformersFactory transformersFactory; - private final String templatesDirectoryPath = "src/test/resources/templates"; - private final String rulesDirectoryPath = "src/test/resources/rules"; + private final String templatesDirectoryPath = "src/test/resources/transformation/templates"; + private final String rulesDirectoryPath = "src/test/resources/transformation/rules"; private final String validPluginName = "testPlugin"; private final String invalidPluginName = ""; + private TransformersFactory transformersFactory; @BeforeEach public void setUp() { - transformersFactory = new TransformersFactory(rulesDirectoryPath, templatesDirectoryPath); + transformersFactory = spy(new TransformersFactory(rulesDirectoryPath, templatesDirectoryPath)); } @Test @@ -41,20 +48,6 @@ public void testGetPluginTemplateFileLocation_invalidPluginName() { assertEquals("Transformation plugin not found", exception.getMessage()); } - @Test - public void testGetPluginRuleFileLocation_validPluginName() { - String expectedPath = rulesDirectoryPath + "/" + validPluginName + "-rule.yaml"; - assertEquals(expectedPath, transformersFactory.getPluginRuleFileLocation(validPluginName)); - } - - @Test - public void testGetPluginRuleFileLocation_invalidPluginName() { - Exception exception = assertThrows(RuntimeException.class, () -> { - transformersFactory.getPluginRuleFileLocation(invalidPluginName); - }); - assertEquals("Transformation plugin not found", exception.getMessage()); - } - @Test public void testGetTemplateModel_throwsRuntimeExceptionOnIOException() throws IOException { ObjectMapper mockedYamlMapper = Mockito.mock(ObjectMapper.class); @@ -72,5 +65,34 @@ public void testGetTemplateModel_invalidPluginNameThrowsRuntimeException() { assertThrows(RuntimeException.class, () -> transformersFactory.getTemplateModel(invalidPluginName), "Should throw a RuntimeException for empty plugin name."); } + + @Test + public void testGetRuleFiles() throws IOException { + List mockRuleFiles = Arrays.asList( + Paths.get("src/test/resources/transformation/rules/documentdb-rule1.yaml"), + Paths.get("src/test/resources/transformation/rules/documentdb-rule.yaml") + ); + doReturn(mockRuleFiles).when(transformersFactory).getRuleFiles(); + assertTrue(mockRuleFiles.size() > 0, "There should be at least one rule file."); + } + + + @Test + public void testReadFile() throws IOException { + // Mocking the getRuleFiles method + List mockRuleFiles = Arrays.asList( + Paths.get("src/test/resources/transformation/rules/documentdb-rule1.yaml"), + Paths.get("src/test/resources/transformation/rules/documentdb-rule.yaml") + ); + doReturn(mockRuleFiles).when(transformersFactory).getRuleFiles(); + + List ruleFiles = transformersFactory.getRuleFiles(); + assertEquals(ruleFiles.size(), 2); + Path firstRuleFile = ruleFiles.get(0); + Path secondRuleFile = ruleFiles.get(1); + + assertEquals(firstRuleFile.getFileName().toString(), "documentdb-rule1.yaml"); + } + } diff --git a/data-prepper-pipeline-parser/src/test/resources/transformation/rules/documentdb-rule.yaml b/data-prepper-pipeline-parser/src/test/resources/transformation/rules/documentdb-rule.yaml index 28da1406f8..1127f51dbd 100644 --- a/data-prepper-pipeline-parser/src/test/resources/transformation/rules/documentdb-rule.yaml +++ b/data-prepper-pipeline-parser/src/test/resources/transformation/rules/documentdb-rule.yaml @@ -1,3 +1,4 @@ +plugin_name: "documentdb" apply_when: - "$..source.documentdb" - - "$..source.documentdb.collections[0].s3_bucket" \ No newline at end of file + - "$..source.documentdb.collections[0].s3_bucket" diff --git a/data-prepper-pipeline-parser/src/test/resources/transformation/rules/documentdb1-rule.yaml b/data-prepper-pipeline-parser/src/test/resources/transformation/rules/documentdb1-rule.yaml index cb10684065..ed3c4b8b57 100644 --- a/data-prepper-pipeline-parser/src/test/resources/transformation/rules/documentdb1-rule.yaml +++ b/data-prepper-pipeline-parser/src/test/resources/transformation/rules/documentdb1-rule.yaml @@ -1,2 +1,3 @@ +plugin_name: "documentdb" apply_when: - "$..source.documentdb" \ No newline at end of file