From 4a7d5d4cd9f2a4cfbb135a96020977aa3e97fcae Mon Sep 17 00:00:00 2001 From: Srikanth Govindarajan Date: Tue, 6 Aug 2024 22:41:18 -0700 Subject: [PATCH] Address comments Signed-off-by: Srikanth Govindarajan --- .../pipeline/parser/rule/RuleEvaluator.java | 74 +++++----- .../parser/rule/RuleFileEvaluation.java | 6 +- .../pipeline/parser/rule/RuleInputStream.java | 33 +++++ .../transformer/TransformersFactory.java | 47 +++++++ .../parser/rule/RuleEvaluatorTest.java | 25 ++-- .../DynamicConfigTransformerTest.java | 129 +++++++++--------- .../transformer/TransformersFactoryTest.java | 12 -- 7 files changed, 192 insertions(+), 134 deletions(-) create mode 100644 data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleInputStream.java diff --git a/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluator.java b/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluator.java index 9278416379..948e94cd0c 100644 --- a/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluator.java +++ b/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluator.java @@ -24,7 +24,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.nio.file.Path; +import java.util.Collection; import java.util.List; import java.util.Map; @@ -47,9 +47,9 @@ public RuleEvaluatorResult isTransformationNeeded(PipelinesDataFlowModel pipelin String pipelineJson = OBJECT_MAPPER.writeValueAsString(entry); RuleFileEvaluation ruleFileEvaluation = evaluate(pipelineJson); - if (ruleFileEvaluation.result) { - String pluginName = ruleFileEvaluation.pluginName; - LOG.info("Applying rule {}",ruleFileEvaluation.ruleFileName.toString()); + if (ruleFileEvaluation.getResult()) { + String pluginName = ruleFileEvaluation.getPluginName(); + LOG.info("Applying rule {}",ruleFileEvaluation.getRuleFileName().toString()); LOG.info("Rule for {} is evaluated true for pipelineJson {}", pluginName, pipelineJson); InputStream templateStream = transformersFactory.getPluginTemplateFileStream(pluginName); @@ -82,7 +82,6 @@ public RuleEvaluatorResult isTransformationNeeded(PipelinesDataFlowModel pipelin } private RuleFileEvaluation evaluate(String pipelinesJson) { - Configuration parseConfig = Configuration.builder() .jsonProvider(new JacksonJsonNodeJsonProvider()) .mappingProvider(new JacksonMappingProvider()) @@ -90,41 +89,40 @@ private RuleFileEvaluation evaluate(String pipelinesJson) { .build(); RuleTransformerModel rulesModel = null; - InputStream ruleStream = null; - try { - List ruleFiles = transformersFactory.getRuleFiles(); - for (Path ruleFile : ruleFiles) { + try { + Collection ruleStreams = transformersFactory.loadRules(); - ruleStream = transformersFactory.readRuleFile(ruleFile); - if(ruleStream == null){ - continue; - } - rulesModel = yamlMapper.readValue(ruleStream, RuleTransformerModel.class); - List rules = rulesModel.getApplyWhen(); - String pluginName = rulesModel.getPluginName(); - boolean allRulesValid = true; - - for (String rule : rules) { - try { - JsonNode result = JsonPath.using(parseConfig).parse(pipelinesJson).read(rule); - if (result == null || result.size()==0) { + for (RuleInputStream ruleStream : ruleStreams) { + try { + rulesModel = yamlMapper.readValue(ruleStream.getInputStream(), RuleTransformerModel.class); + List rules = rulesModel.getApplyWhen(); + String pluginName = rulesModel.getPluginName(); + boolean allRulesValid = true; + + for (String rule : rules) { + try { + JsonNode result = JsonPath.using(parseConfig).parse(pipelinesJson).read(rule); + if (result == null || result.size() == 0) { + allRulesValid = false; + break; + } + } catch (PathNotFoundException e) { + LOG.debug("Json Path not found for {}", ruleStream.getName()); allRulesValid = false; break; } - } catch (PathNotFoundException e) { - LOG.debug("Json Path not found for {}", ruleFile.getFileName().toString()); - allRulesValid = false; - break; } - } - if (allRulesValid) { - return RuleFileEvaluation.builder() - .withPluginName(pluginName) - .withRuleFileName(ruleFile.getFileName().toString()) - .withResult(true) - .build(); + if (allRulesValid) { + return RuleFileEvaluation.builder() + .withPluginName(pluginName) + .withRuleFileName(ruleStream.getName()) + .withResult(true) + .build(); + } + } finally { + ruleStream.close(); } } @@ -137,19 +135,13 @@ private RuleFileEvaluation evaluate(String pipelinesJson) { .build(); } catch (IOException e) { throw new RuntimeException(e); - } finally { - if (ruleStream != null) { - try { - ruleStream.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } } + return RuleFileEvaluation.builder() .withPluginName(null) .withRuleFileName(null) .withResult(false) .build(); } + } \ No newline at end of file diff --git a/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleFileEvaluation.java b/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleFileEvaluation.java index e819afe878..88ee0f8d6c 100644 --- a/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleFileEvaluation.java +++ b/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleFileEvaluation.java @@ -8,9 +8,9 @@ @AllArgsConstructor @Data public class RuleFileEvaluation { - Boolean result; - String ruleFileName; - String pluginName; + private Boolean result; + private String ruleFileName; + private String pluginName; public RuleFileEvaluation() { diff --git a/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleInputStream.java b/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleInputStream.java new file mode 100644 index 0000000000..bf87adfbfe --- /dev/null +++ b/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleInputStream.java @@ -0,0 +1,33 @@ +package org.opensearch.dataprepper.pipeline.parser.rule; + +import java.io.IOException; +import java.io.InputStream; + +public class RuleInputStream { + private String name; + private InputStream inputStream; + + public RuleInputStream(String name, InputStream inputStream) { + this.name = name; + this.inputStream = inputStream; + } + + public String getName() { + return name; + } + + public InputStream getInputStream() { + return inputStream; + } + + public void close() { + if (inputStream != null) { + try { + inputStream.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } +} + diff --git a/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/transformer/TransformersFactory.java b/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/transformer/TransformersFactory.java index 8e12f96317..a1a46685da 100644 --- a/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/transformer/TransformersFactory.java +++ b/data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/transformer/TransformersFactory.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import static org.opensearch.dataprepper.pipeline.parser.PipelineTransformationConfiguration.RULES_DIRECTORY_PATH; import static org.opensearch.dataprepper.pipeline.parser.PipelineTransformationConfiguration.TEMPLATES_DIRECTORY_PATH; +import org.opensearch.dataprepper.pipeline.parser.rule.RuleInputStream; import javax.inject.Named; import java.io.File; @@ -22,6 +23,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -68,6 +70,51 @@ public InputStream getPluginTemplateFileStream(String pluginName) { return filestream; } + public Collection loadRules() { + URI uri; + try { + uri = getClass().getClassLoader().getResource("rules").toURI(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + + List ruleInputStreams = new ArrayList<>(); + + if ("jar".equals(uri.getScheme())) { + try (FileSystem fileSystem = FileSystems.newFileSystem(uri, Collections.emptyMap())) { + Path rulesFolderPath = fileSystem.getPath("rules"); + try (Stream paths = Files.walk(rulesFolderPath)) { + paths.filter(Files::isRegularFile) + .forEach(path -> { + InputStream ruleStream = getClass().getClassLoader().getResourceAsStream("rules" + "/" + path.getFileName().toString()); + if (ruleStream != null) { + ruleInputStreams.add(new RuleInputStream(path.getFileName().toString(), ruleStream)); + } + }); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } else { + Path rulesFolderPath = Paths.get(uri); + try (Stream paths = Files.walk(rulesFolderPath)) { + paths.filter(Files::isRegularFile) + .forEach(path -> { + try { + InputStream ruleStream = Files.newInputStream(path); + ruleInputStreams.add(new RuleInputStream(path.getFileName().toString(), ruleStream)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return ruleInputStreams; + } + + public List getRuleFiles() { // Get the URI of the rules folder URI uri = null; diff --git a/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluatorTest.java b/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluatorTest.java index df00ddab07..e46607ebc5 100644 --- a/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluatorTest.java +++ b/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluatorTest.java @@ -9,7 +9,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.Test; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import org.opensearch.dataprepper.model.configuration.PipelineExtensions; @@ -23,7 +22,6 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collections; @@ -41,10 +39,10 @@ void test_isTransformationNeeded_ForDocDBSource_ShouldReturn_True() throws IOExc String pluginName = "documentdb"; String pipelineName = "test-pipeline"; Map sourceOptions = new HashMap<>(); - Map s3_bucket = new HashMap<>(); - s3_bucket.put("s3_bucket", "bucket-name"); + Map s3Bucket = new HashMap<>(); + s3Bucket.put("s3_bucket", "bucket-name"); List> collections = new ArrayList<>(); - collections.add(s3_bucket); + collections.add(s3Bucket); sourceOptions.put("collections", collections); final PluginModel source = new PluginModel(pluginName, sourceOptions); final List processors = Collections.singletonList(new PluginModel("testProcessor", null)); @@ -56,13 +54,12 @@ void test_isTransformationNeeded_ForDocDBSource_ShouldReturn_True() throws IOExc TransformersFactory transformersFactory = mock(TransformersFactory.class); - Path ruleFile = mock(Path.class); - List ruleFiles = Collections.singletonList(ruleFile); InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); InputStream templateStream = new FileInputStream(ruleDocDBTemplatePath); - when(ruleFile.getFileName()).thenReturn(Paths.get("documentdb-rule.yaml").getFileName()); - when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles); - when(transformersFactory.readRuleFile(eq(ruleFile))).thenReturn(ruleStream); + RuleInputStream ruleInputStream = new RuleInputStream(Paths.get(ruleDocDBFilePath).getFileName().toString(), ruleStream); + + List ruleStreams = Collections.singletonList(ruleInputStream); + when(transformersFactory.loadRules()).thenReturn(ruleStreams); when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); RuleEvaluator ruleEvaluator = new RuleEvaluator(transformersFactory); @@ -79,10 +76,10 @@ void test_isTransformationNeeded_ForOtherSource_ShouldReturn_False() throws IOEx String pluginName = "http"; String pipelineName = "test-pipeline"; Map sourceOptions = new HashMap<>(); - Map s3_bucket = new HashMap<>(); - s3_bucket.put("s3_bucket", "bucket-name"); + Map s3Bucket = new HashMap<>(); + s3Bucket.put("s3_bucket", "bucket-name"); List> collections = new ArrayList<>(); - collections.add(s3_bucket); + collections.add(s3Bucket); sourceOptions.put("collections", collections); final PluginModel source = new PluginModel(pluginName, sourceOptions); final List processors = Collections.singletonList(new PluginModel("testProcessor", null)); @@ -118,7 +115,7 @@ void testThrowsExceptionOnFileError() { final PipelinesDataFlowModel pipelinesDataFlowModel = new PipelinesDataFlowModel( (PipelineExtensions) null, Collections.singletonMap(pipelineName, pipelineModel)); - when(transformersFactory.getRuleFiles()).thenThrow(new RuntimeException("File not found")); + when(transformersFactory.loadRules()).thenThrow(new RuntimeException("File not found")); RuleEvaluator ruleEvaluator = new RuleEvaluator(transformersFactory); diff --git a/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/transformer/DynamicConfigTransformerTest.java b/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/transformer/DynamicConfigTransformerTest.java index 86ac0a6b68..ce85863e2b 100644 --- a/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/transformer/DynamicConfigTransformerTest.java +++ b/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/transformer/DynamicConfigTransformerTest.java @@ -19,6 +19,7 @@ import org.opensearch.dataprepper.pipeline.parser.PipelinesDataflowModelParser; import org.opensearch.dataprepper.pipeline.parser.TestConfigurationProvider; import org.opensearch.dataprepper.pipeline.parser.rule.RuleEvaluator; +import org.opensearch.dataprepper.pipeline.parser.rule.RuleInputStream; import java.io.File; import java.io.FileInputStream; @@ -42,30 +43,25 @@ class DynamicConfigTransformerTest { @Test void test_successful_transformation_with_only_source_and_sink() throws IOException { - String docDBUserConfig = TestConfigurationProvider.USER_CONFIG_TRANSFORMATION_DOCDB1_CONFIG_FILE; String templateDocDBFilePath = TestConfigurationProvider.TEMPLATE_TRANSFORMATION_DOCDB1_CONFIG_FILE; String ruleDocDBFilePath = TestConfigurationProvider.RULES_TRANSFORMATION_DOCDB1_CONFIG_FILE; String expectedDocDBFilePath = TestConfigurationProvider.EXPECTED_TRANSFORMATION_DOCDB1_CONFIG_FILE; - String pluginName = "documentdb"; + String pluginName = "documentdb"; PipelineConfigurationReader pipelineConfigurationReader = new PipelineConfigurationFileReader(docDBUserConfig); final PipelinesDataflowModelParser pipelinesDataflowModelParser = new PipelinesDataflowModelParser(pipelineConfigurationReader); TransformersFactory transformersFactory = mock(TransformersFactory.class); - - Path ruleFile = mock(Path.class); - when(ruleFile.getFileName()).thenReturn(Paths.get(ruleDocDBFilePath).getFileName()); InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); InputStream templateStream = new FileInputStream(templateDocDBFilePath); - when(transformersFactory.readRuleFile(ruleFile)).thenReturn(ruleStream); - List ruleFiles = Collections.singletonList(ruleFile); - when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles); + RuleInputStream ruleInputStream = new RuleInputStream(Paths.get(ruleDocDBFilePath).getFileName().toString(), ruleStream); + List ruleStreams = Collections.singletonList(ruleInputStream); + when(transformersFactory.loadRules()).thenReturn(ruleStreams); when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); RuleEvaluator ruleEvaluator = new RuleEvaluator(transformersFactory); - // Load the original and template YAML files from the test resources directory PipelinesDataFlowModel pipelinesDataFlowModel = pipelinesDataflowModelParser.parseConfiguration(); PipelineConfigurationTransformer transformer = new DynamicConfigTransformer(ruleEvaluator); @@ -94,9 +90,9 @@ void test_successful_transformation_with_documentdb() throws IOException { when(ruleFile.getFileName()).thenReturn(Paths.get(ruleDocDBFilePath).getFileName()); InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); InputStream templateStream = new FileInputStream(templateDocDBFilePath); - when(transformersFactory.readRuleFile(ruleFile)).thenReturn(ruleStream); - List ruleFiles = Collections.singletonList(ruleFile); - when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles); + RuleInputStream ruleInputStream = new RuleInputStream(Paths.get(ruleDocDBFilePath).getFileName().toString(), ruleStream); + List ruleStreams = Collections.singletonList(ruleInputStream); + when(transformersFactory.loadRules()).thenReturn(ruleStreams); when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); ruleEvaluator = new RuleEvaluator(transformersFactory); @@ -112,7 +108,6 @@ void test_successful_transformation_with_documentdb() throws IOException { } @RepeatedTest(5) - @Test void test_successful_transformation_with_subpipelines() throws IOException { String docDBUserConfig = TestConfigurationProvider.USER_CONFIG_TRANSFORMATION_DOCUMENTDB_SUBPIPELINES_CONFIG_FILE; @@ -125,17 +120,19 @@ void test_successful_transformation_with_subpipelines() throws IOException { new PipelinesDataflowModelParser(pipelineConfigurationReader); TransformersFactory transformersFactory = mock(TransformersFactory.class); - Path ruleFile = mock(Path.class); - when(ruleFile.getFileName()).thenReturn(Paths.get(ruleDocDBFilePath).getFileName()); - InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); + InputStream ruleStream1 = new FileInputStream(ruleDocDBFilePath); InputStream ruleStream2 = new FileInputStream(ruleDocDBFilePath); InputStream templateStream = new FileInputStream(templateDocDBFilePath); - when(transformersFactory.readRuleFile(ruleFile)).thenReturn(ruleStream).thenReturn(ruleStream1).thenReturn(ruleStream2); - List ruleFiles = Collections.singletonList(ruleFile); - when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles); + RuleInputStream ruleInputStream1 = new RuleInputStream(Paths.get(ruleDocDBFilePath).getFileName().toString(), ruleStream1); + RuleInputStream ruleInputStream2 = new RuleInputStream(Paths.get(ruleDocDBFilePath).getFileName().toString(), ruleStream2); + + List ruleStreams1 = Collections.singletonList(ruleInputStream1); + List ruleStreams2 = Collections.singletonList(ruleInputStream2); + when(transformersFactory.loadRules()).thenReturn(ruleStreams1).thenReturn(ruleStreams2); when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); - ruleEvaluator = new RuleEvaluator(transformersFactory); + + RuleEvaluator ruleEvaluator = new RuleEvaluator(transformersFactory); // Load the original and template YAML files from the test resources directory PipelinesDataFlowModel pipelinesDataFlowModel = pipelinesDataflowModelParser.parseConfiguration(); @@ -150,7 +147,6 @@ void test_successful_transformation_with_subpipelines() throws IOException { @Test void test_successful_transformation_with_functionPlaceholder() throws IOException { - String docDBUserConfig = TestConfigurationProvider.USER_CONFIG_TRANSFORMATION_DOCUMENTDB_FUNCTION_CONFIG_FILE; String templateDocDBFilePath = TestConfigurationProvider.TEMPLATE_TRANSFORMATION_DOCUMENTDB_FUNCTION_CONFIG_FILE; String ruleDocDBFilePath = TestConfigurationProvider.RULES_TRANSFORMATION_DOCDB1_CONFIG_FILE; @@ -161,15 +157,16 @@ void test_successful_transformation_with_functionPlaceholder() throws IOExceptio new PipelinesDataflowModelParser(pipelineConfigurationReader); TransformersFactory transformersFactory = mock(TransformersFactory.class); - Path ruleFile = mock(Path.class); - when(ruleFile.getFileName()).thenReturn(Paths.get(ruleDocDBFilePath).getFileName()); + InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); InputStream templateStream = new FileInputStream(templateDocDBFilePath); - when(transformersFactory.readRuleFile(ruleFile)).thenReturn(ruleStream); - List ruleFiles = Collections.singletonList(ruleFile); - when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles); + RuleInputStream ruleInputStream = new RuleInputStream(Paths.get(ruleDocDBFilePath).getFileName().toString(), ruleStream); + + List ruleStreams = Collections.singletonList(ruleInputStream); + when(transformersFactory.loadRules()).thenReturn(ruleStreams); when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); - ruleEvaluator = new RuleEvaluator(transformersFactory); + + RuleEvaluator ruleEvaluator = new RuleEvaluator(transformersFactory); // Load the original and template YAML files from the test resources directory PipelinesDataFlowModel pipelinesDataFlowModel = pipelinesDataflowModelParser.parseConfiguration(); @@ -182,10 +179,8 @@ void test_successful_transformation_with_functionPlaceholder() throws IOExceptio assertThat(expectedYamlasMap).usingRecursiveComparison().isEqualTo(transformedYamlasMap); } - @Test void test_successful_transformation_with_complete_template() throws IOException { - String docDBUserConfig = TestConfigurationProvider.USER_CONFIG_TRANSFORMATION_DOCDB2_CONFIG_FILE; String templateDocDBFilePath = TestConfigurationProvider.TEMPLATE_TRANSFORMATION_DOCDB2_CONFIG_FILE; String ruleDocDBFilePath = TestConfigurationProvider.RULES_TRANSFORMATION_DOCDB1_CONFIG_FILE; @@ -196,15 +191,16 @@ void test_successful_transformation_with_complete_template() throws IOException new PipelinesDataflowModelParser(pipelineConfigurationReader); TransformersFactory transformersFactory = mock(TransformersFactory.class); - Path ruleFile = mock(Path.class); - when(ruleFile.getFileName()).thenReturn(Paths.get(ruleDocDBFilePath).getFileName()); + InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); InputStream templateStream = new FileInputStream(templateDocDBFilePath); - when(transformersFactory.readRuleFile(ruleFile)).thenReturn(ruleStream); - List ruleFiles = Collections.singletonList(ruleFile); - when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles); + RuleInputStream ruleInputStream = new RuleInputStream(Paths.get(ruleDocDBFilePath).getFileName().toString(), ruleStream); + + List ruleStreams = Collections.singletonList(ruleInputStream); + when(transformersFactory.loadRules()).thenReturn(ruleStreams); when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); - ruleEvaluator = new RuleEvaluator(transformersFactory); + + RuleEvaluator ruleEvaluator = new RuleEvaluator(transformersFactory); // Load the original and template YAML files from the test resources directory PipelinesDataFlowModel pipelinesDataFlowModel = pipelinesDataflowModelParser.parseConfiguration(); @@ -218,9 +214,9 @@ void test_successful_transformation_with_complete_template() throws IOException assertThat(expectedYamlasMap).usingRecursiveComparison().isEqualTo(transformedYamlasMap); } + @Test void test_successful_transformation_with_routes_keyword() throws IOException { - String docDBUserConfig = TestConfigurationProvider.USER_CONFIG_TRANSFORMATION_DOCUMENTDB_ROUTES_CONFIG_FILE; String templateDocDBFilePath = TestConfigurationProvider.TEMPLATE_TRANSFORMATION_DOCUMENTDB_FINAL_CONFIG_FILE; String ruleDocDBFilePath = TestConfigurationProvider.RULES_TRANSFORMATION_DOCDB1_CONFIG_FILE; @@ -231,15 +227,16 @@ void test_successful_transformation_with_routes_keyword() throws IOException { new PipelinesDataflowModelParser(pipelineConfigurationReader); TransformersFactory transformersFactory = mock(TransformersFactory.class); - Path ruleFile = mock(Path.class); - when(ruleFile.getFileName()).thenReturn(Paths.get(ruleDocDBFilePath).getFileName()); + InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); InputStream templateStream = new FileInputStream(templateDocDBFilePath); - when(transformersFactory.readRuleFile(ruleFile)).thenReturn(ruleStream); - List ruleFiles = Collections.singletonList(ruleFile); - when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles); + RuleInputStream ruleInputStream = new RuleInputStream(Paths.get(ruleDocDBFilePath).getFileName().toString(), ruleStream); + + List ruleStreams = Collections.singletonList(ruleInputStream); + when(transformersFactory.loadRules()).thenReturn(ruleStreams); when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); - ruleEvaluator = new RuleEvaluator(transformersFactory); + + RuleEvaluator ruleEvaluator = new RuleEvaluator(transformersFactory); // Load the original and template YAML files from the test resources directory PipelinesDataFlowModel pipelinesDataFlowModel = pipelinesDataflowModelParser.parseConfiguration(); @@ -253,10 +250,8 @@ void test_successful_transformation_with_routes_keyword() throws IOException { assertThat(expectedYamlasMap).usingRecursiveComparison().isEqualTo(transformedYamlasMap); } - @Test void test_successful_transformation_with_route_keyword() throws IOException { - String docDBUserConfig = TestConfigurationProvider.USER_CONFIG_TRANSFORMATION_DOCUMENTDB_ROUTE_CONFIG_FILE; String templateDocDBFilePath = TestConfigurationProvider.TEMPLATE_TRANSFORMATION_DOCUMENTDB_FINAL_CONFIG_FILE; String ruleDocDBFilePath = TestConfigurationProvider.RULES_TRANSFORMATION_DOCDB1_CONFIG_FILE; @@ -268,15 +263,16 @@ void test_successful_transformation_with_route_keyword() throws IOException { new PipelinesDataflowModelParser(pipelineConfigurationReader); TransformersFactory transformersFactory = mock(TransformersFactory.class); - Path ruleFile = mock(Path.class); - when(ruleFile.getFileName()).thenReturn(Paths.get(ruleDocDBFilePath).getFileName()); + InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); InputStream templateStream = new FileInputStream(templateDocDBFilePath); - when(transformersFactory.readRuleFile(ruleFile)).thenReturn(ruleStream); - List ruleFiles = Collections.singletonList(ruleFile); - when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles); + RuleInputStream ruleInputStream = new RuleInputStream(Paths.get(ruleDocDBFilePath).getFileName().toString(), ruleStream); + + List ruleStreams = Collections.singletonList(ruleInputStream); + when(transformersFactory.loadRules()).thenReturn(ruleStreams); when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); - ruleEvaluator = new RuleEvaluator(transformersFactory); + + RuleEvaluator ruleEvaluator = new RuleEvaluator(transformersFactory); // Load the original and template YAML files from the test resources directory PipelinesDataFlowModel pipelinesDataFlowModel = pipelinesDataflowModelParser.parseConfiguration(); @@ -290,10 +286,8 @@ void test_successful_transformation_with_route_keyword() throws IOException { assertThat(expectedYamlasMap).usingRecursiveComparison().isEqualTo(transformedYamlasMap); } - @Test void test_successful_transformation_with_routes_and_subpipelines() throws IOException { - String docDBUserConfig = TestConfigurationProvider.USER_CONFIG_TRANSFORMATION_DOCUMENTDB_SUBPIPELINES_ROUTES_CONFIG_FILE; String templateDocDBFilePath = TestConfigurationProvider.TEMPLATE_TRANSFORMATION_DOCUMENTDB_FINAL_CONFIG_FILE; String ruleDocDBFilePath = TestConfigurationProvider.RULES_TRANSFORMATION_DOCDB1_CONFIG_FILE; @@ -304,17 +298,23 @@ void test_successful_transformation_with_routes_and_subpipelines() throws IOExce new PipelinesDataflowModelParser(pipelineConfigurationReader); TransformersFactory transformersFactory = mock(TransformersFactory.class); - Path ruleFile = mock(Path.class); - when(ruleFile.getFileName()).thenReturn(Paths.get(ruleDocDBFilePath).getFileName()); - InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); + + InputStream ruleStream1 = new FileInputStream(ruleDocDBFilePath); InputStream ruleStream2 = new FileInputStream(ruleDocDBFilePath); InputStream ruleStream3 = new FileInputStream(ruleDocDBFilePath); InputStream templateStream = new FileInputStream(templateDocDBFilePath); - when(transformersFactory.readRuleFile(ruleFile)).thenReturn(ruleStream).thenReturn(ruleStream2).thenReturn(ruleStream3); - List ruleFiles = Collections.singletonList(ruleFile); - when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles).thenReturn(ruleFiles); + RuleInputStream ruleInputStream1 = new RuleInputStream(Paths.get(ruleDocDBFilePath).getFileName().toString(), ruleStream1); + RuleInputStream ruleInputStream2 = new RuleInputStream(Paths.get(ruleDocDBFilePath).getFileName().toString(), ruleStream2); + RuleInputStream ruleInputStream3 = new RuleInputStream(Paths.get(ruleDocDBFilePath).getFileName().toString(), ruleStream3); + + List ruleStreams1 = Collections.singletonList(ruleInputStream1); + List ruleStreams2 = Collections.singletonList(ruleInputStream2); + List ruleStreams3 = Collections.singletonList(ruleInputStream3); + + when(transformersFactory.loadRules()).thenReturn(ruleStreams1).thenReturn(ruleStreams2).thenReturn(ruleStreams3); when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); - ruleEvaluator = new RuleEvaluator(transformersFactory); + + RuleEvaluator ruleEvaluator = new RuleEvaluator(transformersFactory); // Load the original and template YAML files from the test resources directory PipelinesDataFlowModel pipelinesDataFlowModel = pipelinesDataflowModelParser.parseConfiguration(); @@ -339,14 +339,15 @@ void testInvalidJsonPathThrowsException() throws IOException { new PipelinesDataflowModelParser(pipelineConfigurationReader); TransformersFactory transformersFactory = mock(TransformersFactory.class); - Path ruleFile = mock(Path.class); - when(ruleFile.getFileName()).thenReturn(Paths.get(ruleDocDBFilePath).getFileName()); + InputStream ruleStream = new FileInputStream(ruleDocDBFilePath); InputStream templateStream = new FileInputStream(templateDocDBFilePath); - when(transformersFactory.readRuleFile(ruleFile)).thenReturn(ruleStream); - List ruleFiles = Collections.singletonList(ruleFile); - when(transformersFactory.getRuleFiles()).thenReturn(ruleFiles); + RuleInputStream ruleInputStream = new RuleInputStream(Paths.get(ruleDocDBFilePath).getFileName().toString(), ruleStream); + + List ruleStreams = Collections.singletonList(ruleInputStream); + when(transformersFactory.loadRules()).thenReturn(ruleStreams); when(transformersFactory.getPluginTemplateFileStream(pluginName)).thenReturn(templateStream); + ruleEvaluator = new RuleEvaluator(transformersFactory); // Load the original and template YAML files from the test resources directory diff --git a/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/transformer/TransformersFactoryTest.java b/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/transformer/TransformersFactoryTest.java index 6233295ad3..b67a64a9d4 100644 --- a/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/transformer/TransformersFactoryTest.java +++ b/data-prepper-pipeline-parser/src/test/java/org/opensearch/dataprepper/pipeline/parser/transformer/TransformersFactoryTest.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -66,17 +65,6 @@ public void testGetTemplateModel_invalidPluginNameThrowsRuntimeException() { "Should throw a RuntimeException for empty plugin name."); } - @Test - public void testGetRuleFiles() throws IOException { - List mockRuleFiles = Arrays.asList( - Paths.get("src/test/resources/transformation/rules/documentdb-rule1.yaml"), - Paths.get("src/test/resources/transformation/rules/documentdb-rule.yaml") - ); - doReturn(mockRuleFiles).when(transformersFactory).getRuleFiles(); - assertTrue(mockRuleFiles.size() > 0, "There should be at least one rule file."); - } - - @Test public void testReadFile() throws IOException { // Mocking the getRuleFiles method