forked from opensearch-project/data-prepper
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Pipeline Configuration Transformation (opensearch-project#4446)
* Adding templates Signed-off-by: srigovs <[email protected]> * Added Dynamic yaml transformer Signed-off-by: srigovs <[email protected]> * Added Rule evaluator Signed-off-by: srigovs <[email protected]> * Added rule evaluator Signed-off-by: srigovs <[email protected]> * Added json walk Signed-off-by: srigovs <[email protected]> * Add transformation logic Signed-off-by: srigovs <[email protected]> * Add dynamic rule Signed-off-by: srigovs <[email protected]> * Almost working Signed-off-by: srigovs <[email protected]> * Adding multiple pipelines part1 Signed-off-by: srigovs <[email protected]> * Adding multiple pipelines part2-incomplete Signed-off-by: srigovs <[email protected]> * Works e2e for 1 pipeline Signed-off-by: srigovs <[email protected]> * Added multi pipeline template and pipelinemodel support, works for docDB with big template Signed-off-by: srigovs <[email protected]> * added tests for models and fixed beans, one more fix needed for bean Signed-off-by: srigovs <[email protected]> * Fixed IT and beans Signed-off-by: srigovs <[email protected]> * Update bean to have only pipelineDataModel and not parser Signed-off-by: srigovs <[email protected]> * Add banner Signed-off-by: srigovs <[email protected]> * Code cleanup and add comments Signed-off-by: srigovs <[email protected]> * Support user pipeline configuration dynamic transformation based on templates and rules Signed-off-by: srigovs <[email protected]> * Address comments Signed-off-by: srigovs <[email protected]> * Added Function Call support in templates Signed-off-by: srigovs <[email protected]> * Added Function Call support in templates Signed-off-by: srigovs <[email protected]> * Modify documentDB template. Signed-off-by: srigovs <[email protected]> * Code clean up Signed-off-by: srigovs <[email protected]> * Code clean up Signed-off-by: srigovs <[email protected]> --------- Signed-off-by: srigovs <[email protected]>
- Loading branch information
1 parent
8bd7fc9
commit e590fde
Showing
47 changed files
with
2,224 additions
and
23 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
43 changes: 43 additions & 0 deletions
43
.../java/org/opensearch/dataprepper/pipeline/parser/PipelineTransformationConfiguration.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
| package org.opensearch.dataprepper.pipeline.parser; | ||
|
|
||
| import org.opensearch.dataprepper.pipeline.parser.rule.RuleEvaluator; | ||
| import org.opensearch.dataprepper.pipeline.parser.transformer.TransformersFactory; | ||
| import org.springframework.context.annotation.Bean; | ||
| import org.springframework.context.annotation.Configuration; | ||
|
|
||
| import javax.inject.Named; | ||
|
|
||
| @Configuration | ||
| public class PipelineTransformationConfiguration { | ||
| public static final String TEMPLATES_DIRECTORY_PATH = "TEMPLATES_DIRECTORY_PATH"; | ||
| public static final String RULES_DIRECTORY_PATH = "VALIDATORS_DIRECTORY_PATH"; | ||
|
|
||
| @Bean | ||
| @Named(RULES_DIRECTORY_PATH) | ||
| static String provideRulesDirectoryPath() { | ||
| return "resources/rules"; | ||
| } | ||
|
|
||
| @Bean | ||
| @Named(TEMPLATES_DIRECTORY_PATH) | ||
| static String provideTemplateDirectoryPath() { | ||
| return "resources/templates"; | ||
| } | ||
|
|
||
| @Bean | ||
| TransformersFactory transformersFactory( | ||
| @Named(TEMPLATES_DIRECTORY_PATH) String provideTransformerDirectoryPath, | ||
| @Named(RULES_DIRECTORY_PATH) String provideTemplateDirectoryPath | ||
| ) { | ||
| return new TransformersFactory(RULES_DIRECTORY_PATH, TEMPLATES_DIRECTORY_PATH); | ||
| } | ||
|
|
||
| @Bean | ||
| public RuleEvaluator ruleEvaluator(TransformersFactory transformersFactory) { | ||
| return new RuleEvaluator(transformersFactory); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
118 changes: 118 additions & 0 deletions
118
...e-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
| package org.opensearch.dataprepper.pipeline.parser.rule; | ||
|
|
||
| import com.fasterxml.jackson.core.JsonProcessingException; | ||
| import com.fasterxml.jackson.databind.ObjectMapper; | ||
| import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; | ||
| import com.jayway.jsonpath.Configuration; | ||
| import com.jayway.jsonpath.JsonPath; | ||
| import com.jayway.jsonpath.Option; | ||
| import com.jayway.jsonpath.ParseContext; | ||
| import com.jayway.jsonpath.PathNotFoundException; | ||
| import com.jayway.jsonpath.ReadContext; | ||
| import com.jayway.jsonpath.spi.json.JacksonJsonProvider; | ||
| import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; | ||
| import org.opensearch.dataprepper.model.configuration.PipelineModel; | ||
| import org.opensearch.dataprepper.model.configuration.PipelinesDataFlowModel; | ||
| import org.opensearch.dataprepper.pipeline.parser.transformer.PipelineTemplateModel; | ||
| import org.opensearch.dataprepper.pipeline.parser.transformer.TransformersFactory; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| import java.io.File; | ||
| import java.io.IOException; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
|
|
||
| public class RuleEvaluator { | ||
|
|
||
| private static final Logger LOG = LoggerFactory.getLogger(RuleEvaluator.class); | ||
| private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); | ||
| private static final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory()); | ||
| private final TransformersFactory transformersFactory; | ||
| private String PLUGIN_NAME = null; | ||
|
|
||
| public RuleEvaluator(TransformersFactory transformersFactory) { | ||
| this.transformersFactory = transformersFactory; | ||
| } | ||
|
|
||
| public RuleEvaluatorResult isTransformationNeeded(PipelinesDataFlowModel pipelineModel) { | ||
| return isDocDBSource(pipelineModel); | ||
| } | ||
|
|
||
| /** | ||
| * Evaluates model based on pre defined rules and | ||
| * result contains the name of the pipeline that will need transformation, | ||
| * evaluated boolean result and the corresponding template model | ||
| * Assumption: only one pipeline can have transformation. | ||
| * @param pipelinesModel | ||
| * @return RuleEvaluatorResult | ||
| */ | ||
| private RuleEvaluatorResult isDocDBSource(PipelinesDataFlowModel pipelinesModel) { | ||
| PLUGIN_NAME = "documentdb"; | ||
| String pluginRulesPath = transformersFactory.getPluginRuleFileLocation(PLUGIN_NAME); | ||
| Map<String, PipelineModel> pipelines = pipelinesModel.getPipelines(); | ||
|
|
||
| for (Map.Entry<String, PipelineModel> entry : pipelines.entrySet()) { | ||
| try { | ||
| String pipelineJson = OBJECT_MAPPER.writeValueAsString(entry); | ||
| if (evaluate(pipelineJson, pluginRulesPath)) { | ||
| LOG.debug("Rule path {} is evaluated true for pipelineJson {}",pluginRulesPath, pipelineJson); | ||
|
|
||
| String templateFilePath = transformersFactory.getPluginTemplateFileLocation(PLUGIN_NAME); | ||
| PipelineTemplateModel templateModel = yamlMapper.readValue(new File(templateFilePath), | ||
| PipelineTemplateModel.class); | ||
| LOG.debug("Chosen template file {}",templateFilePath); | ||
|
|
||
| return RuleEvaluatorResult.builder() | ||
| .withEvaluatedResult(true) | ||
| .withPipelineTemplateModel(templateModel) | ||
| .withPipelineName(entry.getKey()) | ||
| .build(); | ||
| } | ||
| } catch (JsonProcessingException e) { | ||
| LOG.error("Error processing json"); | ||
| throw new RuntimeException(e); | ||
| } catch (IOException e) { | ||
| LOG.error("Error reading file"); | ||
| throw new RuntimeException(e); | ||
| } | ||
| } | ||
| return RuleEvaluatorResult.builder() | ||
| .withEvaluatedResult(false) | ||
| .withPipelineName(null) | ||
| .withPipelineTemplateModel(null) | ||
| .build(); | ||
| } | ||
|
|
||
| private Boolean evaluate(String pipelinesJson, | ||
| String rulePath) { | ||
|
|
||
| Configuration parseConfig = Configuration.builder() | ||
| .jsonProvider(new JacksonJsonProvider()) | ||
| .mappingProvider(new JacksonMappingProvider()) | ||
| .options(Option.AS_PATH_LIST) | ||
| .build(); | ||
| ParseContext parseContext = JsonPath.using(parseConfig); | ||
| ReadContext readPathContext = parseContext.parse(pipelinesJson); | ||
|
|
||
| try { | ||
| RuleTransformerModel rulesModel = yamlMapper.readValue(new File(rulePath), RuleTransformerModel.class); | ||
| List<String> rules = rulesModel.getApplyWhen(); | ||
| for (String rule : rules) { | ||
| Object result = readPathContext.read(rule); | ||
| } | ||
| } catch (IOException e) { | ||
| LOG.warn("Error reading file {}", rulePath); | ||
| return false; | ||
| } catch (PathNotFoundException e) { | ||
| LOG.warn("Path not found {}", rulePath); | ||
| return false; | ||
| } | ||
| return true; | ||
| } | ||
| } | ||
|
|
26 changes: 26 additions & 0 deletions
26
...er/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluatorResult.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
| package org.opensearch.dataprepper.pipeline.parser.rule; | ||
|
|
||
| import lombok.AllArgsConstructor; | ||
| import lombok.Builder; | ||
| import lombok.Getter; | ||
| import org.opensearch.dataprepper.pipeline.parser.transformer.PipelineTemplateModel; | ||
|
|
||
| @Builder(setterPrefix = "with") | ||
| @Getter | ||
| @AllArgsConstructor | ||
| public class RuleEvaluatorResult { | ||
|
|
||
| private boolean evaluatedResult; | ||
|
|
||
| private String pipelineName; | ||
|
|
||
| private PipelineTemplateModel pipelineTemplateModel; | ||
|
|
||
| public RuleEvaluatorResult() { | ||
|
|
||
| } | ||
| } |
Oops, something went wrong.