diff --git a/CHANGELOG.md b/CHANGELOG.md index 97746af5..f6acadd7 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ Types of changes ## [1.12.0] +- `Added` possibility to pass multiple masking config to the command line - `Added` markov mask to generate pseudo text based on a sample text - `Added` command to export maskings as a mermaid flow chart - `Added` possibility to use option `preserve: notInCache` with mask `fromCache` diff --git a/README.md b/README.md index 3adbb724..6dfc3a20 100755 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ This takes the `data.json` file, masks the data contained inside it and put the * `--skip-line-on-error` This flag will totally skip a line if an error occurs masking a field. * `--skip-field-on-error` This flag will return output without a field if an error occurs masking this field. * `--empty-input` This flag will give PIMO a `{}` input, usable with `--repeat` flag. -* `--config=filename.yml` This flag allow to use another file for config than the default `masking.yml`. +* `--config=filename.yml` This flag allow to use another file for config than the default `masking.yml`. It's possible to pass multiple yaml configuration, that will be applied in the order provided by command line arguments: `--config=masking1.yml --config=masking2.yml` * `--load-cache cacheName=filename.json` This flag load an initial cache content from a file (json line format `{"key":"a", "value":"b"}`). * `--dump-cache cacheName=filename.json` This flag dump final cache content to a file (json line format `{"key":"a", "value":"b"}`). * `--verbosity ` or `-v` This flag increase verbosity on the stderr output, possible values: none (0), error (1), warn (2), info (3), debug (4), trace (5). diff --git a/cmd/pimo/main.go b/cmd/pimo/main.go index f0a91422..53c3a491 100755 --- a/cmd/pimo/main.go +++ b/cmd/pimo/main.go @@ -78,7 +78,7 @@ var ( colormode string iteration int emptyInput bool - maskingFile string + maskingFiles []string cachesToDump map[string]string cachesToLoad map[string]string skipLineOnError bool @@ -109,7 +109,7 @@ There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDa rootCmd.PersistentFlags().StringVar(&colormode, "color", "auto", "use colors in log outputs : yes, no or auto") rootCmd.PersistentFlags().IntVarP(&iteration, "repeat", "r", 1, "number of iteration to mask each input") rootCmd.PersistentFlags().BoolVar(&emptyInput, "empty-input", false, "generate data without any input, to use with repeat flag") - rootCmd.PersistentFlags().StringVarP(&maskingFile, "config", "c", "masking.yml", "name and location of the masking-config file") + rootCmd.PersistentFlags().StringArrayVarP(&maskingFiles, "config", "c", []string{"masking.yml"}, "name and location of the masking-config file") rootCmd.PersistentFlags().StringToStringVar(&cachesToDump, "dump-cache", map[string]string{}, "path for dumping cache into file") rootCmd.PersistentFlags().StringToStringVar(&cachesToLoad, "load-cache", map[string]string{}, "path for loading cache from file") rootCmd.PersistentFlags().BoolVar(&skipLineOnError, "skip-line-on-error", false, "skip a line if an error occurs while masking a field") @@ -132,7 +132,7 @@ There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDa rootCmd.AddCommand(&cobra.Command{ Use: "flow", Run: func(cmd *cobra.Command, args []string) { - pdef, err := model.LoadPipelineDefinitionFromYAML(maskingFile) + pdef, err := model.LoadPipelineDefinitionFromYAML(maskingFiles...) if err != nil { log.Err(err).Msg("Cannot load pipeline definition from file") log.Warn().Int("return", 1).Msg("End PIMO") @@ -208,7 +208,7 @@ func run() { if len(maskingOneLiner) > 0 { pdef, err = model.LoadPipelineDefintionFromOneLiner(maskingOneLiner) } else { - pdef, err = model.LoadPipelineDefinitionFromYAML(maskingFile) + pdef, err = model.LoadPipelineDefinitionFromYAML(maskingFiles...) } if err != nil { @@ -364,7 +364,7 @@ func initLog() { default: zerolog.SetGlobalLevel(zerolog.Disabled) } - over.MDC().Set("config", maskingFile) + over.MDC().Set("config", maskingFiles) over.SetGlobalFields([]string{"config"}) } diff --git a/pkg/model/pipeline.go b/pkg/model/pipeline.go index c02ae1fa..0e6194c8 100644 --- a/pkg/model/pipeline.go +++ b/pkg/model/pipeline.go @@ -159,15 +159,32 @@ func BuildPipeline(pipeline Pipeline, conf Definition, caches map[string]Cache) return pipeline, caches, nil } -func LoadPipelineDefinitionFromYAML(filename string) (Definition, error) { - source, err := ioutil.ReadFile(filename) - if err != nil { - return Definition{}, err - } +func LoadPipelineDefinitionFromYAML(filenames ...string) (Definition, error) { var conf Definition - err = yaml.Unmarshal(source, &conf) - if err != nil { - return conf, err + for _, filename := range filenames { + source, err := ioutil.ReadFile(filename) + if err != nil { + return Definition{}, err + } + var conf_temp Definition + err = yaml.Unmarshal(source, &conf_temp) + if err != nil { + return conf, err + } + if conf.Masking == nil && conf_temp.Masking != nil { + conf.Masking = []Masking{} + } + if conf.Caches == nil && conf_temp.Caches != nil { + conf.Caches = map[string]CacheDefinition{} + } + conf.Masking = append(conf.Masking, conf_temp.Masking...) + for key, val := range conf_temp.Caches { + conf.Caches[key] = val + } + conf.Seed += conf_temp.Seed + if v := conf_temp.Version; v != "1" { + return Definition{}, fmt.Errorf("Version of %s is V%s. Incompatible with current PIMO version", filename, v) + } } if conf.Seed == 0 { conf.Seed = time.Now().UnixNano() diff --git a/test/suites/mask_fromjson.yml b/test/suites/mask_fromjson.yml index 9b60d976..117c5ee2 100644 --- a/test/suites/mask_fromjson.yml +++ b/test/suites/mask_fromjson.yml @@ -90,7 +90,7 @@ testcases: - script: rm -rf masking.yml - script: |- cat > masking.yml < masking.yml < masking1.yml < masking2.yml <