From 344c7236e1477e00291fb206c50ff63d2f9b4f79 Mon Sep 17 00:00:00 2001 From: Tuan Tran Date: Sun, 24 Jul 2022 23:32:14 +0700 Subject: [PATCH 1/2] feat: add converter, reader and cmd --- .gitignore | 23 +++++++ Makefile | 8 +++ csv_reader.go | 3 + csv_writer.go | 62 +++++++++++++++++ go.mod | 10 +++ go.sum | 10 +++ json_converter.go | 74 +++++++++++++++++++++ json_converter_test.go | 18 +++++ json_flattener.go | 46 +++++++++++++ json_reader.go | 49 ++++++++++++++ json_reader_test.go | 53 +++++++++++++++ json_writer.go | 3 + tests/read-json-array.json | 5 ++ tests/read-json-object.json | 3 + tool/cmd/csv.go | 129 ++++++++++++++++++++++++++++++++++++ tool/cmd/flatten.go | 19 ++++++ tool/main.go | 48 ++++++++++++++ tool/params/params.go | 8 +++ types.go | 9 +++ utils/map.go | 7 ++ utils/os.go | 14 ++++ 21 files changed, 601 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 csv_reader.go create mode 100644 csv_writer.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 json_converter.go create mode 100644 json_converter_test.go create mode 100644 json_flattener.go create mode 100644 json_reader.go create mode 100644 json_reader_test.go create mode 100644 json_writer.go create mode 100644 tests/read-json-array.json create mode 100644 tests/read-json-object.json create mode 100644 tool/cmd/csv.go create mode 100644 tool/cmd/flatten.go create mode 100644 tool/main.go create mode 100644 tool/params/params.go create mode 100644 types.go create mode 100644 utils/map.go create mode 100644 utils/os.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..54aadc7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +.idea +.vscode +.DS_Store + +# Project's unused files +bin/ +output-* \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..11dd7d8 --- /dev/null +++ b/Makefile @@ -0,0 +1,8 @@ +.PHONY: all test +all: build + +build: + go build -o bin/jsonconv github.com/tuan78/jsonconv/tool + +test: + go test ./... \ No newline at end of file diff --git a/csv_reader.go b/csv_reader.go new file mode 100644 index 0000000..e9cec06 --- /dev/null +++ b/csv_reader.go @@ -0,0 +1,3 @@ +package jsonconv + +// WIP diff --git a/csv_writer.go b/csv_writer.go new file mode 100644 index 0000000..b67eeed --- /dev/null +++ b/csv_writer.go @@ -0,0 +1,62 @@ +package jsonconv + +import ( + "bytes" + "encoding/csv" + "io" + "os" + "path/filepath" +) + +type CsvWriter struct { + Delimiter *rune // Field delimiter. If nil, it uses default value from csv.NewWriter + UseCRLF bool // True to use \r\n as the line terminator + writer io.Writer + closer io.Closer +} + +func NewCsvWriterFromFile(path string) (*CsvWriter, error) { + err := os.MkdirAll(filepath.Dir(path), os.ModePerm) + if err != nil { + return nil, err + } + fi, err := os.Create(path) + if err != nil { + return nil, err + } + wr := NewCsvWriter(fi) + wr.closer = fi + return NewCsvWriter(fi), nil +} + +func NewCsvWriterFromByteBuffer() (*CsvWriter, *bytes.Buffer) { + buf := bytes.NewBuffer([]byte{}) + wr := NewCsvWriter(buf) + return wr, buf +} + +func NewCsvWriter(w io.Writer) *CsvWriter { + return &CsvWriter{ + writer: w, + } +} + +func (w *CsvWriter) Write(data CsvData) error { + if w.closer != nil { + defer w.closer.Close() + } + + writer := csv.NewWriter(w.writer) + if w.Delimiter != nil { + writer.Comma = *w.Delimiter + } + writer.UseCRLF = w.UseCRLF + + defer writer.Flush() + for _, v := range data { + if err := writer.Write(v); err != nil { + return err + } + } + return nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..32b54d8 --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module github.com/tuan78/jsonconv + +go 1.18 + +require ( + github.com/spf13/cobra v1.5.0 + github.com/spf13/pflag v1.0.5 +) + +require github.com/inconshreveable/mousetrap v1.0.0 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..0d85248 --- /dev/null +++ b/go.sum @@ -0,0 +1,10 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= +github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/json_converter.go b/json_converter.go new file mode 100644 index 0000000..2982c55 --- /dev/null +++ b/json_converter.go @@ -0,0 +1,74 @@ +package jsonconv + +import ( + "fmt" + "sort" +) + +type ConvertInput struct { + JsonArray JsonArray + FlattenLevel int // -1: unlimited, 0: no nested, [1...n]: n level + BaseHeaders CsvRow +} + +func Convert(input *ConvertInput) (CsvData, error) { + csvData := make(CsvData, 0) + if len(input.JsonArray) == 0 { + return nil, fmt.Errorf("empty JSON array") + } + + // Flatten JSON object, so can display nested JSON values in CSV columns. + for _, jsonObject := range input.JsonArray { + FlattenJsonObject(jsonObject, input.FlattenLevel) + } + + // Create CSV headers. + headers := CreateCsvHeader(input.JsonArray, input.BaseHeaders) + if len(headers) == 0 { + return nil, fmt.Errorf("empty CSV headers") + } + + // Create CSV rows. + csvData = append(csvData, headers) + for _, jsonObj := range input.JsonArray { + row := make(CsvRow, 0) + for _, header := range headers { + if val, exist := jsonObj[header]; exist { + row = append(row, fmt.Sprintf("%v", val)) + continue + } + row = append(row, "") + } + csvData = append(csvData, row) + } + + return csvData, nil +} + +func CreateCsvHeader(jsonArray JsonArray, baseHeaders CsvRow) CsvRow { + headers := make(sort.StringSlice, 0) + headerSet := make(map[string]struct{}) + + // Get CSV header from json. + for _, jsonObj := range jsonArray { + for key := range jsonObj { + headerSet[key] = struct{}{} + } + } + + // Exclude base headers from detected headers, then sort filtered list. + for _, header := range baseHeaders { + delete(headerSet, header) + } + for header := range headerSet { + headers = append(headers, header) + } + headers.Sort() + + // Insert BaseHeaders to the beginning of headers. + if len(baseHeaders) > 0 { + headers = append(baseHeaders, headers...) + } + + return headers +} diff --git a/json_converter_test.go b/json_converter_test.go new file mode 100644 index 0000000..d2c46fb --- /dev/null +++ b/json_converter_test.go @@ -0,0 +1,18 @@ +package jsonconv + +import ( + "testing" +) + +func TestCreateCsvHeader(t *testing.T) { + jsonArray := []map[string]interface{}{ + { + "a": true, + "b": false, + }, + } + headers := CreateCsvHeader(jsonArray, nil) + if len(headers) == 0 { + t.Fatalf("headers must not be empty") + } +} diff --git a/json_flattener.go b/json_flattener.go new file mode 100644 index 0000000..69207fa --- /dev/null +++ b/json_flattener.go @@ -0,0 +1,46 @@ +package jsonconv + +import ( + "reflect" + + "github.com/tuan78/jsonconv/utils" +) + +func FlattenJsonObject(jsonObj JsonObject, flattenLevel int) { + nestedLevel := 0 + needLoop := true + for needLoop { + needLoop = false // To exit the loop. + + // Find and extract nested json object. + extractedKeys := make([]string, 0) + flattenedKV := make(JsonObject) + for key, val := range jsonObj { + switch nested := val.(type) { + case JsonObject: + for nkey, nval := range nested { + // Append json object's key with nested json object's key. + csvKey := key + "__" + nkey + flattenedKV[csvKey] = nval + + // Check if nested value is kind of map. + if reflect.ValueOf(nval).Kind() == reflect.Map { + if flattenLevel == -1 || (flattenLevel > nestedLevel) { + needLoop = true // Need one more loop. + } + } + } + // Store extracted nested json object's key for later uses. + extractedKeys = append(extractedKeys, key) + } + } + + // Update json object with its flattened children. + utils.CopyMap(flattenedKV, jsonObj) + for _, key := range extractedKeys { + delete(jsonObj, key) + } + + nestedLevel++ + } +} diff --git a/json_reader.go b/json_reader.go new file mode 100644 index 0000000..a5f1b50 --- /dev/null +++ b/json_reader.go @@ -0,0 +1,49 @@ +package jsonconv + +import ( + "encoding/json" + "io" + "os" + "strings" +) + +type JsonReader struct { + reader io.Reader + closer io.Closer +} + +func NewJsonReaderFromFile(path string) (*JsonReader, error) { + fi, err := os.Open(path) + if err != nil { + return nil, err + } + re := NewJsonReader(fi) + re.closer = fi + return re, nil +} + +func NewJsonReaderFromString(rawData string) *JsonReader { + re := strings.NewReader(rawData) + return NewJsonReader(re) +} + +func NewJsonReader(r io.Reader) *JsonReader { + return &JsonReader{ + reader: r, + } +} + +func (r *JsonReader) Read(v any) error { + if r.closer != nil { + defer r.closer.Close() + } + + decoder := json.NewDecoder(r.reader) + for decoder.More() { + err := decoder.Decode(v) + if err != nil { + return err + } + } + return nil +} diff --git a/json_reader_test.go b/json_reader_test.go new file mode 100644 index 0000000..ef490f5 --- /dev/null +++ b/json_reader_test.go @@ -0,0 +1,53 @@ +package jsonconv + +import ( + "path/filepath" + "testing" +) + +func TestJsonReaderFromString(t *testing.T) { + rawData := ` + { + "a": "test", + "b": "test" + }` + jsonObject := make(JsonObject) + reader := NewJsonReaderFromString(rawData) + err := reader.Read(&jsonObject) + if err != nil { + t.Fatalf("failed to read file %v", err) + } + if _, exist := jsonObject["a"]; !exist { + t.Fatalf("failed to read json object") + } +} + +func TestJsonReaderFromFile_JsonObject(t *testing.T) { + jsonObject := make(JsonObject) + reader, err := NewJsonReaderFromFile(filepath.Join("tests", "read-json-object.json")) + if err != nil { + t.Fatalf("failed to read file %v", err) + } + err = reader.Read(&jsonObject) + if err != nil { + t.Fatalf("failed to read file %v", err) + } + if _, exist := jsonObject["a"]; !exist { + t.Fatalf("failed to read json object") + } +} + +func TestJsonReaderFromFile_JsonArray(t *testing.T) { + jsonArray := make(JsonArray, 0) + reader, err := NewJsonReaderFromFile(filepath.Join("tests", "read-json-array.json")) + if err != nil { + t.Fatalf("failed to read file %v", err) + } + err = reader.Read(&jsonArray) + if err != nil { + t.Fatalf("failed to read file %v", err) + } + if len(jsonArray) == 0 { + t.Fatalf("failed to read json array") + } +} diff --git a/json_writer.go b/json_writer.go new file mode 100644 index 0000000..e9cec06 --- /dev/null +++ b/json_writer.go @@ -0,0 +1,3 @@ +package jsonconv + +// WIP diff --git a/tests/read-json-array.json b/tests/read-json-array.json new file mode 100644 index 0000000..7a22b1e --- /dev/null +++ b/tests/read-json-array.json @@ -0,0 +1,5 @@ +[ + { + "a": "test" + } +] diff --git a/tests/read-json-object.json b/tests/read-json-object.json new file mode 100644 index 0000000..cc9b078 --- /dev/null +++ b/tests/read-json-object.json @@ -0,0 +1,3 @@ +{ + "a": "test" +} diff --git a/tool/cmd/csv.go b/tool/cmd/csv.go new file mode 100644 index 0000000..83f164b --- /dev/null +++ b/tool/cmd/csv.go @@ -0,0 +1,129 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/spf13/cobra" + "github.com/tuan78/jsonconv" + "github.com/tuan78/jsonconv/tool/params" + "github.com/tuan78/jsonconv/utils" +) + +func NewCsvCmd() *cobra.Command { + var ( + baseHeaders []string + delimiter string + useCRLF bool + ) + + cmd := &cobra.Command{ + Use: "csv", + Short: "Convert JSON to CSV", + Long: "Convert JSON to CSV", + RunE: func(cmd *cobra.Command, args []string) error { + return processCsvCmd(baseHeaders, delimiter, useCRLF) + }, + } + + cmd.PersistentFlags().SortFlags = false + cmd.PersistentFlags().StringSliceVar(&baseHeaders, "headers", nil, "headers in CSV that always appears before dynamic headers (auto detected from JSON)") + cmd.PersistentFlags().StringVar(&delimiter, "delimiter", ",", "field delimiter") + cmd.PersistentFlags().BoolVar(&useCRLF, "useCRLF", false, "set it true to use \\r\\n as the line terminator") + return cmd +} + +func processCsvCmd(baseHeaders []string, delimiter string, useCRLF bool) error { + var err error + + // Create JSON reader. + var jsonReader *jsonconv.JsonReader + switch { + case params.RawData != "": + jsonReader = jsonconv.NewJsonReaderFromString(params.RawData) + case params.InputPath != "": + jsonReader, err = jsonconv.NewJsonReaderFromFile(params.InputPath) + if err != nil { + return err + } + case !utils.IsStdinEmpty(): + jsonReader = jsonconv.NewJsonReader(os.Stdin) + default: + return fmt.Errorf("need to input either raw data, input file path or data from stdin") + } + + // Read JSON data and store in jsonArray. + fmt.Println("Processing...") + var encoded interface{} + err = jsonReader.Read(&encoded) + if err != nil { + return err + } + var jsonArray jsonconv.JsonArray + switch val := encoded.(type) { + case []interface{}: + for _, v := range val { + if jsonObject, ok := v.(jsonconv.JsonObject); ok { + jsonArray = append(jsonArray, jsonObject) + continue + } + return fmt.Errorf("unknown type of JSON data") + } + case jsonconv.JsonObject: + jsonArray = append(jsonArray, val) + default: + return fmt.Errorf("unknown type of JSON data") + } + + // Convert JSON to CSV. + var csvData [][]string + csvData, err = jsonconv.Convert(&jsonconv.ConvertInput{ + JsonArray: jsonArray, + FlattenLevel: params.FlattenLevel, + BaseHeaders: baseHeaders, + }) + if err != nil { + return err + } + if len(csvData) == 0 { + return fmt.Errorf("empty CSV data") + } + + return writeToCsvFile(csvData, delimiter, useCRLF) +} + +func writeToCsvFile(csvData jsonconv.CsvData, delimiter string, useCRLF bool) error { + // Check and override outputPath if necessary. + if params.OutputPath == "" { + workingDir, err := os.Getwd() + if err != nil { + return err + } + fileName := fmt.Sprintf("output-%v.csv", time.Now().UTC().Unix()) + params.OutputPath = filepath.Join(workingDir, fileName) + } + + // Create CSV writer. + runes := []rune(delimiter) + var delimiterRune *rune + if len(runes) > 0 { + delimiterRune = &runes[0] + } + csvWriter, err := jsonconv.NewCsvWriterFromFile(params.OutputPath) + if err != nil { + return err + } + csvWriter.Delimiter = delimiterRune + csvWriter.UseCRLF = useCRLF + + // Write to CSV file. + err = csvWriter.Write(csvData) + if err != nil { + return err + } + + fmt.Printf("Done. The CSV file is located at %s\n", params.OutputPath) + return nil +} diff --git a/tool/cmd/flatten.go b/tool/cmd/flatten.go new file mode 100644 index 0000000..594cb14 --- /dev/null +++ b/tool/cmd/flatten.go @@ -0,0 +1,19 @@ +package cmd + +import "github.com/spf13/cobra" + +func NewFlattenCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "flatten", + Short: "Flatten JSON object and JSON array", + Long: "Flatten JSON object and JSON array", + RunE: func(cmd *cobra.Command, args []string) error { + return processFlattenCmd() + }, + } + return cmd +} + +func processFlattenCmd() error { + return nil +} diff --git a/tool/main.go b/tool/main.go new file mode 100644 index 0000000..f316f57 --- /dev/null +++ b/tool/main.go @@ -0,0 +1,48 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/tuan78/jsonconv/tool/cmd" + "github.com/tuan78/jsonconv/tool/params" +) + +var ( + version = "v1.0.0" +) + +var ( + rootCmd = &cobra.Command{ + Use: "jsonconv", + Short: "Tool for flattening and converting JSON.", + Long: "Tool for flattening and converting JSON (JSON to CSV, JSON from CSV, JSON from Excel, and more).", + Version: version, + } +) + +func main() { + // Add flags. + rootCmd.PersistentFlags().StringVarP(¶ms.RawData, "data", "d", "", "raw JSON array data") + rootCmd.PersistentFlags().StringVarP(¶ms.InputPath, "in", "i", "", "input file path") + rootCmd.PersistentFlags().StringVarP(¶ms.OutputPath, "out", "o", "", "output file path") + rootCmd.PersistentFlags().IntVarP(¶ms.FlattenLevel, "level", "l", -1, "level for flattening a nested JSON (-1: unlimited, 0: no nested, [1...n]: n level of nested JSON)") + + // Add commands. + rootCmd.AddCommand(cmd.NewFlattenCmd()) + rootCmd.AddCommand(cmd.NewCsvCmd()) + + // Parse flags. + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + flag.Parse() + + // Execute command. + if err := rootCmd.Execute(); err != nil { + fmt.Fprintln(os.Stderr, fmt.Sprintf("Command execution failed, error: %v", err)) + os.Exit(1) + } +} diff --git a/tool/params/params.go b/tool/params/params.go new file mode 100644 index 0000000..d311799 --- /dev/null +++ b/tool/params/params.go @@ -0,0 +1,8 @@ +package params + +var ( + InputPath string + OutputPath string + RawData string + FlattenLevel int +) diff --git a/types.go b/types.go new file mode 100644 index 0000000..b881999 --- /dev/null +++ b/types.go @@ -0,0 +1,9 @@ +package jsonconv + +// CSV types. +type CsvRow = []string +type CsvData = []CsvRow + +// JSON types. +type JsonObject = map[string]interface{} +type JsonArray = []JsonObject diff --git a/utils/map.go b/utils/map.go new file mode 100644 index 0000000..c2027c5 --- /dev/null +++ b/utils/map.go @@ -0,0 +1,7 @@ +package utils + +func CopyMap(from map[string]interface{}, to map[string]interface{}) { + for k, v := range from { + to[k] = v + } +} diff --git a/utils/os.go b/utils/os.go new file mode 100644 index 0000000..77b99af --- /dev/null +++ b/utils/os.go @@ -0,0 +1,14 @@ +package utils + +import ( + "os" +) + +func IsStdinEmpty() bool { + fi := os.Stdin + info, err := fi.Stat() + if err != nil { + return true + } + return info.Size() == 0 +} From b684bcc5a4da630fa5ec484000c5a35fe057e8c0 Mon Sep 17 00:00:00 2001 From: Tuan Tran Date: Wed, 27 Jul 2022 22:29:11 +0700 Subject: [PATCH 2/2] feat: update converter, flattener and cmd and add unit tests --- .gitignore | 3 +- Makefile | 2 +- README.md | 2 +- csv_reader.go | 3 - csv_writer.go | 36 +--- csv_writer_test.go | 75 +++++++ json_converter.go | 79 +++---- json_converter_test.go | 244 ++++++++++++++++++++- json_flattener.go | 93 +++++--- json_flattener_test.go | 408 ++++++++++++++++++++++++++++++++++++ json_reader.go | 26 +-- json_reader_test.go | 111 +++++++--- json_writer.go | 27 ++- json_writer_test.go | 90 ++++++++ tests/read-json-array.json | 5 - tests/read-json-object.json | 3 - tool/cmd/csv.go | 184 +++++++++++----- tool/cmd/flatten.go | 160 +++++++++++++- tool/main.go | 3 +- tool/params/params.go | 7 +- types.go | 8 +- utils/map.go | 7 - 22 files changed, 1331 insertions(+), 245 deletions(-) delete mode 100644 csv_reader.go create mode 100644 csv_writer_test.go create mode 100644 json_flattener_test.go create mode 100644 json_writer_test.go delete mode 100644 tests/read-json-array.json delete mode 100644 tests/read-json-object.json delete mode 100644 utils/map.go diff --git a/.gitignore b/.gitignore index 54aadc7..b23a35c 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,5 @@ # Project's unused files bin/ -output-* \ No newline at end of file +*.csv +*.json \ No newline at end of file diff --git a/Makefile b/Makefile index 11dd7d8..b4766d3 100644 --- a/Makefile +++ b/Makefile @@ -5,4 +5,4 @@ build: go build -o bin/jsonconv github.com/tuan78/jsonconv/tool test: - go test ./... \ No newline at end of file + go test ./... -cover \ No newline at end of file diff --git a/README.md b/README.md index 0176117..f556c0e 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ # jsonconv -Golang library and cmd for JSON converter (flatten JSON, JSON to CSV, JSON from CSV, JSON from Excel, and more). +Golang library and cmd for flattening JSON and converting JSON to CSV. diff --git a/csv_reader.go b/csv_reader.go deleted file mode 100644 index e9cec06..0000000 --- a/csv_reader.go +++ /dev/null @@ -1,3 +0,0 @@ -package jsonconv - -// WIP diff --git a/csv_writer.go b/csv_writer.go index b67eeed..043db35 100644 --- a/csv_writer.go +++ b/csv_writer.go @@ -1,51 +1,31 @@ package jsonconv import ( - "bytes" "encoding/csv" "io" - "os" - "path/filepath" ) +// A CsvWriter writes records using CSV encoding. type CsvWriter struct { Delimiter *rune // Field delimiter. If nil, it uses default value from csv.NewWriter UseCRLF bool // True to use \r\n as the line terminator writer io.Writer - closer io.Closer -} - -func NewCsvWriterFromFile(path string) (*CsvWriter, error) { - err := os.MkdirAll(filepath.Dir(path), os.ModePerm) - if err != nil { - return nil, err - } - fi, err := os.Create(path) - if err != nil { - return nil, err - } - wr := NewCsvWriter(fi) - wr.closer = fi - return NewCsvWriter(fi), nil -} - -func NewCsvWriterFromByteBuffer() (*CsvWriter, *bytes.Buffer) { - buf := bytes.NewBuffer([]byte{}) - wr := NewCsvWriter(buf) - return wr, buf } +// NewCsvWriter returns a new CsvWriter that writes to w. func NewCsvWriter(w io.Writer) *CsvWriter { return &CsvWriter{ writer: w, } } -func (w *CsvWriter) Write(data CsvData) error { - if w.closer != nil { - defer w.closer.Close() - } +// NewDelimiter returns a pointer to v. +func NewDelimiter(v rune) *rune { + return &v +} +// Write writes all CSV data to w. +func (w *CsvWriter) Write(data CsvData) error { writer := csv.NewWriter(w.writer) if w.Delimiter != nil { writer.Comma = *w.Delimiter diff --git a/csv_writer_test.go b/csv_writer_test.go new file mode 100644 index 0000000..1d02e3f --- /dev/null +++ b/csv_writer_test.go @@ -0,0 +1,75 @@ +package jsonconv + +import ( + "bytes" + "testing" +) + +func TestCsvWriter_InvalidDelimiter(t *testing.T) { + // Prepare + data := CsvData{ + { + "id", "user", "score", "is active", + }, + } + buf := &bytes.Buffer{} + wr := NewCsvWriter(buf) + wr.Delimiter = NewDelimiter('\n') + + // Process + err := wr.Write(data) + + // Check + if err == nil { + t.Fatalf("Should throw an error for invalid delimiter") + } +} + +func TestCsvWriter(t *testing.T) { + // Prepare + data := CsvData{ + { + "id", "user", "score", "is active", + }, + { + "ce06f5b1-5721-42c0-91e1-9f72a09c250a", "Tuấn", "1.5", "true", + }, + { + "b042ab5c-ca73-4460-b739-96410ea9d3a6", "Jon Doe", "-100", "false", + }, + { + "4e01b638-44e5-4079-8043-baabbff21cc8", "高橋", "100000000000000000000000", "true", + }, + { + "6f0d6265-545c-4366-a78b-4f80c337aa69", "김슬기", "1234567890", "true", + }, + { + "3fbae214-006d-4ac5-9eea-76c5d611f54a", "Comma,", "0", "false", + }, + } + buf := &bytes.Buffer{} + wr := NewCsvWriter(buf) + wr.Delimiter = NewDelimiter('|') + + // Process + err := wr.Write(data) + if err != nil { + t.Fatalf("failed to write csv, err: %v", err) + } + + // Check + s := buf.String() + expect := `id|user|score|is active +ce06f5b1-5721-42c0-91e1-9f72a09c250a|Tuấn|1.5|true +b042ab5c-ca73-4460-b739-96410ea9d3a6|Jon Doe|-100|false +4e01b638-44e5-4079-8043-baabbff21cc8|高橋|100000000000000000000000|true +6f0d6265-545c-4366-a78b-4f80c337aa69|김슬기|1234567890|true +3fbae214-006d-4ac5-9eea-76c5d611f54a|Comma,|0|false +` + if s == "" { + t.Fatalf("failed to write csv to byte buffer") + } + if s != expect { + t.Fatalf("csv output is not correct") + } +} diff --git a/json_converter.go b/json_converter.go index 2982c55..5723ee5 100644 --- a/json_converter.go +++ b/json_converter.go @@ -5,35 +5,38 @@ import ( "sort" ) -type ConvertInput struct { - JsonArray JsonArray - FlattenLevel int // -1: unlimited, 0: no nested, [1...n]: n level - BaseHeaders CsvRow +// A ToCsvOption converts a JSON Array to CSV data. +type ToCsvOption struct { + FlattenOption *FlattenOption // Set it to apply JSON flattening + BaseHeaders CsvRow // Base CSV headers used to add before dynamic headers } -func Convert(input *ConvertInput) (CsvData, error) { - csvData := make(CsvData, 0) - if len(input.JsonArray) == 0 { - return nil, fmt.Errorf("empty JSON array") +// ToCsv converts a JsonArray to CsvData with given op. +func ToCsv(arr JsonArray, op *ToCsvOption) CsvData { + if len(arr) == 0 { + return CsvData{} } - // Flatten JSON object, so can display nested JSON values in CSV columns. - for _, jsonObject := range input.JsonArray { - FlattenJsonObject(jsonObject, input.FlattenLevel) - } - - // Create CSV headers. - headers := CreateCsvHeader(input.JsonArray, input.BaseHeaders) - if len(headers) == 0 { - return nil, fmt.Errorf("empty CSV headers") + // Flatten JSON. + if op != nil && op.FlattenOption != nil { + for _, obj := range arr { + FlattenJsonObject(obj, op.FlattenOption) + } } // Create CSV rows. - csvData = append(csvData, headers) - for _, jsonObj := range input.JsonArray { + var csvData CsvData + var hs []string + if op != nil && len(op.BaseHeaders) > 0 { + hs = CreateCsvHeader(arr, op.BaseHeaders) + } else { + hs = CreateCsvHeader(arr, nil) + } + csvData = append(csvData, hs) + for _, obj := range arr { row := make(CsvRow, 0) - for _, header := range headers { - if val, exist := jsonObj[header]; exist { + for _, h := range hs { + if val, exist := obj[h]; exist { row = append(row, fmt.Sprintf("%v", val)) continue } @@ -42,33 +45,35 @@ func Convert(input *ConvertInput) (CsvData, error) { csvData = append(csvData, row) } - return csvData, nil + return csvData } -func CreateCsvHeader(jsonArray JsonArray, baseHeaders CsvRow) CsvRow { - headers := make(sort.StringSlice, 0) - headerSet := make(map[string]struct{}) +// CreateCsvHeader creates CsvRow from arr and baseHs. +// A baseHs is base header that we want to put at the beginning of dynamic header, +// we can set baseHs to nil if we just want to have dynamic header only. +func CreateCsvHeader(arr JsonArray, baseHs CsvRow) CsvRow { + hs := make(sort.StringSlice, 0) + hss := make(map[string]struct{}) // Get CSV header from json. - for _, jsonObj := range jsonArray { - for key := range jsonObj { - headerSet[key] = struct{}{} + for _, obj := range arr { + for k := range obj { + hss[k] = struct{}{} } } // Exclude base headers from detected headers, then sort filtered list. - for _, header := range baseHeaders { - delete(headerSet, header) + for _, h := range baseHs { + delete(hss, h) } - for header := range headerSet { - headers = append(headers, header) + for h := range hss { + hs = append(hs, h) } - headers.Sort() + hs.Sort() // Insert BaseHeaders to the beginning of headers. - if len(baseHeaders) > 0 { - headers = append(baseHeaders, headers...) + if len(baseHs) > 0 { + hs = append(baseHs, hs...) } - - return headers + return hs } diff --git a/json_converter_test.go b/json_converter_test.go index d2c46fb..efc6123 100644 --- a/json_converter_test.go +++ b/json_converter_test.go @@ -1,18 +1,250 @@ package jsonconv import ( + "strings" "testing" ) +func TestToCsv_EmptyArray(t *testing.T) { + // Prepare + data := JsonArray{} + + // Process + csvData := ToCsv(data, nil) + + // Check + if len(csvData) != 0 { + t.Fatalf("It should be safe to put empty JSON array as param") + } +} + +func TestToCsv_NonFlatten(t *testing.T) { + // Prepare + data := JsonArray{ + { + "id": "b042ab5c-ca73-4460-b739-96410ea9d3a6", + "user": "Jon Doe", + "score": -100, + "is active": false, + "special1": "&", + "special2": "<", + "special3": ">", + "special4": "\u0026", + "special5": "\u003c", + "special6": "\u003e", + }, + { + "id": "ce06f5b1-5721-42c0-91e1-9f72a09c250a", + "user": "Tuấn", + "score": 1.5, + "is active": true, + "nested": JsonObject{ + "a": 1, + "b": 2, + }, + }, + { + "id": "4e01b638-44e5-4079-8043-baabbff21cc8", + "user": "高橋", + "score": 100000000000000000, + "is active": true, + }, + } + + // Process + csvData := ToCsv(data, nil) + + // Check + r1 := strings.Join(csvData[0], ",") + r2 := strings.Join(csvData[1], ",") + r3 := strings.Join(csvData[2], ",") + r4 := strings.Join(csvData[3], ",") + exp1 := "id,is active,nested,score,special1,special2,special3,special4,special5,special6,user" + exp2 := "b042ab5c-ca73-4460-b739-96410ea9d3a6,false,,-100,&,<,>,&,<,>,Jon Doe" + exp3 := "ce06f5b1-5721-42c0-91e1-9f72a09c250a,true,map[a:1 b:2],1.5,,,,,,,Tuấn" + exp4 := "4e01b638-44e5-4079-8043-baabbff21cc8,true,,100000000000000000,,,,,,,高橋" + if r1 != exp1 { + t.Fatalf("created headers are incorrect, %s is not equal expected %s", r1, exp1) + } + if r2 != exp2 { + t.Fatalf("created headers are incorrect, %s is not equal expected %s", r2, exp2) + } + if r3 != exp3 { + t.Fatalf("created headers are incorrect, %s is not equal expected %s", r3, exp3) + } + if r4 != exp4 { + t.Fatalf("created headers are incorrect, %s is not equal expected %s", r4, exp4) + } +} + +func TestToCsv_WithBaseHeader(t *testing.T) { + // Prepare + data := JsonArray{ + { + "id": "b042ab5c-ca73-4460-b739-96410ea9d3a6", + "user": "Jon Doe", + "score": -100, + "is active": false, + "special1": "&", + "special2": "<", + "special3": ">", + "special4": "\u0026", + "special5": "\u003c", + "special6": "\u003e", + }, + { + "id": "ce06f5b1-5721-42c0-91e1-9f72a09c250a", + "user": "Tuấn", + "score": 1.5, + "is active": true, + "nested": JsonObject{ + "a": 1, + "b": 2, + }, + }, + { + "id": "4e01b638-44e5-4079-8043-baabbff21cc8", + "user": "高橋", + "score": 100000000000000000, + "is active": true, + }, + } + + // Process + csvData := ToCsv(data, &ToCsvOption{ + BaseHeaders: []string{"x", "y", "z", "3", "2", "1"}, + }) + + // Check + r1 := strings.Join(csvData[0], ",") + r2 := strings.Join(csvData[1], ",") + r3 := strings.Join(csvData[2], ",") + r4 := strings.Join(csvData[3], ",") + exp1 := "x,y,z,3,2,1,id,is active,nested,score,special1,special2,special3,special4,special5,special6,user" + exp2 := ",,,,,,b042ab5c-ca73-4460-b739-96410ea9d3a6,false,,-100,&,<,>,&,<,>,Jon Doe" + exp3 := ",,,,,,ce06f5b1-5721-42c0-91e1-9f72a09c250a,true,map[a:1 b:2],1.5,,,,,,,Tuấn" + exp4 := ",,,,,,4e01b638-44e5-4079-8043-baabbff21cc8,true,,100000000000000000,,,,,,,高橋" + if r1 != exp1 { + t.Fatalf("created headers are incorrect, %s is not equal expected %s", r1, exp1) + } + if r2 != exp2 { + t.Fatalf("created headers are incorrect, %s is not equal expected %s", r2, exp2) + } + if r3 != exp3 { + t.Fatalf("created headers are incorrect, %s is not equal expected %s", r3, exp3) + } + if r4 != exp4 { + t.Fatalf("created headers are incorrect, %s is not equal expected %s", r4, exp4) + } +} + +func TestToCsv_WithFlattening(t *testing.T) { + // Prepare + data := JsonArray{ + { + "id": "b042ab5c-ca73-4460-b739-96410ea9d3a6", + "user": "Jon Doe", + "score": -100, + "is active": false, + "special1": "&", + "special2": "<", + "special3": ">", + "special4": "\u0026", + "special5": "\u003c", + "special6": "\u003e", + }, + { + "id": "ce06f5b1-5721-42c0-91e1-9f72a09c250a", + "user": "Tuấn", + "score": 1.5, + "is active": true, + "nested": JsonObject{ + "a": 1, + "b": 2, + }, + }, + { + "id": "4e01b638-44e5-4079-8043-baabbff21cc8", + "user": "高橋", + "score": 100000000000000000, + "is active": true, + }, + } + + // Process + csvData := ToCsv(data, &ToCsvOption{ + FlattenOption: &FlattenOption{ + Level: FlattenLevelUnlimited, + Gap: "_", + }, + }) + + // Check + r1 := strings.Join(csvData[0], ",") + r2 := strings.Join(csvData[1], ",") + r3 := strings.Join(csvData[2], ",") + r4 := strings.Join(csvData[3], ",") + exp1 := "id,is active,nested_a,nested_b,score,special1,special2,special3,special4,special5,special6,user" + exp2 := "b042ab5c-ca73-4460-b739-96410ea9d3a6,false,,,-100,&,<,>,&,<,>,Jon Doe" + exp3 := "ce06f5b1-5721-42c0-91e1-9f72a09c250a,true,1,2,1.5,,,,,,,Tuấn" + exp4 := "4e01b638-44e5-4079-8043-baabbff21cc8,true,,,100000000000000000,,,,,,,高橋" + if r1 != exp1 { + t.Fatalf("created headers are incorrect, %s is not equal expected %s", r1, exp1) + } + if r2 != exp2 { + t.Fatalf("created headers are incorrect, %s is not equal expected %s", r2, exp2) + } + if r3 != exp3 { + t.Fatalf("created headers are incorrect, %s is not equal expected %s", r3, exp3) + } + if r4 != exp4 { + t.Fatalf("created headers are incorrect, %s is not equal expected %s", r4, exp4) + } +} + func TestCreateCsvHeader(t *testing.T) { - jsonArray := []map[string]interface{}{ + // Prepare + data := JsonArray{ { - "a": true, - "b": false, + "id": "b042ab5c-ca73-4460-b739-96410ea9d3a6", + "user": "Jon Doe", + "score": -100, + "is active": false, + "special1": "&", + "special2": "<", + "special3": ">", + "special4": "\u0026", + "special5": "\u003c", + "special6": "\u003e", }, + { + "id": "ce06f5b1-5721-42c0-91e1-9f72a09c250a", + "user": "Tuấn", + "score": 1.5, + "is active": true, + "nested": JsonObject{ + "a": 1, + "b": 2, + }, + }, + { + "id": "4e01b638-44e5-4079-8043-baabbff21cc8", + "user": "高橋", + "score": 100000000000000000, + "is active": true, + }, + } + + // Process + headers := CreateCsvHeader(data, []string{"x", "y", "z", "3", "2", "1"}) + + // Check + if len(headers) != 17 { + t.Fatalf("created headers have wrong length %v", len(headers)) } - headers := CreateCsvHeader(jsonArray, nil) - if len(headers) == 0 { - t.Fatalf("headers must not be empty") + s := strings.Join(headers, ",") + expected := `x,y,z,3,2,1,id,is active,nested,score,special1,special2,special3,special4,special5,special6,user` + if s != expected { + t.Fatalf("created headers are incorrect") } } diff --git a/json_flattener.go b/json_flattener.go index 69207fa..fab7afc 100644 --- a/json_flattener.go +++ b/json_flattener.go @@ -1,46 +1,73 @@ package jsonconv import ( + "fmt" "reflect" +) - "github.com/tuan78/jsonconv/utils" +const ( + FlattenLevelUnlimited = -1 // Set it to FlattenOption.Level for unlimited flattening + FlattenLevelNonNested = 0 // Set it to FlattenOption.Level for non-nested flattening (equivalent to non-flattening) ) -func FlattenJsonObject(jsonObj JsonObject, flattenLevel int) { - nestedLevel := 0 - needLoop := true - for needLoop { - needLoop = false // To exit the loop. +// A FlattenOption is for JSON object flattening. +type FlattenOption struct { + Level int // Level of flattening, it can be FlattenLevelUnlimited, FlattenLevelNonNested or an int value in [1..n] + Gap string // A gap between nested JSON and its parent JSON. It will be used when merging nested JSON's key with parent JSON's key + SkipMap bool // Skip Map type (typically JSON Object type) from flattening process + SkipArray bool // Skip Array type (JSON array, string array, int array, float array, etc.) from flattening process +} - // Find and extract nested json object. - extractedKeys := make([]string, 0) - flattenedKV := make(JsonObject) - for key, val := range jsonObj { - switch nested := val.(type) { - case JsonObject: - for nkey, nval := range nested { - // Append json object's key with nested json object's key. - csvKey := key + "__" + nkey - flattenedKV[csvKey] = nval +// FlattenJsonObject flattens obj with given op. +func FlattenJsonObject(obj JsonObject, op *FlattenOption) { + kset := make(map[string]struct{}) + ks := make([]string, 0) + for k := range obj { + ks = append(ks, k) + } + for _, k := range ks { + curLvl := 0 + val := reflect.ValueOf(obj[k]) + extractJsonObject(k, &val, obj, kset, op, curLvl) + } + for k := range kset { + delete(obj, k) + } +} - // Check if nested value is kind of map. - if reflect.ValueOf(nval).Kind() == reflect.Map { - if flattenLevel == -1 || (flattenLevel > nestedLevel) { - needLoop = true // Need one more loop. - } - } - } - // Store extracted nested json object's key for later uses. - extractedKeys = append(extractedKeys, key) - } +// extractJsonObject processes obj extraction with k, refval pairs and given op. +// When extracting map, slice and array, a new key will be stores in kset and curLvl will be increased. +func extractJsonObject(k string, refval *reflect.Value, obj JsonObject, kset map[string]struct{}, op *FlattenOption, curLvl int) { + more := op.Level == FlattenLevelUnlimited || op.Level > curLvl + for refval.Kind() == reflect.Interface { + *refval = refval.Elem() + } + switch refval.Kind() { + case reflect.Map: + if !more || op.SkipMap { + obj[k] = refval.Interface() + return } - - // Update json object with its flattened children. - utils.CopyMap(flattenedKV, jsonObj) - for _, key := range extractedKeys { - delete(jsonObj, key) + kset[k] = struct{}{} + ks := refval.MapKeys() + for _, nk := range ks { + newK := fmt.Sprintf("%s%s%s", k, op.Gap, nk.String()) + nv := refval.MapIndex(nk) + extractJsonObject(newK, &nv, obj, kset, op, curLvl+1) } - - nestedLevel++ + case reflect.Slice, reflect.Array: + if !more || op.SkipArray { + obj[k] = refval.Interface() + return + } + kset[k] = struct{}{} + len := refval.Len() + for i := 0; i < len; i++ { + nv := refval.Index(i) + newK := fmt.Sprintf("%s[%v]", k, i) + extractJsonObject(newK, &nv, obj, kset, op, curLvl+1) + } + default: + obj[k] = refval.Interface() } } diff --git a/json_flattener_test.go b/json_flattener_test.go new file mode 100644 index 0000000..dff1346 --- /dev/null +++ b/json_flattener_test.go @@ -0,0 +1,408 @@ +package jsonconv + +import ( + "testing" +) + +func sampleJsonObject() JsonObject { + return JsonObject{ + "id": "b042ab5c-ca73-4460-b739-96410ea9d3a6", + "user": "Jon Doe", + "score": -100, + "is active": false, + "special1": "&", + "special2": "<", + "special3": ">", + "special4": "\u0026", + "special5": "\u003c", + "special6": "\u003e", + "nested": JsonObject{ + "a": 1, + "b": 2, + "c": JsonObject{ + "d": JsonObject{ + "e": 3, + }, + }, + "f": []int{4, 5, 6}, + "g": JsonObject{ + "h": "A", + "i": true, + "j": 1, + "k": 1.5, + }, + }, + } +} + +func TestFlattenJsonObject_UnlimitedLevel(t *testing.T) { + // Prepare + data := sampleJsonObject() + + // Process + FlattenJsonObject(data, &FlattenOption{ + Level: FlattenLevelUnlimited, + Gap: "__", + }) + + // Check + expected := JsonObject{ + "id": "b042ab5c-ca73-4460-b739-96410ea9d3a6", + "user": "Jon Doe", + "score": -100, + "is active": false, + "special1": "&", + "special2": "<", + "special3": ">", + "special4": "\u0026", + "special5": "\u003c", + "special6": "\u003e", + "nested__a": 1, + "nested__b": 2, + "nested__c__d__e": 3, + "nested__f[0]": 4, + "nested__f[1]": 5, + "nested__f[2]": 6, + "nested__g__h": "A", + "nested__g__i": true, + "nested__g__j": 1, + "nested__g__k": 1.5, + } + for k := range expected { + ev := expected[k] + v := data[k] + if ev != v { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", v, ev) + } + } +} + +func TestFlattenJsonObject_NonNestedLevel(t *testing.T) { + // Prepare + data := sampleJsonObject() + + // Process + FlattenJsonObject(data, &FlattenOption{ + Level: FlattenLevelNonNested, + Gap: "|", + }) + + // Check + expected := JsonObject{ + "id": "b042ab5c-ca73-4460-b739-96410ea9d3a6", + "user": "Jon Doe", + "score": -100, + "is active": false, + "special1": "&", + "special2": "<", + "special3": ">", + "special4": "\u0026", + "special5": "\u003c", + "special6": "\u003e", + "nested": JsonObject{ + "a": 1, + "b": 2, + "c": JsonObject{ + "d": JsonObject{ + "e": 3, + }, + }, + "f": []int{4, 5, 6}, + "g": JsonObject{ + "h": "A", + "i": true, + "j": 1, + "k": 1.5, + }, + }, + } + + // Check flattened values. + for k := range expected { + ev := expected[k] + v := data[k] + if k != "nested" && ev != v { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", v, ev) + } + } + + // Check nested object. + nes := data["nested"].(JsonObject) + enes := expected["nested"].(JsonObject) + if nes["a"] != enes["a"] { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", nes["a"], enes["a"]) + } + if nes["b"] != enes["b"] { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", nes["b"], enes["b"]) + } + + c := nes["c"].(JsonObject) + d := c["d"].(JsonObject) + ec := enes["c"].(JsonObject) + ed := ec["d"].(JsonObject) + if d["e"] != ed["e"] { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", d["e"], ed["e"]) + } + + f := nes["f"].([]int) + ef := enes["f"].([]int) + for idx := range ef { + if f[idx] != ef[idx] { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", f[idx], ef[idx]) + } + } + + g := nes["g"].(JsonObject) + eg := enes["g"].(JsonObject) + for k, v := range eg { + if g[k] != v { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", g[k], v) + } + } +} + +func TestFlattenJsonObject_FirstLevel(t *testing.T) { + // Prepare + data := sampleJsonObject() + + // Process + FlattenJsonObject(data, &FlattenOption{ + Level: 1, + Gap: "|", + }) + + // Check + expected := JsonObject{ + "id": "b042ab5c-ca73-4460-b739-96410ea9d3a6", + "user": "Jon Doe", + "score": -100, + "is active": false, + "special1": "&", + "special2": "<", + "special3": ">", + "special4": "\u0026", + "special5": "\u003c", + "special6": "\u003e", + "nested|a": 1, + "nested|b": 2, + "nested|c": JsonObject{ + "d": JsonObject{ + "e": 3, + }, + }, + "nested|f": []int{4, 5, 6}, + "nested|g": JsonObject{ + "h": "A", + "i": true, + "j": 1, + "k": 1.5, + }, + } + for k := range expected { + ev := expected[k] + v := data[k] + if k != "nested|c" && k != "nested|f" && k != "nested|g" && ev != v { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", v, ev) + } + } + + // Check nested object. + c := data["nested|c"].(JsonObject) + d := c["d"].(JsonObject) + ec := expected["nested|c"].(JsonObject) + ed := ec["d"].(JsonObject) + if d["e"] != ed["e"] { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", d["e"], ed["e"]) + } + + f := data["nested|f"].([]int) + ef := expected["nested|f"].([]int) + for idx := range ef { + if f[idx] != ef[idx] { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", f[idx], ef[idx]) + } + } + + g := data["nested|g"].(JsonObject) + eg := expected["nested|g"].(JsonObject) + for k, v := range eg { + if g[k] != v { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", g[k], v) + } + } +} + +func TestFlattenJsonObject_Ignores_Map(t *testing.T) { + // Prepare + data := sampleJsonObject() + + // Process + FlattenJsonObject(data, &FlattenOption{ + Level: FlattenLevelUnlimited, + Gap: "|", + SkipMap: true, + }) + + // Check + expected := JsonObject{ + "id": "b042ab5c-ca73-4460-b739-96410ea9d3a6", + "user": "Jon Doe", + "score": -100, + "is active": false, + "special1": "&", + "special2": "<", + "special3": ">", + "special4": "\u0026", + "special5": "\u003c", + "special6": "\u003e", + "nested": JsonObject{ + "a": 1, + "b": 2, + "c": JsonObject{ + "d": JsonObject{ + "e": 3, + }, + }, + "f": []int{4, 5, 6}, + "g": JsonObject{ + "h": "A", + "i": true, + "j": 1, + "k": 1.5, + }, + }, + } + for k := range expected { + ev := expected[k] + v := data[k] + if k != "nested" && ev != v { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", v, ev) + } + } + + // Check nested object. + nes := data["nested"].(JsonObject) + enes := expected["nested"].(JsonObject) + if nes["a"] != enes["a"] { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", nes["a"], enes["a"]) + } + if nes["b"] != enes["b"] { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", nes["b"], enes["b"]) + } + + c := nes["c"].(JsonObject) + d := c["d"].(JsonObject) + ec := enes["c"].(JsonObject) + ed := ec["d"].(JsonObject) + if d["e"] != ed["e"] { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", d["e"], ed["e"]) + } + + f := nes["f"].([]int) + ef := enes["f"].([]int) + for idx := range ef { + if f[idx] != ef[idx] { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", f[idx], ef[idx]) + } + } + + g := nes["g"].(JsonObject) + eg := enes["g"].(JsonObject) + for k, v := range eg { + if g[k] != v { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", g[k], v) + } + } +} + +func TestFlattenJsonObject_Ignores_Array(t *testing.T) { + // Prepare + data := sampleJsonObject() + + // Process + FlattenJsonObject(data, &FlattenOption{ + Level: FlattenLevelUnlimited, + Gap: "|", + SkipArray: true, + }) + + // Check + expected := JsonObject{ + "id": "b042ab5c-ca73-4460-b739-96410ea9d3a6", + "user": "Jon Doe", + "score": -100, + "is active": false, + "special1": "&", + "special2": "<", + "special3": ">", + "special4": "\u0026", + "special5": "\u003c", + "special6": "\u003e", + "nested|a": 1, + "nested|b": 2, + "nested|c|d|e": 3, + "nested|f": []int{4, 5, 6}, + "nested|g|h": "A", + "nested|g|i": true, + "nested|g|j": 1, + "nested|g|k": 1.5, + } + for k := range expected { + ev := expected[k] + v := data[k] + if k != "nested|f" && ev != v { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", v, ev) + } + } + + // Check nested object. + f := data["nested|f"].([]int) + ef := expected["nested|f"].([]int) + for idx := range ef { + if f[idx] != ef[idx] { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", f[idx], ef[idx]) + } + } +} + +func TestFlattenJsonObject_Gap(t *testing.T) { + // Prepare + data := sampleJsonObject() + + // Process + FlattenJsonObject(data, &FlattenOption{ + Level: FlattenLevelUnlimited, + Gap: "|", + }) + + // Check + expected := JsonObject{ + "id": "b042ab5c-ca73-4460-b739-96410ea9d3a6", + "user": "Jon Doe", + "score": -100, + "is active": false, + "special1": "&", + "special2": "<", + "special3": ">", + "special4": "\u0026", + "special5": "\u003c", + "special6": "\u003e", + "nested|a": 1, + "nested|b": 2, + "nested|c|d|e": 3, + "nested|f[0]": 4, + "nested|f[1]": 5, + "nested|f[2]": 6, + "nested|g|h": "A", + "nested|g|i": true, + "nested|g|j": 1, + "nested|g|k": 1.5, + } + for k := range expected { + ev := expected[k] + v := data[k] + if ev != v { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", v, ev) + } + } +} diff --git a/json_reader.go b/json_reader.go index a5f1b50..932209f 100644 --- a/json_reader.go +++ b/json_reader.go @@ -3,41 +3,23 @@ package jsonconv import ( "encoding/json" "io" - "os" - "strings" ) +// A JsonReader reads and decodes JSON values from an input stream. type JsonReader struct { reader io.Reader - closer io.Closer -} - -func NewJsonReaderFromFile(path string) (*JsonReader, error) { - fi, err := os.Open(path) - if err != nil { - return nil, err - } - re := NewJsonReader(fi) - re.closer = fi - return re, nil -} - -func NewJsonReaderFromString(rawData string) *JsonReader { - re := strings.NewReader(rawData) - return NewJsonReader(re) } +// NewJsonReader returns a new JsonReader that reads from r. func NewJsonReader(r io.Reader) *JsonReader { return &JsonReader{ reader: r, } } +// Read reads the next JSON-encoded value from its +// input and stores it in the value pointed to by v. func (r *JsonReader) Read(v any) error { - if r.closer != nil { - defer r.closer.Close() - } - decoder := json.NewDecoder(r.reader) for decoder.More() { err := decoder.Decode(v) diff --git a/json_reader_test.go b/json_reader_test.go index ef490f5..c213a72 100644 --- a/json_reader_test.go +++ b/json_reader_test.go @@ -1,53 +1,104 @@ package jsonconv import ( - "path/filepath" + "strings" "testing" ) -func TestJsonReaderFromString(t *testing.T) { - rawData := ` +func TestJsonReader_InvalidJson(t *testing.T) { + // Prepare + raw := `"id": "b042ab5c-ca73-4460-b739-96410ea9d3a6" }` + obj := make(JsonObject) + re := NewJsonReader(strings.NewReader(raw)) + + // Process + err := re.Read(&obj) + + // Check + if err == nil { + t.Fatalf("Should throw an error for invalid JSON") + } +} + +func TestJsonReader_JsonObject(t *testing.T) { + // Prepare + raw := ` { - "a": "test", - "b": "test" + "id": "b042ab5c-ca73-4460-b739-96410ea9d3a6", + "user": "Jon Doe", + "score": "-100", + "is active": "false" }` - jsonObject := make(JsonObject) - reader := NewJsonReaderFromString(rawData) - err := reader.Read(&jsonObject) + obj := make(JsonObject) + re := NewJsonReader(strings.NewReader(raw)) + + // Process + err := re.Read(&obj) if err != nil { - t.Fatalf("failed to read file %v", err) + t.Fatalf("failed to read json, err: %v", err) } - if _, exist := jsonObject["a"]; !exist { + + // Check + if obj["id"] != "b042ab5c-ca73-4460-b739-96410ea9d3a6" || + obj["user"] != "Jon Doe" || + obj["score"] != "-100" || + obj["is active"] != "false" { t.Fatalf("failed to read json object") } } -func TestJsonReaderFromFile_JsonObject(t *testing.T) { - jsonObject := make(JsonObject) - reader, err := NewJsonReaderFromFile(filepath.Join("tests", "read-json-object.json")) - if err != nil { - t.Fatalf("failed to read file %v", err) - } - err = reader.Read(&jsonObject) +func TestJsonReader_JsonArray(t *testing.T) { + // Prepare + raw := ` + [ + { + "id": "ce06f5b1-5721-42c0-91e1-9f72a09c250a", + "user": "Tuấn", + "score": "1.5", + "is active": "true" + }, + { + "id": "b042ab5c-ca73-4460-b739-96410ea9d3a6", + "user": "Jon Doe", + "score": "-100", + "is active": "false" + }, + { + "id": "4e01b638-44e5-4079-8043-baabbff21cc8", + "user": "高橋", + "score": "100000000000000000000000", + "is active": "true" + } + ]` + arr := make(JsonArray, 0) + re := NewJsonReader(strings.NewReader(raw)) + + // Process + err := re.Read(&arr) if err != nil { t.Fatalf("failed to read file %v", err) } - if _, exist := jsonObject["a"]; !exist { - t.Fatalf("failed to read json object") - } -} -func TestJsonReaderFromFile_JsonArray(t *testing.T) { - jsonArray := make(JsonArray, 0) - reader, err := NewJsonReaderFromFile(filepath.Join("tests", "read-json-array.json")) - if err != nil { - t.Fatalf("failed to read file %v", err) + // Check + if len(arr) == 0 { + t.Fatalf("failed to read json array") } - err = reader.Read(&jsonArray) - if err != nil { - t.Fatalf("failed to read file %v", err) + if arr[0]["id"] != "ce06f5b1-5721-42c0-91e1-9f72a09c250a" || + arr[0]["user"] != "Tuấn" || + arr[0]["score"] != "1.5" || + arr[0]["is active"] != "true" { + t.Fatalf("failed to read json array") + } + if arr[1]["id"] != "b042ab5c-ca73-4460-b739-96410ea9d3a6" || + arr[1]["user"] != "Jon Doe" || + arr[1]["score"] != "-100" || + arr[1]["is active"] != "false" { + t.Fatalf("failed to read json array") } - if len(jsonArray) == 0 { + if arr[2]["id"] != "4e01b638-44e5-4079-8043-baabbff21cc8" || + arr[2]["user"] != "高橋" || + arr[2]["score"] != "100000000000000000000000" || + arr[2]["is active"] != "true" { t.Fatalf("failed to read json array") } } diff --git a/json_writer.go b/json_writer.go index e9cec06..7f017b9 100644 --- a/json_writer.go +++ b/json_writer.go @@ -1,3 +1,28 @@ package jsonconv -// WIP +import ( + "encoding/json" + "io" +) + +// A JsonWriter writes JSON values to an output stream. +type JsonWriter struct { + EscapeHTML bool + writer io.Writer +} + +// NewJsonWriter returns a new JsonWriter that writes to w. +func NewJsonWriter(w io.Writer) *JsonWriter { + return &JsonWriter{ + writer: w, + EscapeHTML: true, + } +} + +// Write writes the JSON encoding of v to the stream, +// followed by a newline character. +func (r *JsonWriter) Write(v any) error { + encoder := json.NewEncoder(r.writer) + encoder.SetEscapeHTML(r.EscapeHTML) + return encoder.Encode(v) +} diff --git a/json_writer_test.go b/json_writer_test.go new file mode 100644 index 0000000..6f21831 --- /dev/null +++ b/json_writer_test.go @@ -0,0 +1,90 @@ +package jsonconv + +import ( + "bytes" + "testing" +) + +func TestJsonWriter_JsonObject(t *testing.T) { + // Prepare + data := JsonObject{ + "id": "b042ab5c-ca73-4460-b739-96410ea9d3a6", + "user": "Jon Doe", + "score": -100, + "is active": false, + "special1": "&", + "special2": "<", + "special3": ">", + "special4": "\u0026", + "special5": "\u003c", + "special6": "\u003e", + } + buf := &bytes.Buffer{} + wr := NewJsonWriter(buf) + + // Process + err := wr.Write(data) + if err != nil { + t.Fatalf("failed to write json, err: %v", err) + } + + // Check + s := buf.String() + expect := `{"id":"b042ab5c-ca73-4460-b739-96410ea9d3a6","is active":false,"score":-100,"special1":"\u0026","special2":"\u003c","special3":"\u003e","special4":"\u0026","special5":"\u003c","special6":"\u003e","user":"Jon Doe"} +` + if s == "" { + t.Fatalf("failed to write json to byte buffer") + } + if s != expect { + t.Fatalf("json output is not correct") + } +} + +func TestJsonWriter_JsonArray(t *testing.T) { + // Prepare + data := JsonArray{ + { + "id": "b042ab5c-ca73-4460-b739-96410ea9d3a6", + "user": "Jon Doe", + "score": -100, + "is active": false, + "special1": "&", + "special2": "<", + "special3": ">", + "special4": "\u0026", + "special5": "\u003c", + "special6": "\u003e", + }, + { + "id": "ce06f5b1-5721-42c0-91e1-9f72a09c250a", + "user": "Tuấn", + "score": 1.5, + "is active": true, + }, + { + "id": "4e01b638-44e5-4079-8043-baabbff21cc8", + "user": "高橋", + "score": 100000000000000000, + "is active": true, + }, + } + buf := &bytes.Buffer{} + wr := NewJsonWriter(buf) + + // Process + err := wr.Write(data) + if err != nil { + t.Fatalf("failed to write json, err: %v", err) + } + + // Check + s := buf.String() + expected := `[{"id":"b042ab5c-ca73-4460-b739-96410ea9d3a6","is active":false,"score":-100,"special1":"\u0026","special2":"\u003c","special3":"\u003e","special4":"\u0026","special5":"\u003c","special6":"\u003e","user":"Jon Doe"},{"id":"ce06f5b1-5721-42c0-91e1-9f72a09c250a","is active":true,"score":1.5,"user":"Tuấn"},{"id":"4e01b638-44e5-4079-8043-baabbff21cc8","is active":true,"score":100000000000000000,"user":"高橋"}] +` + if s == "" { + t.Fatalf("failed to write json to byte buffer") + } + if s != expected { + t.Fatalf("json output is not correct") + } +} diff --git a/tests/read-json-array.json b/tests/read-json-array.json deleted file mode 100644 index 7a22b1e..0000000 --- a/tests/read-json-array.json +++ /dev/null @@ -1,5 +0,0 @@ -[ - { - "a": "test" - } -] diff --git a/tests/read-json-object.json b/tests/read-json-object.json deleted file mode 100644 index cc9b078..0000000 --- a/tests/read-json-object.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "a": "test" -} diff --git a/tool/cmd/csv.go b/tool/cmd/csv.go index 83f164b..07a7216 100644 --- a/tool/cmd/csv.go +++ b/tool/cmd/csv.go @@ -1,10 +1,11 @@ package cmd import ( + "bytes" "fmt" "os" "path/filepath" - "time" + "strings" "github.com/spf13/cobra" "github.com/tuan78/jsonconv" @@ -14,9 +15,14 @@ import ( func NewCsvCmd() *cobra.Command { var ( - baseHeaders []string - delimiter string - useCRLF bool + baseHs []string + delim string + crlf bool + noft bool + flv int + fga string + fsm bool + fsa bool ) cmd := &cobra.Command{ @@ -24,106 +30,172 @@ func NewCsvCmd() *cobra.Command { Short: "Convert JSON to CSV", Long: "Convert JSON to CSV", RunE: func(cmd *cobra.Command, args []string) error { - return processCsvCmd(baseHeaders, delimiter, useCRLF) + in := &csvCmdInput{ + inputPath: params.InputPath, + outputPath: params.OutputPath, + raw: params.RawData, + baseHs: baseHs, + delim: delim, + useCRLF: crlf, + } + if !noft { + in.flattenOp = &jsonconv.FlattenOption{ + Level: flv, + Gap: fga, + SkipMap: fsm, + SkipArray: fsa, + } + } + return processCsvCmd(in) }, } cmd.PersistentFlags().SortFlags = false - cmd.PersistentFlags().StringSliceVar(&baseHeaders, "headers", nil, "headers in CSV that always appears before dynamic headers (auto detected from JSON)") - cmd.PersistentFlags().StringVar(&delimiter, "delimiter", ",", "field delimiter") - cmd.PersistentFlags().BoolVar(&useCRLF, "useCRLF", false, "set it true to use \\r\\n as the line terminator") + cmd.PersistentFlags().StringSliceVar(&baseHs, "hs", nil, "headers in CSV that always appears before dynamic headers (auto detected from JSON)") + cmd.PersistentFlags().StringVar(&delim, "delim", ",", "field delimiter") + cmd.PersistentFlags().BoolVar(&crlf, "crlf", false, "set it true to use \\r\\n as the line terminator") + cmd.PersistentFlags().BoolVar(&noft, "noft", false, "set it true to skip JSON flattening") + cmd.PersistentFlags().IntVar(&flv, "flv", jsonconv.FlattenLevelUnlimited, "flatten level for flattening a nested JSON (-1: unlimited, 0: no nested, [1...n]: n level of nested JSON)") + cmd.PersistentFlags().StringVar(&fga, "fga", "__", "flatten gap for separating JSON object with its nested data") + cmd.PersistentFlags().BoolVar(&fsm, "fsm", false, "flatten but skip map type") + cmd.PersistentFlags().BoolVar(&fsa, "fsa", false, "flatten but skip array type") return cmd } -func processCsvCmd(baseHeaders []string, delimiter string, useCRLF bool) error { +type csvCmdInput struct { + inputPath string + outputPath string + raw string + baseHs []string + delim string + useCRLF bool + flattenOp *jsonconv.FlattenOption +} + +func processCsvCmd(in *csvCmdInput) error { var err error // Create JSON reader. - var jsonReader *jsonconv.JsonReader + var jr *jsonconv.JsonReader switch { - case params.RawData != "": - jsonReader = jsonconv.NewJsonReaderFromString(params.RawData) - case params.InputPath != "": - jsonReader, err = jsonconv.NewJsonReaderFromFile(params.InputPath) + case in.raw != "": + jr = jsonconv.NewJsonReader(strings.NewReader(in.raw)) + case in.inputPath != "": + fi, err := os.Open(in.inputPath) if err != nil { return err } + defer fi.Close() + jr = jsonconv.NewJsonReader(fi) case !utils.IsStdinEmpty(): - jsonReader = jsonconv.NewJsonReader(os.Stdin) + fi := os.Stdin + defer fi.Close() + jr = jsonconv.NewJsonReader(fi) default: return fmt.Errorf("need to input either raw data, input file path or data from stdin") } - // Read JSON data and store in jsonArray. - fmt.Println("Processing...") + // Read and parse JSON data. var encoded interface{} - err = jsonReader.Read(&encoded) + err = jr.Read(&encoded) if err != nil { return err } - var jsonArray jsonconv.JsonArray + var arr jsonconv.JsonArray switch val := encoded.(type) { case []interface{}: for _, v := range val { - if jsonObject, ok := v.(jsonconv.JsonObject); ok { - jsonArray = append(jsonArray, jsonObject) + if obj, ok := v.(jsonconv.JsonObject); ok { + arr = append(arr, obj) continue } return fmt.Errorf("unknown type of JSON data") } case jsonconv.JsonObject: - jsonArray = append(jsonArray, val) + arr = append(arr, val) default: return fmt.Errorf("unknown type of JSON data") } // Convert JSON to CSV. - var csvData [][]string - csvData, err = jsonconv.Convert(&jsonconv.ConvertInput{ - JsonArray: jsonArray, - FlattenLevel: params.FlattenLevel, - BaseHeaders: baseHeaders, + data := jsonconv.ToCsv(arr, &jsonconv.ToCsvOption{ + FlattenOption: in.flattenOp, + BaseHeaders: in.baseHs, }) - if err != nil { - return err - } - if len(csvData) == 0 { + if len(data) == 0 { return fmt.Errorf("empty CSV data") } - return writeToCsvFile(csvData, delimiter, useCRLF) + // Convert in.delim to rune. + runes := []rune(in.delim) + var delimRune *rune + if len(runes) > 0 { + delimRune = &runes[0] + } + + // Output the CSV content. + return outputCsvContent(data, in.outputPath, delimRune, in.useCRLF) } -func writeToCsvFile(csvData jsonconv.CsvData, delimiter string, useCRLF bool) error { +func outputCsvContent(data jsonconv.CsvData, filePath string, delim *rune, useCRLF bool) error { + var err error + // Check and override outputPath if necessary. - if params.OutputPath == "" { - workingDir, err := os.Getwd() + path := filePath + if path == "" { + // Create CSV writer with byte buffer. + buf := &bytes.Buffer{} + cw := jsonconv.NewCsvWriter(buf) + cw.Delimiter = delim + cw.UseCRLF = useCRLF + + // Write to CSV file. + err = cw.Write(data) if err != nil { return err } - fileName := fmt.Sprintf("output-%v.csv", time.Now().UTC().Unix()) - params.OutputPath = filepath.Join(workingDir, fileName) - } + fmt.Printf("%s\n", buf.String()) + } else { + var fi *os.File + // Check file path and make dir accordingly. + if strings.Contains(path, string(filepath.Separator)) || + strings.HasPrefix(path, ".") || + strings.HasPrefix(path, "~") { + // Ensure all dir in path exists. + err := os.MkdirAll(filepath.Dir(path), os.ModePerm) + if err != nil { + return err + } + fi, err = os.Create(path) + if err != nil { + return err + } + defer fi.Close() + } else { + // Path is only file name so override it with full path (working dir + file name). + dir, err := os.Getwd() + if err != nil { + return err + } + path = filepath.Join(dir, filePath) + fi, err = os.Create(path) + if err != nil { + return err + } + defer fi.Close() + } - // Create CSV writer. - runes := []rune(delimiter) - var delimiterRune *rune - if len(runes) > 0 { - delimiterRune = &runes[0] - } - csvWriter, err := jsonconv.NewCsvWriterFromFile(params.OutputPath) - if err != nil { - return err - } - csvWriter.Delimiter = delimiterRune - csvWriter.UseCRLF = useCRLF + // Create CSV writer with output file. + cw := jsonconv.NewCsvWriter(fi) + cw.Delimiter = delim + cw.UseCRLF = useCRLF - // Write to CSV file. - err = csvWriter.Write(csvData) - if err != nil { - return err + // Write to CSV file. + err = cw.Write(data) + if err != nil { + return err + } + fmt.Printf("The CSV file is located at %s\n", path) } - - fmt.Printf("Done. The CSV file is located at %s\n", params.OutputPath) return nil } diff --git a/tool/cmd/flatten.go b/tool/cmd/flatten.go index 594cb14..00ee832 100644 --- a/tool/cmd/flatten.go +++ b/tool/cmd/flatten.go @@ -1,19 +1,173 @@ package cmd -import "github.com/spf13/cobra" +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/spf13/cobra" + "github.com/tuan78/jsonconv" + "github.com/tuan78/jsonconv/tool/params" + "github.com/tuan78/jsonconv/utils" +) func NewFlattenCmd() *cobra.Command { + var ( + lvl int + gap string + sm bool + sa bool + ) + cmd := &cobra.Command{ Use: "flatten", Short: "Flatten JSON object and JSON array", Long: "Flatten JSON object and JSON array", RunE: func(cmd *cobra.Command, args []string) error { - return processFlattenCmd() + in := &flattenCmdInput{ + inputPath: params.InputPath, + outputPath: params.OutputPath, + raw: params.RawData, + flattenOp: &jsonconv.FlattenOption{ + Level: lvl, + Gap: gap, + SkipMap: sm, + SkipArray: sa, + }, + } + return processFlattenCmd(in) }, } + + cmd.PersistentFlags().IntVar(&lvl, "lv", jsonconv.FlattenLevelUnlimited, "level for flattening a nested JSON (-1: unlimited, 0: no nested, [1...n]: n level of nested JSON)") + cmd.PersistentFlags().StringVar(&gap, "ga", "__", "gap for separating JSON object with its nested data") + cmd.PersistentFlags().BoolVar(&sm, "sm", false, "skip map type") + cmd.PersistentFlags().BoolVar(&sa, "sa", false, "skip array type") return cmd } -func processFlattenCmd() error { +type flattenCmdInput struct { + inputPath string + outputPath string + raw string + flattenOp *jsonconv.FlattenOption +} + +func processFlattenCmd(in *flattenCmdInput) error { + var err error + + // Create JSON reader. + var jr *jsonconv.JsonReader + switch { + case in.raw != "": + jr = jsonconv.NewJsonReader(strings.NewReader(in.raw)) + case in.inputPath != "": + fi, err := os.Open(in.inputPath) + if err != nil { + return err + } + defer fi.Close() + jr = jsonconv.NewJsonReader(fi) + case !utils.IsStdinEmpty(): + fi := os.Stdin + defer fi.Close() + jr = jsonconv.NewJsonReader(fi) + default: + return fmt.Errorf("need to input either raw data, input file path or data from stdin") + } + + // Read and parse JSON data. + var encoded interface{} + err = jr.Read(&encoded) + if err != nil { + return err + } + + switch val := encoded.(type) { + case []interface{}: + var arr jsonconv.JsonArray + for _, v := range val { + if obj, ok := v.(jsonconv.JsonObject); ok { + arr = append(arr, obj) + continue + } + return fmt.Errorf("unknown type of JSON data") + } + // Flatten JSON array. + for _, obj := range arr { + jsonconv.FlattenJsonObject(obj, in.flattenOp) + } + + // Output the JSON content. + return outputJsonContent(arr, in.outputPath) + case jsonconv.JsonObject: + // Flatten JSON object. + jsonconv.FlattenJsonObject(val, in.flattenOp) + + // Output the JSON content. + return outputJsonContent(val, in.outputPath) + default: + return fmt.Errorf("unknown type of JSON data") + } +} + +func outputJsonContent(data interface{}, filePath string) error { + var err error + + // Check and override outputPath if necessary. + path := filePath + if path == "" { + // Create JSON writer with byte buffer. + buf := &bytes.Buffer{} + jw := jsonconv.NewJsonWriter(buf) + + // Write to JSON file. + err = jw.Write(data) + if err != nil { + return err + } + fmt.Printf("%s\n", buf.String()) + } else { + var fi *os.File + // Check file path and make dir accordingly. + if strings.Contains(path, string(filepath.Separator)) || + strings.HasPrefix(path, ".") || + strings.HasPrefix(path, "~") { + // Ensure all dir in path exists. + err := os.MkdirAll(filepath.Dir(path), os.ModePerm) + if err != nil { + return err + } + fi, err = os.Create(path) + if err != nil { + return err + } + defer fi.Close() + } else { + // Path is only file name so override it with full path (working dir + file name). + dir, err := os.Getwd() + if err != nil { + return err + } + path = filepath.Join(dir, filePath) + fi, err = os.Create(path) + if err != nil { + return err + } + defer fi.Close() + } + + // Create JSON writer with output file. + jw := jsonconv.NewJsonWriter(fi) + + // Write to JSON file. + err = jw.Write(data) + if err != nil { + return err + } + fmt.Printf("The JSON file is located at %s\n", path) + } return nil } diff --git a/tool/main.go b/tool/main.go index f316f57..123ce1f 100644 --- a/tool/main.go +++ b/tool/main.go @@ -13,7 +13,7 @@ import ( ) var ( - version = "v1.0.0" + version = "v0.1.0" ) var ( @@ -30,7 +30,6 @@ func main() { rootCmd.PersistentFlags().StringVarP(¶ms.RawData, "data", "d", "", "raw JSON array data") rootCmd.PersistentFlags().StringVarP(¶ms.InputPath, "in", "i", "", "input file path") rootCmd.PersistentFlags().StringVarP(¶ms.OutputPath, "out", "o", "", "output file path") - rootCmd.PersistentFlags().IntVarP(¶ms.FlattenLevel, "level", "l", -1, "level for flattening a nested JSON (-1: unlimited, 0: no nested, [1...n]: n level of nested JSON)") // Add commands. rootCmd.AddCommand(cmd.NewFlattenCmd()) diff --git a/tool/params/params.go b/tool/params/params.go index d311799..bab9188 100644 --- a/tool/params/params.go +++ b/tool/params/params.go @@ -1,8 +1,7 @@ package params var ( - InputPath string - OutputPath string - RawData string - FlattenLevel int + InputPath string + OutputPath string + RawData string ) diff --git a/types.go b/types.go index b881999..a1db58d 100644 --- a/types.go +++ b/types.go @@ -1,9 +1,13 @@ package jsonconv -// CSV types. +// A CsvRow (equivalent to list of string) type CsvRow = []string + +// CsvData contains list of CsvRow. type CsvData = []CsvRow -// JSON types. +// A JsonObject used to hold JSON object data. type JsonObject = map[string]interface{} + +// A JsonArray contains list of JsonObject. type JsonArray = []JsonObject diff --git a/utils/map.go b/utils/map.go deleted file mode 100644 index c2027c5..0000000 --- a/utils/map.go +++ /dev/null @@ -1,7 +0,0 @@ -package utils - -func CopyMap(from map[string]interface{}, to map[string]interface{}) { - for k, v := range from { - to[k] = v - } -}