diff --git a/csv_writer.go b/csv_writer.go index 043db35..de4c054 100644 --- a/csv_writer.go +++ b/csv_writer.go @@ -7,9 +7,13 @@ import ( // A CsvWriter writes records using CSV encoding. type CsvWriter struct { - Delimiter *rune // Field delimiter. If nil, it uses default value from csv.NewWriter - UseCRLF bool // True to use \r\n as the line terminator - writer io.Writer + // Field delimiter. If nil, it uses default value from csv.NewWriter + Delimiter *rune + + // True to use \r\n as the line terminator + UseCRLF bool + + writer io.Writer } // NewCsvWriter returns a new CsvWriter that writes to w. diff --git a/json_converter.go b/json_converter.go index 5723ee5..9254751 100644 --- a/json_converter.go +++ b/json_converter.go @@ -7,8 +7,11 @@ import ( // A ToCsvOption converts a JSON Array to CSV data. type ToCsvOption struct { - FlattenOption *FlattenOption // Set it to apply JSON flattening - BaseHeaders CsvRow // Base CSV headers used to add before dynamic headers + // Set it to apply JSON flattening + FlattenOption *FlattenOption + + // Base CSV headers used to add before dynamic headers + BaseHeaders CsvRow } // ToCsv converts a JsonArray to CsvData with given op. diff --git a/json_flattener.go b/json_flattener.go index fab7afc..183d919 100644 --- a/json_flattener.go +++ b/json_flattener.go @@ -6,20 +6,53 @@ import ( ) const ( - FlattenLevelUnlimited = -1 // Set it to FlattenOption.Level for unlimited flattening - FlattenLevelNonNested = 0 // Set it to FlattenOption.Level for non-nested flattening (equivalent to non-flattening) + // Set it to FlattenOption.Level for unlimited flattening. + FlattenLevelUnlimited = -1 + + // Set it to FlattenOption.Level for non-nested flattening + // (equivalent to non-flattening). + FlattenLevelNonNested = 0 + + // Set it to FlattenOption.Level for default level flattening + // (equivalent to FlattenLevelUnlimited). + FlattenLevelDefault = FlattenLevelUnlimited ) +// Set it to FlattenOption.Gap for default gap flattening. +const FlattenGapDefault = "__" + // A FlattenOption is for JSON object flattening. type FlattenOption struct { - Level int // Level of flattening, it can be FlattenLevelUnlimited, FlattenLevelNonNested or an int value in [1..n] - Gap string // A gap between nested JSON and its parent JSON. It will be used when merging nested JSON's key with parent JSON's key - SkipMap bool // Skip Map type (typically JSON Object type) from flattening process - SkipArray bool // Skip Array type (JSON array, string array, int array, float array, etc.) from flattening process + // Level of flattening, it can be FlattenLevelUnlimited, + // FlattenLevelNonNested or an int value in [1..n] + Level int + + // A gap between nested JSON and its parent JSON. + // It will be used when merging nested JSON's key with parent JSON's key + Gap string + + // Skip Map type (typically JSON Object type) from flattening process + SkipMap bool + + // Skip Array type (JSON array, string array, int array, float array, etc.) + // from flattening process + SkipArray bool } -// FlattenJsonObject flattens obj with given op. +func DefaultFlattenOption() *FlattenOption { + return &FlattenOption{ + Level: FlattenLevelDefault, + Gap: FlattenGapDefault, + } +} + +// FlattenJsonObject flattens obj with given op. If op is nil, +// it will use op value from DefaultFlattenOption instead. func FlattenJsonObject(obj JsonObject, op *FlattenOption) { + if op == nil { + op = DefaultFlattenOption() + } + kset := make(map[string]struct{}) ks := make([]string, 0) for k := range obj { @@ -67,6 +100,8 @@ func extractJsonObject(k string, refval *reflect.Value, obj JsonObject, kset map newK := fmt.Sprintf("%s[%v]", k, i) extractJsonObject(newK, &nv, obj, kset, op, curLvl+1) } + case reflect.Invalid: + obj[k] = nil default: obj[k] = refval.Interface() } diff --git a/json_flattener_test.go b/json_flattener_test.go index dff1346..7f037ca 100644 --- a/json_flattener_test.go +++ b/json_flattener_test.go @@ -10,12 +10,6 @@ func sampleJsonObject() JsonObject { "user": "Jon Doe", "score": -100, "is active": false, - "special1": "&", - "special2": "<", - "special3": ">", - "special4": "\u0026", - "special5": "\u003c", - "special6": "\u003e", "nested": JsonObject{ "a": 1, "b": 2, @@ -30,11 +24,46 @@ func sampleJsonObject() JsonObject { "i": true, "j": 1, "k": 1.5, + "l": nil, }, }, } } +func TestFlattenJsonObject_NilFlattenOption(t *testing.T) { + // Prepare + data := sampleJsonObject() + + // Process + FlattenJsonObject(data, nil) + + // Check + expected := JsonObject{ + "id": "b042ab5c-ca73-4460-b739-96410ea9d3a6", + "user": "Jon Doe", + "score": -100, + "is active": false, + "nested__a": 1, + "nested__b": 2, + "nested__c__d__e": 3, + "nested__f[0]": 4, + "nested__f[1]": 5, + "nested__f[2]": 6, + "nested__g__h": "A", + "nested__g__i": true, + "nested__g__j": 1, + "nested__g__k": 1.5, + "nested__g__l": nil, + } + for k := range expected { + ev := expected[k] + v := data[k] + if ev != v { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key %s", v, ev, k) + } + } +} + func TestFlattenJsonObject_UnlimitedLevel(t *testing.T) { // Prepare data := sampleJsonObject() @@ -51,12 +80,6 @@ func TestFlattenJsonObject_UnlimitedLevel(t *testing.T) { "user": "Jon Doe", "score": -100, "is active": false, - "special1": "&", - "special2": "<", - "special3": ">", - "special4": "\u0026", - "special5": "\u003c", - "special6": "\u003e", "nested__a": 1, "nested__b": 2, "nested__c__d__e": 3, @@ -67,12 +90,13 @@ func TestFlattenJsonObject_UnlimitedLevel(t *testing.T) { "nested__g__i": true, "nested__g__j": 1, "nested__g__k": 1.5, + "nested__g__l": nil, } for k := range expected { ev := expected[k] v := data[k] if ev != v { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", v, ev) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key %s", v, ev, k) } } } @@ -93,12 +117,6 @@ func TestFlattenJsonObject_NonNestedLevel(t *testing.T) { "user": "Jon Doe", "score": -100, "is active": false, - "special1": "&", - "special2": "<", - "special3": ">", - "special4": "\u0026", - "special5": "\u003c", - "special6": "\u003e", "nested": JsonObject{ "a": 1, "b": 2, @@ -113,6 +131,7 @@ func TestFlattenJsonObject_NonNestedLevel(t *testing.T) { "i": true, "j": 1, "k": 1.5, + "l": nil, }, }, } @@ -122,7 +141,7 @@ func TestFlattenJsonObject_NonNestedLevel(t *testing.T) { ev := expected[k] v := data[k] if k != "nested" && ev != v { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", v, ev) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key %s", v, ev, k) } } @@ -130,10 +149,10 @@ func TestFlattenJsonObject_NonNestedLevel(t *testing.T) { nes := data["nested"].(JsonObject) enes := expected["nested"].(JsonObject) if nes["a"] != enes["a"] { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", nes["a"], enes["a"]) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key a", nes["a"], enes["a"]) } if nes["b"] != enes["b"] { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", nes["b"], enes["b"]) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key b", nes["b"], enes["b"]) } c := nes["c"].(JsonObject) @@ -141,14 +160,14 @@ func TestFlattenJsonObject_NonNestedLevel(t *testing.T) { ec := enes["c"].(JsonObject) ed := ec["d"].(JsonObject) if d["e"] != ed["e"] { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", d["e"], ed["e"]) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key e", d["e"], ed["e"]) } f := nes["f"].([]int) ef := enes["f"].([]int) for idx := range ef { if f[idx] != ef[idx] { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", f[idx], ef[idx]) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key f", f[idx], ef[idx]) } } @@ -156,7 +175,7 @@ func TestFlattenJsonObject_NonNestedLevel(t *testing.T) { eg := enes["g"].(JsonObject) for k, v := range eg { if g[k] != v { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", g[k], v) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key %s", g[k], v, k) } } } @@ -177,12 +196,6 @@ func TestFlattenJsonObject_FirstLevel(t *testing.T) { "user": "Jon Doe", "score": -100, "is active": false, - "special1": "&", - "special2": "<", - "special3": ">", - "special4": "\u0026", - "special5": "\u003c", - "special6": "\u003e", "nested|a": 1, "nested|b": 2, "nested|c": JsonObject{ @@ -196,13 +209,14 @@ func TestFlattenJsonObject_FirstLevel(t *testing.T) { "i": true, "j": 1, "k": 1.5, + "l": nil, }, } for k := range expected { ev := expected[k] v := data[k] if k != "nested|c" && k != "nested|f" && k != "nested|g" && ev != v { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", v, ev) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key %s", v, ev, k) } } @@ -212,14 +226,14 @@ func TestFlattenJsonObject_FirstLevel(t *testing.T) { ec := expected["nested|c"].(JsonObject) ed := ec["d"].(JsonObject) if d["e"] != ed["e"] { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", d["e"], ed["e"]) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key e", d["e"], ed["e"]) } f := data["nested|f"].([]int) ef := expected["nested|f"].([]int) for idx := range ef { if f[idx] != ef[idx] { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", f[idx], ef[idx]) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key nested|f", f[idx], ef[idx]) } } @@ -227,7 +241,7 @@ func TestFlattenJsonObject_FirstLevel(t *testing.T) { eg := expected["nested|g"].(JsonObject) for k, v := range eg { if g[k] != v { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", g[k], v) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key %s", g[k], v, k) } } } @@ -249,12 +263,6 @@ func TestFlattenJsonObject_Ignores_Map(t *testing.T) { "user": "Jon Doe", "score": -100, "is active": false, - "special1": "&", - "special2": "<", - "special3": ">", - "special4": "\u0026", - "special5": "\u003c", - "special6": "\u003e", "nested": JsonObject{ "a": 1, "b": 2, @@ -269,6 +277,7 @@ func TestFlattenJsonObject_Ignores_Map(t *testing.T) { "i": true, "j": 1, "k": 1.5, + "l": nil, }, }, } @@ -276,7 +285,7 @@ func TestFlattenJsonObject_Ignores_Map(t *testing.T) { ev := expected[k] v := data[k] if k != "nested" && ev != v { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", v, ev) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key %s", v, ev, k) } } @@ -284,10 +293,10 @@ func TestFlattenJsonObject_Ignores_Map(t *testing.T) { nes := data["nested"].(JsonObject) enes := expected["nested"].(JsonObject) if nes["a"] != enes["a"] { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", nes["a"], enes["a"]) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key a", nes["a"], enes["a"]) } if nes["b"] != enes["b"] { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", nes["b"], enes["b"]) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key b", nes["b"], enes["b"]) } c := nes["c"].(JsonObject) @@ -295,14 +304,14 @@ func TestFlattenJsonObject_Ignores_Map(t *testing.T) { ec := enes["c"].(JsonObject) ed := ec["d"].(JsonObject) if d["e"] != ed["e"] { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", d["e"], ed["e"]) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key e", d["e"], ed["e"]) } f := nes["f"].([]int) ef := enes["f"].([]int) for idx := range ef { if f[idx] != ef[idx] { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", f[idx], ef[idx]) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key f", f[idx], ef[idx]) } } @@ -310,7 +319,7 @@ func TestFlattenJsonObject_Ignores_Map(t *testing.T) { eg := enes["g"].(JsonObject) for k, v := range eg { if g[k] != v { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", g[k], v) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key %s", g[k], v, k) } } } @@ -332,12 +341,6 @@ func TestFlattenJsonObject_Ignores_Array(t *testing.T) { "user": "Jon Doe", "score": -100, "is active": false, - "special1": "&", - "special2": "<", - "special3": ">", - "special4": "\u0026", - "special5": "\u003c", - "special6": "\u003e", "nested|a": 1, "nested|b": 2, "nested|c|d|e": 3, @@ -346,12 +349,13 @@ func TestFlattenJsonObject_Ignores_Array(t *testing.T) { "nested|g|i": true, "nested|g|j": 1, "nested|g|k": 1.5, + "nested|g|l": nil, } for k := range expected { ev := expected[k] v := data[k] if k != "nested|f" && ev != v { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", v, ev) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key %s", v, ev, k) } } @@ -360,7 +364,7 @@ func TestFlattenJsonObject_Ignores_Array(t *testing.T) { ef := expected["nested|f"].([]int) for idx := range ef { if f[idx] != ef[idx] { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", f[idx], ef[idx]) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key nested|f", f[idx], ef[idx]) } } } @@ -381,12 +385,6 @@ func TestFlattenJsonObject_Gap(t *testing.T) { "user": "Jon Doe", "score": -100, "is active": false, - "special1": "&", - "special2": "<", - "special3": ">", - "special4": "\u0026", - "special5": "\u003c", - "special6": "\u003e", "nested|a": 1, "nested|b": 2, "nested|c|d|e": 3, @@ -397,12 +395,131 @@ func TestFlattenJsonObject_Gap(t *testing.T) { "nested|g|i": true, "nested|g|j": 1, "nested|g|k": 1.5, + "nested|g|l": nil, } for k := range expected { ev := expected[k] v := data[k] if ev != v { - t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v", v, ev) + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key %s", v, ev, k) + } + } +} + +func TestFlattenJsonObject_Special(t *testing.T) { + type specialS struct{} + type specialI interface{} + + // Prepare + spI := new(specialI) + data := JsonObject{ + "special1": "&", + "special2": "<", + "special3": ">", + "special4": "\u0026", + "special5": "\u003c", + "special6": "\u003e", + "a": 1, + "b": 2, + "c": JsonObject{ + "d": JsonObject{ + "e": 3, + }, + }, + "f": []int{4, 5, 6}, + "g": JsonObject{ + "h": "A", + "i": true, + "j": 1, + "k": 1.5, + "l": nil, + "m": func() {}, + "n": specialS{}, + "o": spI, + }, + "nested": JsonObject{ + "special1": "&", + "special2": "<", + "special3": ">", + "special4": "\u0026", + "special5": "\u003c", + "special6": "\u003e", + "a": 1, + "b": 2, + "c": JsonObject{ + "d": JsonObject{ + "e": 3, + }, + }, + "f": []int{4, 5, 6}, + "g": JsonObject{ + "h": "A", + "i": true, + "j": 1, + "k": 1.5, + "l": nil, + "m": func() {}, + "n": specialS{}, + "o": spI, + }, + }, + } + + // Process + FlattenJsonObject(data, &FlattenOption{ + Level: FlattenLevelUnlimited, + Gap: "|", + }) + + // Check + expected := JsonObject{ + "special1": "&", + "special2": "<", + "special3": ">", + "special4": "\u0026", + "special5": "\u003c", + "special6": "\u003e", + "a": 1, + "b": 2, + "c|d|e": 3, + "f[0]": 4, + "f[1]": 5, + "f[2]": 6, + "g|h": "A", + "g|i": true, + "g|j": 1, + "g|k": 1.5, + "g|l": nil, + "g|m": func() {}, + "g|n": specialS{}, + "g|o": spI, + "nested|special1": "&", + "nested|special2": "<", + "nested|special3": ">", + "nested|special4": "\u0026", + "nested|special5": "\u003c", + "nested|special6": "\u003e", + "nested|a": 1, + "nested|b": 2, + "nested|c|d|e": 3, + "nested|f[0]": 4, + "nested|f[1]": 5, + "nested|f[2]": 6, + "nested|g|h": "A", + "nested|g|i": true, + "nested|g|j": 1, + "nested|g|k": 1.5, + "nested|g|l": nil, + "nested|g|m": func() {}, + "nested|g|n": specialS{}, + "nested|g|o": spI, + } + for k := range expected { + ev := expected[k] + v := data[k] + skipFunc := k != "g|m" && k != "nested|g|m" + if skipFunc && ev != v { + t.Fatalf("flattened JSON object is incorrect, %v is not equal expected value %v for key %s", v, ev, k) } } } diff --git a/json_writer.go b/json_writer.go index 7f017b9..fd3c09c 100644 --- a/json_writer.go +++ b/json_writer.go @@ -7,8 +7,13 @@ import ( // A JsonWriter writes JSON values to an output stream. type JsonWriter struct { + // EscapeHTML specifies whether problematic HTML characters + // should be escaped inside JSON quoted strings. + // The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e + // to avoid certain safety problems that can arise when embedding JSON in HTML. EscapeHTML bool - writer io.Writer + + writer io.Writer } // NewJsonWriter returns a new JsonWriter that writes to w. diff --git a/tool/cmd/csv.go b/tool/cmd/csv.go index 07a7216..863400c 100644 --- a/tool/cmd/csv.go +++ b/tool/cmd/csv.go @@ -55,8 +55,8 @@ func NewCsvCmd() *cobra.Command { cmd.PersistentFlags().StringVar(&delim, "delim", ",", "field delimiter") cmd.PersistentFlags().BoolVar(&crlf, "crlf", false, "set it true to use \\r\\n as the line terminator") cmd.PersistentFlags().BoolVar(&noft, "noft", false, "set it true to skip JSON flattening") - cmd.PersistentFlags().IntVar(&flv, "flv", jsonconv.FlattenLevelUnlimited, "flatten level for flattening a nested JSON (-1: unlimited, 0: no nested, [1...n]: n level of nested JSON)") - cmd.PersistentFlags().StringVar(&fga, "fga", "__", "flatten gap for separating JSON object with its nested data") + cmd.PersistentFlags().IntVar(&flv, "flv", jsonconv.FlattenLevelDefault, "flatten level for flattening a nested JSON (-1: unlimited, 0: no nested, [1...n]: n level of nested JSON)") + cmd.PersistentFlags().StringVar(&fga, "fga", jsonconv.FlattenGapDefault, "flatten gap for separating JSON object with its nested data") cmd.PersistentFlags().BoolVar(&fsm, "fsm", false, "flatten but skip map type") cmd.PersistentFlags().BoolVar(&fsa, "fsa", false, "flatten but skip array type") return cmd diff --git a/tool/cmd/flatten.go b/tool/cmd/flatten.go index 00ee832..58b147e 100644 --- a/tool/cmd/flatten.go +++ b/tool/cmd/flatten.go @@ -41,8 +41,8 @@ func NewFlattenCmd() *cobra.Command { }, } - cmd.PersistentFlags().IntVar(&lvl, "lv", jsonconv.FlattenLevelUnlimited, "level for flattening a nested JSON (-1: unlimited, 0: no nested, [1...n]: n level of nested JSON)") - cmd.PersistentFlags().StringVar(&gap, "ga", "__", "gap for separating JSON object with its nested data") + cmd.PersistentFlags().IntVar(&lvl, "lv", jsonconv.FlattenLevelDefault, "level for flattening a nested JSON (-1: unlimited, 0: no nested, [1...n]: n level of nested JSON)") + cmd.PersistentFlags().StringVar(&gap, "ga", jsonconv.FlattenGapDefault, "gap for separating JSON object with its nested data") cmd.PersistentFlags().BoolVar(&sm, "sm", false, "skip map type") cmd.PersistentFlags().BoolVar(&sa, "sa", false, "skip array type") return cmd diff --git a/tool/main.go b/tool/main.go index 123ce1f..52f87fb 100644 --- a/tool/main.go +++ b/tool/main.go @@ -13,7 +13,7 @@ import ( ) var ( - version = "v0.1.0" + version = "v0.1.1" ) var (