-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1 from tuan78/feat/create-functions-and-tools
feat: create functions and tools
- Loading branch information
Showing
21 changed files
with
1,688 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
# Binaries for programs and plugins | ||
*.exe | ||
*.exe~ | ||
*.dll | ||
*.so | ||
*.dylib | ||
|
||
# Test binary, built with `go test -c` | ||
*.test | ||
|
||
# Output of the go coverage tool, specifically when used with LiteIDE | ||
*.out | ||
|
||
# Dependency directories (remove the comment below to include it) | ||
# vendor/ | ||
|
||
.idea | ||
.vscode | ||
.DS_Store | ||
|
||
# Project's unused files | ||
bin/ | ||
*.csv | ||
*.json |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
.PHONY: all test | ||
all: build | ||
|
||
build: | ||
go build -o bin/jsonconv github.com/tuan78/jsonconv/tool | ||
|
||
test: | ||
go test ./... -cover |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,2 +1,2 @@ | ||
# jsonconv | ||
Golang library and cmd for JSON converter (flatten JSON, JSON to CSV, JSON from CSV, JSON from Excel, and more). | ||
Golang library and cmd for flattening JSON and converting JSON to CSV. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
package jsonconv | ||
|
||
import ( | ||
"encoding/csv" | ||
"io" | ||
) | ||
|
||
// A CsvWriter writes records using CSV encoding. | ||
type CsvWriter struct { | ||
Delimiter *rune // Field delimiter. If nil, it uses default value from csv.NewWriter | ||
UseCRLF bool // True to use \r\n as the line terminator | ||
writer io.Writer | ||
} | ||
|
||
// NewCsvWriter returns a new CsvWriter that writes to w. | ||
func NewCsvWriter(w io.Writer) *CsvWriter { | ||
return &CsvWriter{ | ||
writer: w, | ||
} | ||
} | ||
|
||
// NewDelimiter returns a pointer to v. | ||
func NewDelimiter(v rune) *rune { | ||
return &v | ||
} | ||
|
||
// Write writes all CSV data to w. | ||
func (w *CsvWriter) Write(data CsvData) error { | ||
writer := csv.NewWriter(w.writer) | ||
if w.Delimiter != nil { | ||
writer.Comma = *w.Delimiter | ||
} | ||
writer.UseCRLF = w.UseCRLF | ||
|
||
defer writer.Flush() | ||
for _, v := range data { | ||
if err := writer.Write(v); err != nil { | ||
return err | ||
} | ||
} | ||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,75 @@ | ||
package jsonconv | ||
|
||
import ( | ||
"bytes" | ||
"testing" | ||
) | ||
|
||
func TestCsvWriter_InvalidDelimiter(t *testing.T) { | ||
// Prepare | ||
data := CsvData{ | ||
{ | ||
"id", "user", "score", "is active", | ||
}, | ||
} | ||
buf := &bytes.Buffer{} | ||
wr := NewCsvWriter(buf) | ||
wr.Delimiter = NewDelimiter('\n') | ||
|
||
// Process | ||
err := wr.Write(data) | ||
|
||
// Check | ||
if err == nil { | ||
t.Fatalf("Should throw an error for invalid delimiter") | ||
} | ||
} | ||
|
||
func TestCsvWriter(t *testing.T) { | ||
// Prepare | ||
data := CsvData{ | ||
{ | ||
"id", "user", "score", "is active", | ||
}, | ||
{ | ||
"ce06f5b1-5721-42c0-91e1-9f72a09c250a", "Tuấn", "1.5", "true", | ||
}, | ||
{ | ||
"b042ab5c-ca73-4460-b739-96410ea9d3a6", "Jon Doe", "-100", "false", | ||
}, | ||
{ | ||
"4e01b638-44e5-4079-8043-baabbff21cc8", "高橋", "100000000000000000000000", "true", | ||
}, | ||
{ | ||
"6f0d6265-545c-4366-a78b-4f80c337aa69", "김슬기", "1234567890", "true", | ||
}, | ||
{ | ||
"3fbae214-006d-4ac5-9eea-76c5d611f54a", "Comma,", "0", "false", | ||
}, | ||
} | ||
buf := &bytes.Buffer{} | ||
wr := NewCsvWriter(buf) | ||
wr.Delimiter = NewDelimiter('|') | ||
|
||
// Process | ||
err := wr.Write(data) | ||
if err != nil { | ||
t.Fatalf("failed to write csv, err: %v", err) | ||
} | ||
|
||
// Check | ||
s := buf.String() | ||
expect := `id|user|score|is active | ||
ce06f5b1-5721-42c0-91e1-9f72a09c250a|Tuấn|1.5|true | ||
b042ab5c-ca73-4460-b739-96410ea9d3a6|Jon Doe|-100|false | ||
4e01b638-44e5-4079-8043-baabbff21cc8|高橋|100000000000000000000000|true | ||
6f0d6265-545c-4366-a78b-4f80c337aa69|김슬기|1234567890|true | ||
3fbae214-006d-4ac5-9eea-76c5d611f54a|Comma,|0|false | ||
` | ||
if s == "" { | ||
t.Fatalf("failed to write csv to byte buffer") | ||
} | ||
if s != expect { | ||
t.Fatalf("csv output is not correct") | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
module github.com/tuan78/jsonconv | ||
|
||
go 1.18 | ||
|
||
require ( | ||
github.com/spf13/cobra v1.5.0 | ||
github.com/spf13/pflag v1.0.5 | ||
) | ||
|
||
require github.com/inconshreveable/mousetrap v1.0.0 // indirect |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= | ||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= | ||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= | ||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= | ||
github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= | ||
github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= | ||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= | ||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= | ||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= | ||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,79 @@ | ||
package jsonconv | ||
|
||
import ( | ||
"fmt" | ||
"sort" | ||
) | ||
|
||
// A ToCsvOption converts a JSON Array to CSV data. | ||
type ToCsvOption struct { | ||
FlattenOption *FlattenOption // Set it to apply JSON flattening | ||
BaseHeaders CsvRow // Base CSV headers used to add before dynamic headers | ||
} | ||
|
||
// ToCsv converts a JsonArray to CsvData with given op. | ||
func ToCsv(arr JsonArray, op *ToCsvOption) CsvData { | ||
if len(arr) == 0 { | ||
return CsvData{} | ||
} | ||
|
||
// Flatten JSON. | ||
if op != nil && op.FlattenOption != nil { | ||
for _, obj := range arr { | ||
FlattenJsonObject(obj, op.FlattenOption) | ||
} | ||
} | ||
|
||
// Create CSV rows. | ||
var csvData CsvData | ||
var hs []string | ||
if op != nil && len(op.BaseHeaders) > 0 { | ||
hs = CreateCsvHeader(arr, op.BaseHeaders) | ||
} else { | ||
hs = CreateCsvHeader(arr, nil) | ||
} | ||
csvData = append(csvData, hs) | ||
for _, obj := range arr { | ||
row := make(CsvRow, 0) | ||
for _, h := range hs { | ||
if val, exist := obj[h]; exist { | ||
row = append(row, fmt.Sprintf("%v", val)) | ||
continue | ||
} | ||
row = append(row, "") | ||
} | ||
csvData = append(csvData, row) | ||
} | ||
|
||
return csvData | ||
} | ||
|
||
// CreateCsvHeader creates CsvRow from arr and baseHs. | ||
// A baseHs is base header that we want to put at the beginning of dynamic header, | ||
// we can set baseHs to nil if we just want to have dynamic header only. | ||
func CreateCsvHeader(arr JsonArray, baseHs CsvRow) CsvRow { | ||
hs := make(sort.StringSlice, 0) | ||
hss := make(map[string]struct{}) | ||
|
||
// Get CSV header from json. | ||
for _, obj := range arr { | ||
for k := range obj { | ||
hss[k] = struct{}{} | ||
} | ||
} | ||
|
||
// Exclude base headers from detected headers, then sort filtered list. | ||
for _, h := range baseHs { | ||
delete(hss, h) | ||
} | ||
for h := range hss { | ||
hs = append(hs, h) | ||
} | ||
hs.Sort() | ||
|
||
// Insert BaseHeaders to the beginning of headers. | ||
if len(baseHs) > 0 { | ||
hs = append(baseHs, hs...) | ||
} | ||
return hs | ||
} |
Oops, something went wrong.