Skip to content

Commit

Permalink
Lot of changes for export: wrapped export request
Browse files Browse the repository at this point in the history
  • Loading branch information
@SchawnnDev
SchawnnDev committed Nov 29, 2023
1 parent 19537ef commit ea541ad
Show file tree
Hide file tree
Showing 12 changed files with 361 additions and 320 deletions.
26 changes: 15 additions & 11 deletions internals/export/csv.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,29 +11,30 @@ import (
"go.uber.org/zap"
)

func WriteConvertHitsToCSV(w *csv.Writer, hits []reader.Hit, columns []string, columnsLabel []string, formatColumnsData map[string]string, separator rune) error {
w.Comma = separator
// WriteConvertHitsToCSV writes hits to CSV
func WriteConvertHitsToCSV(w *csv.Writer, hits []reader.Hit, params CSVParameters, writeHeader bool) error {
w.Comma = params.Separator

// avoid to print header when labels are empty
if len(columnsLabel) > 0 {
w.Write(columnsLabel)
if writeHeader && len(params.Columns) > 0 {
w.Write(params.GetColumnsLabel())
}

for _, hit := range hits {
record := make([]string, 0)
for _, column := range columns {
value, err := nestedMapLookup(hit.Fields, strings.Split(column, ".")...)
for _, column := range params.Columns {
value, err := nestedMapLookup(hit.Fields, strings.Split(column.Name, ".")...)
if err != nil {
value = ""
} else if format, ok := formatColumnsData[column]; ok {
} else if column.Format != "" {
if date, ok := value.(time.Time); ok {
value = date.Format(format)
value = date.Format(column.Format)
} else if dateStr, ok := value.(string); ok {
date, err := parseDate(dateStr)
if err != nil {
zap.L().Error("Failed to parse date string:", zap.Any(":", dateStr), zap.Error(err))
} else {
value = date.Format(format)
value = date.Format(column.Format)
}
}
}
Expand All @@ -46,10 +47,11 @@ func WriteConvertHitsToCSV(w *csv.Writer, hits []reader.Hit, columns []string, c
return w.Error()
}

func ConvertHitsToCSV(hits []reader.Hit, columns []string, columnsLabel []string, formatColumnsData map[string]string, separator rune) ([]byte, error) {
// ConvertHitsToCSV converts hits to CSV
func ConvertHitsToCSV(hits []reader.Hit, params CSVParameters, writeHeader bool) ([]byte, error) {
b := new(bytes.Buffer)
w := csv.NewWriter(b)
err := WriteConvertHitsToCSV(w, hits, columns, columnsLabel, formatColumnsData, separator)
err := WriteConvertHitsToCSV(w, hits, params, writeHeader)

if err != nil {
return nil, err
Expand All @@ -58,6 +60,7 @@ func ConvertHitsToCSV(hits []reader.Hit, columns []string, columnsLabel []string
return b.Bytes(), nil
}

// nestedMapLookup looks up a nested map item
func nestedMapLookup(m map[string]interface{}, ks ...string) (rval interface{}, err error) {
var ok bool
if len(ks) == 0 {
Expand All @@ -74,6 +77,7 @@ func nestedMapLookup(m map[string]interface{}, ks ...string) (rval interface{},
}
}

// parseDate parses a date string
func parseDate(dateStr string) (time.Time, error) {
formats := []string{
"2006-01-02T15:04:05.999",
Expand Down
82 changes: 72 additions & 10 deletions internals/export/csv_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,17 @@ func TestConvertHitsToCSV(t *testing.T) {
{ID: "3", Fields: map[string]interface{}{"a": "hello", "b": 20, "c": 3.123456, "date": "2023-06-30T10:42:59.500"}},
{ID: "1", Fields: map[string]interface{}{"a": "hello", "b": 20, "c": 3.123456, "d": map[string]interface{}{"zzz": "nested"}, "date": "2023-06-30T10:42:59.500"}},
}
columns := []string{"a", "b", "c", "d.e", "date"}
columnsLabel := []string{"Label A", "Label B", "Label C", "Label D.E", "Date"}
formatColumnsData := map[string]string{
"date": "02/01/2006",
params := CSVParameters{
Columns: []Column{
{Name: "a", Label: "Label A", Format: ""},
{Name: "b", Label: "Label B", Format: ""},
{Name: "c", Label: "Label C", Format: ""},
{Name: "d.e", Label: "Label D.E", Format: ""},
{Name: "date", Label: "Date", Format: "02/01/2006"},
},
Separator: ',',
}
csv, err := ConvertHitsToCSV(hits, columns, columnsLabel, formatColumnsData, ',')
csv, err := ConvertHitsToCSV(hits, params, true)
if err != nil {
t.Log(err)
t.FailNow()
Expand All @@ -35,17 +40,74 @@ func TestWriteConvertHitsToCSV(t *testing.T) {
{ID: "3", Fields: map[string]interface{}{"a": "hello", "b": 20, "c": 3.123456, "date": "2023-06-30T10:42:59.500"}},
{ID: "1", Fields: map[string]interface{}{"a": "hello", "b": 20, "c": 3.123456, "d": map[string]interface{}{"zzz": "nested"}, "date": "2023-06-30T10:42:59.500"}},
}
columns := []string{"a", "b", "c", "d.e", "date"}
columnsLabel := []string{"Label A", "Label B", "Label C", "Label D.E", "Date"}
formatColumnsData := map[string]string{
"date": "02/01/2006",
params := CSVParameters{
Columns: []Column{
{Name: "a", Label: "Label A", Format: ""},
{Name: "b", Label: "Label B", Format: ""},
{Name: "c", Label: "Label C", Format: ""},
{Name: "d.e", Label: "Label D.E", Format: ""},
{Name: "date", Label: "Date", Format: "02/01/2006"},
},
Separator: ',',
}
b := new(bytes.Buffer)
w := csv2.NewWriter(b)
err := WriteConvertHitsToCSV(w, hits, columns, columnsLabel, formatColumnsData, ',')
err := WriteConvertHitsToCSV(w, hits, params, true)
if err != nil {
t.Log(err)
t.FailNow()
}
t.Log("\n" + string(b.Bytes()))
}

func TestNestedMapLookup_WithEmptyKeys(t *testing.T) {
_, err := nestedMapLookup(map[string]interface{}{}, "")
if err == nil {
t.FailNow()
}
}

func TestNestedMapLookup_WithNonExistentKey(t *testing.T) {
_, err := nestedMapLookup(map[string]interface{}{"a": "hello"}, "b")
if err == nil {
t.FailNow()
}
}

func TestNestedMapLookup_WithNestedNonExistentKey(t *testing.T) {
_, err := nestedMapLookup(map[string]interface{}{"a": map[string]interface{}{"b": "hello"}}, "a", "c")
if err == nil {
t.FailNow()
}
}

func TestNestedMapLookup_WithNestedKey(t *testing.T) {
val, err := nestedMapLookup(map[string]interface{}{"a": map[string]interface{}{"b": "hello"}}, "a", "b")
if err != nil || val != "hello" {
t.Error(err)
t.FailNow()
}
}

func TestParseDate_WithInvalidFormat(t *testing.T) {
_, err := parseDate("2023-06-30")
if err == nil {
t.FailNow()
}
}

func TestParseDate_WithValidFormat(t *testing.T) {
_, err := parseDate("2023-06-30T10:42:59.500")
if err != nil {
t.Error(err)
t.FailNow()
}
}

func TestConvertHitsToCSV_WithEmptyHits(t *testing.T) {
_, err := ConvertHitsToCSV([]reader.Hit{}, CSVParameters{}, true)
if err != nil {
t.Error(err)
t.FailNow()
}
}
60 changes: 32 additions & 28 deletions internals/export/utils.go
View file
Original file line number Diff line number Diff line change
@@ -1,48 +1,52 @@
package export

type CSVParameters struct {
Columns []string
ColumnsLabel []string
FormatColumnsData map[string]string
Separator rune
Limit int64
ChunkSize int64
Columns []Column `json:"columns"`
Separator rune `json:"separator" default:","`
Limit int64 `json:"limit"`
}

// Equals compares two CSVParameters
func (p CSVParameters) Equals(Params CSVParameters) bool {
if p.Separator != Params.Separator {
return false
}
if p.Limit != Params.Limit {
type Column struct {
Name string `json:"name"`
Label string `json:"label"`
Format string `json:"format" default:""`
}

// Equals compares two Column
func (p Column) Equals(column Column) bool {
if p.Name != column.Name {
return false
}
if p.ChunkSize != Params.ChunkSize {
if p.Label != column.Label {
return false
}
if len(p.Columns) != len(Params.Columns) {
if p.Format != column.Format {
return false
}
for i, column := range p.Columns {
if column != Params.Columns[i] {
return false
}
}
if len(p.ColumnsLabel) != len(Params.ColumnsLabel) {
return true
}

// Equals compares two CSVParameters
func (p CSVParameters) Equals(params CSVParameters) bool {
if p.Separator != params.Separator {
return false
}
for i, columnLabel := range p.ColumnsLabel {
if columnLabel != Params.ColumnsLabel[i] {
return false
}
}
if len(p.FormatColumnsData) != len(Params.FormatColumnsData) {
if p.Limit != params.Limit {
return false
}
for key, value := range p.FormatColumnsData {
if value != Params.FormatColumnsData[key] {
for i, column := range p.Columns {
if !column.Equals(params.Columns[i]) {
return false
}
}
return true
}

// GetColumnsLabel returns the label of the columns
func (p CSVParameters) GetColumnsLabel() []string {
columns := make([]string, 0)
for _, column := range p.Columns {
columns = append(columns, column.Label)
}
return columns
}
108 changes: 51 additions & 57 deletions internals/export/utils_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,70 +5,64 @@ import (
"testing"
)

func TestEquals(t *testing.T) {
p1 := CSVParameters{}
p2 := CSVParameters{}
expression.AssertEqual(t, p1.Equals(p2), true)

// make a full test with all variables in parameters filled
params3 := CSVParameters{
Columns: []string{"col1", "col2"},
ColumnsLabel: []string{"col1", "col2"},
FormatColumnsData: map[string]string{"col1": "format1", "col2": "format2"},
Separator: ';',
Limit: 10,
ChunkSize: 100,
}
expression.AssertEqual(t, params3.Equals(p2), false)
expression.AssertEqual(t, params3.Equals(params3), true)

// test separator
p1 = CSVParameters{Separator: ';'}
p2 = CSVParameters{Separator: ','}
expression.AssertEqual(t, p1.Equals(p2), false)
func TestColumnEquals_WithDifferentName(t *testing.T) {
column1 := Column{Name: "name1", Label: "label", Format: "format"}
column2 := Column{Name: "name2", Label: "label", Format: "format"}
expression.AssertEqual(t, column1.Equals(column2), false)
}

// test limit
p1 = CSVParameters{Limit: 10}
p2 = CSVParameters{Limit: 101}
expression.AssertEqual(t, p1.Equals(p2), false)
func TestColumnEquals_WithDifferentLabel(t *testing.T) {
column1 := Column{Name: "name", Label: "label1", Format: "format"}
column2 := Column{Name: "name", Label: "label2", Format: "format"}
expression.AssertEqual(t, column1.Equals(column2), false)
}

// test chunk size
p1 = CSVParameters{ChunkSize: 100}
p2 = CSVParameters{ChunkSize: 10}
expression.AssertEqual(t, p1.Equals(p2), false)
func TestColumnEquals_WithDifferentFormat(t *testing.T) {
column1 := Column{Name: "name", Label: "label", Format: "format1"}
column2 := Column{Name: "name", Label: "label", Format: "format2"}
expression.AssertEqual(t, column1.Equals(column2), false)
}

// test columns size
p1 = CSVParameters{Columns: []string{"col1", "col2"}}
p2 = CSVParameters{Columns: []string{"col1", "col2", "col3"}}
expression.AssertEqual(t, p1.Equals(p2), false)
func TestColumnEquals_WithSameValues(t *testing.T) {
column1 := Column{Name: "name", Label: "label", Format: "format"}
column2 := Column{Name: "name", Label: "label", Format: "format"}
expression.AssertEqual(t, column1.Equals(column2), true)
}

// test columns values
p1 = CSVParameters{Columns: []string{"col1", "col2"}}
p2 = CSVParameters{Columns: []string{"col1", "col3"}}
expression.AssertEqual(t, p1.Equals(p2), false)
func TestCSVParametersEquals_WithDifferentSeparator(t *testing.T) {
params1 := CSVParameters{Separator: ',', Limit: 10, Columns: []Column{{Name: "name", Label: "label", Format: "format"}}}
params2 := CSVParameters{Separator: ';', Limit: 10, Columns: []Column{{Name: "name", Label: "label", Format: "format"}}}
expression.AssertEqual(t, params1.Equals(params2), false)
}

// test columnsLabel size
p1 = CSVParameters{ColumnsLabel: []string{"col1", "col2"}}
p2 = CSVParameters{ColumnsLabel: []string{"col1", "col2", "col3"}}
expression.AssertEqual(t, p1.Equals(p2), false)
func TestCSVParametersEquals_WithDifferentLimit(t *testing.T) {
params1 := CSVParameters{Separator: ',', Limit: 10, Columns: []Column{{Name: "name", Label: "label", Format: "format"}}}
params2 := CSVParameters{Separator: ',', Limit: 20, Columns: []Column{{Name: "name", Label: "label", Format: "format"}}}
expression.AssertEqual(t, params1.Equals(params2), false)
}

// test columnsLabel values
p1 = CSVParameters{ColumnsLabel: []string{"col1", "col2"}}
p2 = CSVParameters{ColumnsLabel: []string{"col1", "col3"}}
expression.AssertEqual(t, p1.Equals(p2), false)
func TestCSVParametersEquals_WithDifferentColumns(t *testing.T) {
params1 := CSVParameters{Separator: ',', Limit: 10, Columns: []Column{{Name: "name1", Label: "label", Format: "format"}}}
params2 := CSVParameters{Separator: ',', Limit: 10, Columns: []Column{{Name: "name2", Label: "label", Format: "format"}}}
expression.AssertEqual(t, params1.Equals(params2), false)
}

// test formatColumnsData size
p1 = CSVParameters{FormatColumnsData: map[string]string{"col1": "format1", "col2": "format2"}}
p2 = CSVParameters{FormatColumnsData: map[string]string{"col1": "format1", "col2": "format2", "col3": "format3"}}
expression.AssertEqual(t, p1.Equals(p2), false)
func TestCSVParametersEquals_WithSameValues(t *testing.T) {
params1 := CSVParameters{Separator: ',', Limit: 10, Columns: []Column{{Name: "name", Label: "label", Format: "format"}}}
params2 := CSVParameters{Separator: ',', Limit: 10, Columns: []Column{{Name: "name", Label: "label", Format: "format"}}}
expression.AssertEqual(t, params1.Equals(params2), true)
}

// test formatColumnsData values
p1 = CSVParameters{FormatColumnsData: map[string]string{"col1": "format1", "col2": "format2"}}
p2 = CSVParameters{FormatColumnsData: map[string]string{"col1": "format1", "col2": "format3"}}
expression.AssertEqual(t, p1.Equals(p2), false)
func TestGetColumnsLabel_WithNoColumns(t *testing.T) {
params := CSVParameters{Separator: ',', Limit: 10, Columns: []Column{}}
labels := params.GetColumnsLabel()
expression.AssertEqual(t, len(labels), 0)
}

// test formatColumnsData keys
p1 = CSVParameters{FormatColumnsData: map[string]string{"col1": "format1", "col2": "format2"}}
p2 = CSVParameters{FormatColumnsData: map[string]string{"col1": "format1", "col3": "format2"}}
expression.AssertEqual(t, p1.Equals(p2), false)
func TestGetColumnsLabel_WithColumns(t *testing.T) {
params := CSVParameters{Separator: ',', Limit: 10, Columns: []Column{{Name: "name1", Label: "label1", Format: "format1"}, {Name: "name2", Label: "label2", Format: "format2"}}}
labels := params.GetColumnsLabel()
expression.AssertEqual(t, len(labels), 2)
expression.AssertEqual(t, labels[0], "label1")
expression.AssertEqual(t, labels[1], "label2")
}
Loading

0 comments on commit ea541ad

Please sign in to comment.