From ea541adaf90d11208851a843c3f7a601b6747aaa Mon Sep 17 00:00:00 2001 From: SchawnnDev Date: Wed, 29 Nov 2023 17:24:47 +0100 Subject: [PATCH] Lot of changes for export: wrapped export request --- internals/export/csv.go | 26 ++-- internals/export/csv_test.go | 82 ++++++++-- internals/export/utils.go | 60 ++++---- internals/export/utils_test.go | 108 ++++++------- internals/export/worker.go | 4 +- internals/handlers/export_handlers.go | 196 +++++++----------------- internals/handlers/utils.go | 43 ++++-- internals/handlers/utils_test.go | 17 -- internals/router/routes.go | 5 +- internals/tasker/situation_reporting.go | 74 +++++---- internals/utils/utils.go | 15 +- internals/utils/utils_test.go | 51 ++++++ 12 files changed, 361 insertions(+), 320 deletions(-) create mode 100644 internals/utils/utils_test.go diff --git a/internals/export/csv.go b/internals/export/csv.go index dd6c93cb..3cf9b644 100644 --- a/internals/export/csv.go +++ b/internals/export/csv.go @@ -11,29 +11,30 @@ import ( "go.uber.org/zap" ) -func WriteConvertHitsToCSV(w *csv.Writer, hits []reader.Hit, columns []string, columnsLabel []string, formatColumnsData map[string]string, separator rune) error { - w.Comma = separator +// WriteConvertHitsToCSV writes hits to CSV +func WriteConvertHitsToCSV(w *csv.Writer, hits []reader.Hit, params CSVParameters, writeHeader bool) error { + w.Comma = params.Separator // avoid to print header when labels are empty - if len(columnsLabel) > 0 { - w.Write(columnsLabel) + if writeHeader && len(params.Columns) > 0 { + w.Write(params.GetColumnsLabel()) } for _, hit := range hits { record := make([]string, 0) - for _, column := range columns { - value, err := nestedMapLookup(hit.Fields, strings.Split(column, ".")...) + for _, column := range params.Columns { + value, err := nestedMapLookup(hit.Fields, strings.Split(column.Name, ".")...) if err != nil { value = "" - } else if format, ok := formatColumnsData[column]; ok { + } else if column.Format != "" { if date, ok := value.(time.Time); ok { - value = date.Format(format) + value = date.Format(column.Format) } else if dateStr, ok := value.(string); ok { date, err := parseDate(dateStr) if err != nil { zap.L().Error("Failed to parse date string:", zap.Any(":", dateStr), zap.Error(err)) } else { - value = date.Format(format) + value = date.Format(column.Format) } } } @@ -46,10 +47,11 @@ func WriteConvertHitsToCSV(w *csv.Writer, hits []reader.Hit, columns []string, c return w.Error() } -func ConvertHitsToCSV(hits []reader.Hit, columns []string, columnsLabel []string, formatColumnsData map[string]string, separator rune) ([]byte, error) { +// ConvertHitsToCSV converts hits to CSV +func ConvertHitsToCSV(hits []reader.Hit, params CSVParameters, writeHeader bool) ([]byte, error) { b := new(bytes.Buffer) w := csv.NewWriter(b) - err := WriteConvertHitsToCSV(w, hits, columns, columnsLabel, formatColumnsData, separator) + err := WriteConvertHitsToCSV(w, hits, params, writeHeader) if err != nil { return nil, err @@ -58,6 +60,7 @@ func ConvertHitsToCSV(hits []reader.Hit, columns []string, columnsLabel []string return b.Bytes(), nil } +// nestedMapLookup looks up a nested map item func nestedMapLookup(m map[string]interface{}, ks ...string) (rval interface{}, err error) { var ok bool if len(ks) == 0 { @@ -74,6 +77,7 @@ func nestedMapLookup(m map[string]interface{}, ks ...string) (rval interface{}, } } +// parseDate parses a date string func parseDate(dateStr string) (time.Time, error) { formats := []string{ "2006-01-02T15:04:05.999", diff --git a/internals/export/csv_test.go b/internals/export/csv_test.go index b71c2d6f..930125eb 100644 --- a/internals/export/csv_test.go +++ b/internals/export/csv_test.go @@ -15,12 +15,17 @@ func TestConvertHitsToCSV(t *testing.T) { {ID: "3", Fields: map[string]interface{}{"a": "hello", "b": 20, "c": 3.123456, "date": "2023-06-30T10:42:59.500"}}, {ID: "1", Fields: map[string]interface{}{"a": "hello", "b": 20, "c": 3.123456, "d": map[string]interface{}{"zzz": "nested"}, "date": "2023-06-30T10:42:59.500"}}, } - columns := []string{"a", "b", "c", "d.e", "date"} - columnsLabel := []string{"Label A", "Label B", "Label C", "Label D.E", "Date"} - formatColumnsData := map[string]string{ - "date": "02/01/2006", + params := CSVParameters{ + Columns: []Column{ + {Name: "a", Label: "Label A", Format: ""}, + {Name: "b", Label: "Label B", Format: ""}, + {Name: "c", Label: "Label C", Format: ""}, + {Name: "d.e", Label: "Label D.E", Format: ""}, + {Name: "date", Label: "Date", Format: "02/01/2006"}, + }, + Separator: ',', } - csv, err := ConvertHitsToCSV(hits, columns, columnsLabel, formatColumnsData, ',') + csv, err := ConvertHitsToCSV(hits, params, true) if err != nil { t.Log(err) t.FailNow() @@ -35,17 +40,74 @@ func TestWriteConvertHitsToCSV(t *testing.T) { {ID: "3", Fields: map[string]interface{}{"a": "hello", "b": 20, "c": 3.123456, "date": "2023-06-30T10:42:59.500"}}, {ID: "1", Fields: map[string]interface{}{"a": "hello", "b": 20, "c": 3.123456, "d": map[string]interface{}{"zzz": "nested"}, "date": "2023-06-30T10:42:59.500"}}, } - columns := []string{"a", "b", "c", "d.e", "date"} - columnsLabel := []string{"Label A", "Label B", "Label C", "Label D.E", "Date"} - formatColumnsData := map[string]string{ - "date": "02/01/2006", + params := CSVParameters{ + Columns: []Column{ + {Name: "a", Label: "Label A", Format: ""}, + {Name: "b", Label: "Label B", Format: ""}, + {Name: "c", Label: "Label C", Format: ""}, + {Name: "d.e", Label: "Label D.E", Format: ""}, + {Name: "date", Label: "Date", Format: "02/01/2006"}, + }, + Separator: ',', } b := new(bytes.Buffer) w := csv2.NewWriter(b) - err := WriteConvertHitsToCSV(w, hits, columns, columnsLabel, formatColumnsData, ',') + err := WriteConvertHitsToCSV(w, hits, params, true) if err != nil { t.Log(err) t.FailNow() } t.Log("\n" + string(b.Bytes())) } + +func TestNestedMapLookup_WithEmptyKeys(t *testing.T) { + _, err := nestedMapLookup(map[string]interface{}{}, "") + if err == nil { + t.FailNow() + } +} + +func TestNestedMapLookup_WithNonExistentKey(t *testing.T) { + _, err := nestedMapLookup(map[string]interface{}{"a": "hello"}, "b") + if err == nil { + t.FailNow() + } +} + +func TestNestedMapLookup_WithNestedNonExistentKey(t *testing.T) { + _, err := nestedMapLookup(map[string]interface{}{"a": map[string]interface{}{"b": "hello"}}, "a", "c") + if err == nil { + t.FailNow() + } +} + +func TestNestedMapLookup_WithNestedKey(t *testing.T) { + val, err := nestedMapLookup(map[string]interface{}{"a": map[string]interface{}{"b": "hello"}}, "a", "b") + if err != nil || val != "hello" { + t.Error(err) + t.FailNow() + } +} + +func TestParseDate_WithInvalidFormat(t *testing.T) { + _, err := parseDate("2023-06-30") + if err == nil { + t.FailNow() + } +} + +func TestParseDate_WithValidFormat(t *testing.T) { + _, err := parseDate("2023-06-30T10:42:59.500") + if err != nil { + t.Error(err) + t.FailNow() + } +} + +func TestConvertHitsToCSV_WithEmptyHits(t *testing.T) { + _, err := ConvertHitsToCSV([]reader.Hit{}, CSVParameters{}, true) + if err != nil { + t.Error(err) + t.FailNow() + } +} diff --git a/internals/export/utils.go b/internals/export/utils.go index 3b683455..d709cc1b 100644 --- a/internals/export/utils.go +++ b/internals/export/utils.go @@ -1,48 +1,52 @@ package export type CSVParameters struct { - Columns []string - ColumnsLabel []string - FormatColumnsData map[string]string - Separator rune - Limit int64 - ChunkSize int64 + Columns []Column `json:"columns"` + Separator rune `json:"separator" default:","` + Limit int64 `json:"limit"` } -// Equals compares two CSVParameters -func (p CSVParameters) Equals(Params CSVParameters) bool { - if p.Separator != Params.Separator { - return false - } - if p.Limit != Params.Limit { +type Column struct { + Name string `json:"name"` + Label string `json:"label"` + Format string `json:"format" default:""` +} + +// Equals compares two Column +func (p Column) Equals(column Column) bool { + if p.Name != column.Name { return false } - if p.ChunkSize != Params.ChunkSize { + if p.Label != column.Label { return false } - if len(p.Columns) != len(Params.Columns) { + if p.Format != column.Format { return false } - for i, column := range p.Columns { - if column != Params.Columns[i] { - return false - } - } - if len(p.ColumnsLabel) != len(Params.ColumnsLabel) { + return true +} + +// Equals compares two CSVParameters +func (p CSVParameters) Equals(params CSVParameters) bool { + if p.Separator != params.Separator { return false } - for i, columnLabel := range p.ColumnsLabel { - if columnLabel != Params.ColumnsLabel[i] { - return false - } - } - if len(p.FormatColumnsData) != len(Params.FormatColumnsData) { + if p.Limit != params.Limit { return false } - for key, value := range p.FormatColumnsData { - if value != Params.FormatColumnsData[key] { + for i, column := range p.Columns { + if !column.Equals(params.Columns[i]) { return false } } return true } + +// GetColumnsLabel returns the label of the columns +func (p CSVParameters) GetColumnsLabel() []string { + columns := make([]string, 0) + for _, column := range p.Columns { + columns = append(columns, column.Label) + } + return columns +} diff --git a/internals/export/utils_test.go b/internals/export/utils_test.go index 18b8d143..2892c45f 100644 --- a/internals/export/utils_test.go +++ b/internals/export/utils_test.go @@ -5,70 +5,64 @@ import ( "testing" ) -func TestEquals(t *testing.T) { - p1 := CSVParameters{} - p2 := CSVParameters{} - expression.AssertEqual(t, p1.Equals(p2), true) - - // make a full test with all variables in parameters filled - params3 := CSVParameters{ - Columns: []string{"col1", "col2"}, - ColumnsLabel: []string{"col1", "col2"}, - FormatColumnsData: map[string]string{"col1": "format1", "col2": "format2"}, - Separator: ';', - Limit: 10, - ChunkSize: 100, - } - expression.AssertEqual(t, params3.Equals(p2), false) - expression.AssertEqual(t, params3.Equals(params3), true) - - // test separator - p1 = CSVParameters{Separator: ';'} - p2 = CSVParameters{Separator: ','} - expression.AssertEqual(t, p1.Equals(p2), false) +func TestColumnEquals_WithDifferentName(t *testing.T) { + column1 := Column{Name: "name1", Label: "label", Format: "format"} + column2 := Column{Name: "name2", Label: "label", Format: "format"} + expression.AssertEqual(t, column1.Equals(column2), false) +} - // test limit - p1 = CSVParameters{Limit: 10} - p2 = CSVParameters{Limit: 101} - expression.AssertEqual(t, p1.Equals(p2), false) +func TestColumnEquals_WithDifferentLabel(t *testing.T) { + column1 := Column{Name: "name", Label: "label1", Format: "format"} + column2 := Column{Name: "name", Label: "label2", Format: "format"} + expression.AssertEqual(t, column1.Equals(column2), false) +} - // test chunk size - p1 = CSVParameters{ChunkSize: 100} - p2 = CSVParameters{ChunkSize: 10} - expression.AssertEqual(t, p1.Equals(p2), false) +func TestColumnEquals_WithDifferentFormat(t *testing.T) { + column1 := Column{Name: "name", Label: "label", Format: "format1"} + column2 := Column{Name: "name", Label: "label", Format: "format2"} + expression.AssertEqual(t, column1.Equals(column2), false) +} - // test columns size - p1 = CSVParameters{Columns: []string{"col1", "col2"}} - p2 = CSVParameters{Columns: []string{"col1", "col2", "col3"}} - expression.AssertEqual(t, p1.Equals(p2), false) +func TestColumnEquals_WithSameValues(t *testing.T) { + column1 := Column{Name: "name", Label: "label", Format: "format"} + column2 := Column{Name: "name", Label: "label", Format: "format"} + expression.AssertEqual(t, column1.Equals(column2), true) +} - // test columns values - p1 = CSVParameters{Columns: []string{"col1", "col2"}} - p2 = CSVParameters{Columns: []string{"col1", "col3"}} - expression.AssertEqual(t, p1.Equals(p2), false) +func TestCSVParametersEquals_WithDifferentSeparator(t *testing.T) { + params1 := CSVParameters{Separator: ',', Limit: 10, Columns: []Column{{Name: "name", Label: "label", Format: "format"}}} + params2 := CSVParameters{Separator: ';', Limit: 10, Columns: []Column{{Name: "name", Label: "label", Format: "format"}}} + expression.AssertEqual(t, params1.Equals(params2), false) +} - // test columnsLabel size - p1 = CSVParameters{ColumnsLabel: []string{"col1", "col2"}} - p2 = CSVParameters{ColumnsLabel: []string{"col1", "col2", "col3"}} - expression.AssertEqual(t, p1.Equals(p2), false) +func TestCSVParametersEquals_WithDifferentLimit(t *testing.T) { + params1 := CSVParameters{Separator: ',', Limit: 10, Columns: []Column{{Name: "name", Label: "label", Format: "format"}}} + params2 := CSVParameters{Separator: ',', Limit: 20, Columns: []Column{{Name: "name", Label: "label", Format: "format"}}} + expression.AssertEqual(t, params1.Equals(params2), false) +} - // test columnsLabel values - p1 = CSVParameters{ColumnsLabel: []string{"col1", "col2"}} - p2 = CSVParameters{ColumnsLabel: []string{"col1", "col3"}} - expression.AssertEqual(t, p1.Equals(p2), false) +func TestCSVParametersEquals_WithDifferentColumns(t *testing.T) { + params1 := CSVParameters{Separator: ',', Limit: 10, Columns: []Column{{Name: "name1", Label: "label", Format: "format"}}} + params2 := CSVParameters{Separator: ',', Limit: 10, Columns: []Column{{Name: "name2", Label: "label", Format: "format"}}} + expression.AssertEqual(t, params1.Equals(params2), false) +} - // test formatColumnsData size - p1 = CSVParameters{FormatColumnsData: map[string]string{"col1": "format1", "col2": "format2"}} - p2 = CSVParameters{FormatColumnsData: map[string]string{"col1": "format1", "col2": "format2", "col3": "format3"}} - expression.AssertEqual(t, p1.Equals(p2), false) +func TestCSVParametersEquals_WithSameValues(t *testing.T) { + params1 := CSVParameters{Separator: ',', Limit: 10, Columns: []Column{{Name: "name", Label: "label", Format: "format"}}} + params2 := CSVParameters{Separator: ',', Limit: 10, Columns: []Column{{Name: "name", Label: "label", Format: "format"}}} + expression.AssertEqual(t, params1.Equals(params2), true) +} - // test formatColumnsData values - p1 = CSVParameters{FormatColumnsData: map[string]string{"col1": "format1", "col2": "format2"}} - p2 = CSVParameters{FormatColumnsData: map[string]string{"col1": "format1", "col2": "format3"}} - expression.AssertEqual(t, p1.Equals(p2), false) +func TestGetColumnsLabel_WithNoColumns(t *testing.T) { + params := CSVParameters{Separator: ',', Limit: 10, Columns: []Column{}} + labels := params.GetColumnsLabel() + expression.AssertEqual(t, len(labels), 0) +} - // test formatColumnsData keys - p1 = CSVParameters{FormatColumnsData: map[string]string{"col1": "format1", "col2": "format2"}} - p2 = CSVParameters{FormatColumnsData: map[string]string{"col1": "format1", "col3": "format2"}} - expression.AssertEqual(t, p1.Equals(p2), false) +func TestGetColumnsLabel_WithColumns(t *testing.T) { + params := CSVParameters{Separator: ',', Limit: 10, Columns: []Column{{Name: "name1", Label: "label1", Format: "format1"}, {Name: "name2", Label: "label2", Format: "format2"}}} + labels := params.GetColumnsLabel() + expression.AssertEqual(t, len(labels), 2) + expression.AssertEqual(t, labels[0], "label1") + expression.AssertEqual(t, labels[1], "label2") } diff --git a/internals/export/worker.go b/internals/export/worker.go index 69d3e806..6314ed4f 100644 --- a/internals/export/worker.go +++ b/internals/export/worker.go @@ -157,7 +157,6 @@ func (e *ExportWorker) Start(item WrapperItem, ctx context.Context) { // Chunk handler first := true - labels := item.Params.ColumnsLabel loop: for { @@ -167,7 +166,7 @@ loop: break loop } - err = WriteConvertHitsToCSV(csvWriter, hits, item.Params.Columns, labels, item.Params.FormatColumnsData, item.Params.Separator) + err = WriteConvertHitsToCSV(csvWriter, hits, item.Params, first) if err != nil { zap.L().Error("WriteConvertHitsToCSV error during export", zap.Error(err)) @@ -180,7 +179,6 @@ loop: if first { first = false - labels = []string{} } case <-ctx.Done(): break loop diff --git a/internals/handlers/export_handlers.go b/internals/handlers/export_handlers.go index 742053d1..f38e0d18 100644 --- a/internals/handlers/export_handlers.go +++ b/internals/handlers/export_handlers.go @@ -2,66 +2,69 @@ package handlers import ( "context" + "encoding/json" "errors" "fmt" - "github.com/myrteametrics/myrtea-sdk/v4/engine" - "net/http" - "strconv" - "strings" - "sync" - "time" - "unicode/utf8" - "github.com/go-chi/chi/v5" "github.com/myrteametrics/myrtea-engine-api/v5/internals/export" - "github.com/myrteametrics/myrtea-engine-api/v5/internals/fact" "github.com/myrteametrics/myrtea-engine-api/v5/internals/handlers/render" "github.com/myrteametrics/myrtea-engine-api/v5/internals/security/permissions" "go.uber.org/zap" + "net/http" + "strconv" + "sync" ) type ExportHandler struct { exportWrapper *export.Wrapper } +// NewExportHandler returns a new ExportHandler func NewExportHandler(exportWrapper *export.Wrapper) *ExportHandler { return &ExportHandler{ exportWrapper: exportWrapper, } } +// ExportRequest represents a request for an export +type ExportRequest struct { + export.CSVParameters + FactIDs []int64 `json:"factIDs"` + FileName string `json:"fileName"` +} + // ExportFactStreamed godoc // @Summary CSV streamed export facts in chunks // @Description CSV Streamed export for facts in chunks // @Tags ExportFactStreamed // @Produce octet-stream // @Security Bearer +// @Param request body handlers.ExportRequest true "request (json)" // @Success 200 {file} Returns data to be saved into a file // @Failure 500 "internal server error" -// @Router /engine/export/facts/{id} [get] +// @Router /engine/facts/streamedexport [get] func ExportFactStreamed(w http.ResponseWriter, r *http.Request) { - id := chi.URLParam(r, "id") - - idFact, err := strconv.ParseInt(id, 10, 64) - - if err != nil { - zap.L().Warn("Error on parsing fact id", zap.String("idFact", id), zap.Error(err)) - render.Error(w, r, render.ErrAPIParsingInteger, err) + userCtx, _ := GetUserFromContext(r) + if !userCtx.HasPermission(permissions.New(permissions.TypeExport, permissions.All, permissions.ActionGet)) { + render.Error(w, r, render.ErrAPISecurityNoPermissions, errors.New("missing permission")) return } - userCtx, _ := GetUserFromContext(r) - if !userCtx.HasPermission(permissions.New(permissions.TypeExport, strconv.FormatInt(idFact, 10), permissions.ActionGet)) { - render.Error(w, r, render.ErrAPISecurityNoPermissions, errors.New("missing permission")) + var request ExportRequest + err := json.NewDecoder(r.Body).Decode(&request) + if err != nil { + zap.L().Warn("Decode export request json", zap.Error(err)) + render.Error(w, r, render.ErrAPIDecodeJSONBody, err) return } - filename, params, combineFacts, done := handleExportArgs(w, r, err, idFact) - if done { + if len(request.FactIDs) == 0 { + zap.L().Warn("Missing factIDs in export request") + render.Error(w, r, render.ErrAPIMissingParam, errors.New("missing factIDs")) return } - err = HandleStreamedExport(r.Context(), w, combineFacts, filename, params) + err = HandleStreamedExport(r.Context(), w, request) if err != nil { render.Error(w, r, render.ErrAPIProcessError, err) } @@ -69,103 +72,19 @@ func ExportFactStreamed(w http.ResponseWriter, r *http.Request) { } -// handleExportArgs handles the export arguments and returns the filename, the parameters and the facts to export -// done is true if an error occurred and the response has already been written -func handleExportArgs(w http.ResponseWriter, r *http.Request, err error, idFact int64) (filename string, params export.CSVParameters, combineFacts []engine.Fact, done bool) { - f, found, err := fact.R().Get(idFact) - if err != nil { - zap.L().Error("Cannot retrieve fact", zap.Int64("factID", idFact), zap.Error(err)) - render.Error(w, r, render.ErrAPIDBSelectFailed, err) - return "", export.CSVParameters{}, nil, true - } - if !found { - zap.L().Warn("fact does not exist", zap.Int64("factID", idFact)) - render.Error(w, r, render.ErrAPIDBResourceNotFound, err) - return "", export.CSVParameters{}, nil, true - } - - filename = r.URL.Query().Get("fileName") - if filename == "" { - filename = fmt.Sprintf("%s_export_%s.csv", f.Name, time.Now().Format("02_01_2006")) - } else { - filename = fmt.Sprintf("%s_%s.csv", time.Now().Format("02_01_2006"), filename) - } - - // suppose that type is csv - params = GetCSVParameters(r) - - combineFacts = append(combineFacts, f) - - // export multiple facts into one file - combineFactIds, err := QueryParamToOptionalInt64Array(r, "combineFactIds", ",", false, []int64{}) - if err != nil { - zap.L().Warn("Could not parse parameter combineFactIds", zap.Error(err)) - } else { - for _, factId := range combineFactIds { - // no duplicates - if factId == idFact { - continue - } - - combineFact, found, err := fact.R().Get(factId) - if err != nil { - zap.L().Error("Export combineFact cannot retrieve fact", zap.Int64("factID", factId), zap.Error(err)) - continue - } - if !found { - zap.L().Warn("Export combineFact fact does not exist", zap.Int64("factID", factId)) - continue - } - combineFacts = append(combineFacts, combineFact) - } - } - return filename, params, combineFacts, false -} - -// GetCSVParameters returns the parameters for the CSV export -func GetCSVParameters(r *http.Request) export.CSVParameters { - result := export.CSVParameters{Separator: ','} - - limit, err := QueryParamToOptionalInt64(r, "limit", -1) - if err != nil { - result.Limit = -1 - } else { - result.Limit = limit - } - - result.Columns = QueryParamToOptionalStringArray(r, "columns", ",", []string{}) - result.ColumnsLabel = QueryParamToOptionalStringArray(r, "columnsLabel", ",", []string{}) - - formatColumnsData := QueryParamToOptionalStringArray(r, "formateColumns", ",", []string{}) - result.FormatColumnsData = make(map[string]string) - for _, formatData := range formatColumnsData { - parts := strings.Split(formatData, ";") - if len(parts) != 2 { - continue - } - key := strings.TrimSpace(parts[0]) - result.FormatColumnsData[key] = parts[1] - } - separator := r.URL.Query().Get("separator") - if separator != "" { - sep, size := utf8.DecodeRuneInString(separator) - if size != 1 { - result.Separator = ',' - } else { - result.Separator = sep - } - } - - return result -} - // HandleStreamedExport actually only handles CSV -func HandleStreamedExport(requestContext context.Context, w http.ResponseWriter, facts []engine.Fact, fileName string, params export.CSVParameters) error { +func HandleStreamedExport(requestContext context.Context, w http.ResponseWriter, request ExportRequest) error { w.Header().Set("Connection", "Keep-Alive") w.Header().Set("Transfer-Encoding", "chunked") w.Header().Set("X-Content-Type-Options", "nosniff") - w.Header().Set("Content-Disposition", "attachment; filename="+strconv.Quote(fileName)) + w.Header().Set("Content-Disposition", "attachment; filename="+strconv.Quote(request.FileName)) w.Header().Set("Content-Type", "application/octet-stream") + + facts := findCombineFacts(request.FactIDs) + if len(facts) == 0 { + return errors.New("no fact found") + } + streamedExport := export.NewStreamedExport() var wg sync.WaitGroup @@ -196,7 +115,7 @@ func HandleStreamedExport(requestContext context.Context, w http.ResponseWriter, defer close(streamedExport.Data) for _, f := range facts { - writerErr = streamedExport.StreamedExportFactHitsFull(ctx, f, params.Limit) + writerErr = streamedExport.StreamedExportFactHitsFull(ctx, f, request.Limit) if writerErr != nil { zap.L().Error("Error during export (StreamedExportFactHitsFullV8)", zap.Error(err)) break // break here when error occurs? @@ -209,7 +128,6 @@ func HandleStreamedExport(requestContext context.Context, w http.ResponseWriter, go func() { defer wg.Done() first := true - labels := params.ColumnsLabel for { select { @@ -218,7 +136,7 @@ func HandleStreamedExport(requestContext context.Context, w http.ResponseWriter, return } - data, err := export.ConvertHitsToCSV(hits, params.Columns, labels, params.FormatColumnsData, params.Separator) + data, err := export.ConvertHitsToCSV(hits, request.CSVParameters, first) if err != nil { zap.L().Error("ConvertHitsToCSV error during export (StreamedExportFactHitsFullV8)", zap.Error(err)) @@ -238,7 +156,6 @@ func HandleStreamedExport(requestContext context.Context, w http.ResponseWriter, if first { first = false - labels = []string{} } case <-requestContext.Done(): @@ -289,7 +206,7 @@ func (e *ExportHandler) GetExports(w http.ResponseWriter, r *http.Request) { // @Failure 403 "Status Forbidden: missing permission" // @Failure 404 "Status Not Found: export not found" // @Failure 500 "internal server error" -// @Router /service/exports/{id} [get] +// @Router /engine/exports/{id} [get] func (e *ExportHandler) GetExport(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "id") if id == "" { @@ -323,7 +240,7 @@ func (e *ExportHandler) GetExport(w http.ResponseWriter, r *http.Request) { // @Failure 403 "Status Forbidden: missing permission" // @Failure 404 "Status Not Found: export not found" // @Failure 500 "internal server error" -// @Router /service/exports/{id} [delete] +// @Router /engine/exports/{id} [delete] func (e *ExportHandler) DeleteExport(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "id") if id == "" { @@ -352,13 +269,7 @@ func (e *ExportHandler) DeleteExport(w http.ResponseWriter, r *http.Request) { // @Tags Exports // @Produce json // @Security Bearer -// @Param id path string true "Fact ID" -// @Param fileName query string false "File name" -// @Param limit query int false "Limit" -// @Param columns query string false "Columns" -// @Param columnsLabel query string false "Columns label" -// @Param formateColumns query string false "Formate columns" -// @Param separator query string false "Separator" +// @Param request body handlers.ExportRequest true "request (json)" // @Success 200 {object} export.WrapperItem "Status OK: user was added to existing export in queue" // @Success 201 {object} export.WrapperItem "Status Created: new export was added in queue" // @Failure 400 "Bad Request: missing fact id / fact id is not an integer" @@ -366,29 +277,36 @@ func (e *ExportHandler) DeleteExport(w http.ResponseWriter, r *http.Request) { // @Failure 409 {object} export.WrapperItem "Status Conflict: user already exists in export queue" // @Failure 429 "Status Too Many Requests: export queue is full" // @Failure 500 "internal server error" -// @Router /service/exports/fact/{id} [post] +// @Router /engine/exports/fact [post] func (e *ExportHandler) ExportFact(w http.ResponseWriter, r *http.Request) { - id := chi.URLParam(r, "id") - idFact, err := strconv.ParseInt(id, 10, 64) + userCtx, _ := GetUserFromContext(r) + if !userCtx.HasPermission(permissions.New(permissions.TypeExport, permissions.All, permissions.ActionCreate)) { + render.Error(w, r, render.ErrAPISecurityNoPermissions, errors.New("missing permission")) + return + } + var request ExportRequest + err := json.NewDecoder(r.Body).Decode(&request) if err != nil { - zap.L().Warn("Error on parsing fact id", zap.String("idFact", id), zap.Error(err)) - render.Error(w, r, render.ErrAPIParsingInteger, err) + zap.L().Warn("Decode export request json", zap.Error(err)) + render.Error(w, r, render.ErrAPIDecodeJSONBody, err) return } - userCtx, _ := GetUserFromContext(r) - if !userCtx.HasPermission(permissions.New(permissions.TypeExport, permissions.All, permissions.ActionCreate)) { - render.Error(w, r, render.ErrAPISecurityNoPermissions, errors.New("missing permission")) + if len(request.FactIDs) == 0 { + zap.L().Warn("Missing factIDs in export request") + render.Error(w, r, render.ErrAPIMissingParam, errors.New("missing factIDs")) return } - filename, params, combinedFacts, done := handleExportArgs(w, r, err, idFact) - if done { + facts := findCombineFacts(request.FactIDs) + if len(facts) == 0 { + zap.L().Warn("No fact was found in export request") + render.Error(w, r, render.ErrAPIDBResourceNotFound, errors.New("No fact was found in export request")) return } - item, status := e.exportWrapper.AddToQueue(combinedFacts, filename, params, userCtx.User) + item, status := e.exportWrapper.AddToQueue(facts, request.FileName, request.CSVParameters, userCtx.User) switch status { case export.CodeAdded: diff --git a/internals/handlers/utils.go b/internals/handlers/utils.go index a163b132..2d5fdad8 100644 --- a/internals/handlers/utils.go +++ b/internals/handlers/utils.go @@ -6,6 +6,9 @@ import ( "crypto/rand" "encoding/base64" "fmt" + "github.com/myrteametrics/myrtea-engine-api/v5/internals/fact" + "github.com/myrteametrics/myrtea-engine-api/v5/internals/utils" + "github.com/myrteametrics/myrtea-sdk/v4/engine" "io" "regexp" "strconv" @@ -31,6 +34,7 @@ const ( parseGlobalVariables = false ) +// QueryParamToOptionalInt parse a string from a string func QueryParamToOptionalInt(r *http.Request, name string, orDefault int) (int, error) { param := r.URL.Query().Get(name) if param != "" { @@ -39,6 +43,7 @@ func QueryParamToOptionalInt(r *http.Request, name string, orDefault int) (int, return orDefault, nil } +// QueryParamToOptionalInt64 parse an int64 from a string func QueryParamToOptionalInt64(r *http.Request, name string, orDefault int64) (int64, error) { param := r.URL.Query().Get(name) if param != "" { @@ -47,6 +52,7 @@ func QueryParamToOptionalInt64(r *http.Request, name string, orDefault int64) (i return orDefault, nil } +// QueryParamToOptionalInt64Array parse multiple int64 entries separated by a separator from a string func QueryParamToOptionalInt64Array(r *http.Request, name, separator string, allowDuplicates bool, orDefault []int64) ([]int64, error) { param := r.URL.Query().Get(name) if param == "" { @@ -64,7 +70,7 @@ func QueryParamToOptionalInt64Array(r *http.Request, name, separator string, all } if !allowDuplicates { - return removeDuplicate(result), nil + return utils.RemoveDuplicates(result), nil } return result, nil @@ -217,25 +223,13 @@ func GetUserFromContext(r *http.Request) (users.UserWithPermissions, bool) { return user, true } -func removeDuplicate[T string | int | int64](sliceList []T) []T { - allKeys := make(map[T]bool) - var list []T - for _, item := range sliceList { - if _, value := allKeys[item]; !value { - allKeys[item] = true - list = append(list, item) - } - } - return list -} - // handleError is a helper function that logs the error and sends a response. func handleError(w http.ResponseWriter, r *http.Request, message string, err error, apiError render.APIError) { zap.L().Error(message, zap.Error(err)) render.Error(w, r, apiError, err) } -// Generate a State use by OIDC authentification +// generateRandomState Generate a State used by OIDC authentication func generateRandomState() (string, error) { b := make([]byte, 32) _, err := rand.Read(b) @@ -244,6 +238,8 @@ func generateRandomState() (string, error) { } return base64.StdEncoding.EncodeToString(b), nil } + +// generateEncryptedState Generate a State used by OIDC authentication func generateEncryptedState(key []byte) (string, error) { // Generate random state plainState, err := generateRandomState() @@ -269,6 +265,8 @@ func generateEncryptedState(key []byte) (string, error) { b64State := base64.StdEncoding.EncodeToString(ciphertext) return b64State, nil } + +// verifyEncryptedState Verify the State used by OIDC authentication func verifyEncryptedState(state string, key []byte) (string, error) { // Decode from base64 decodedState, err := base64.StdEncoding.DecodeString(state) @@ -292,3 +290,20 @@ func verifyEncryptedState(state string, key []byte) (string, error) { return string(decodedState), nil } + +// findCombineFacts returns the combine facts +func findCombineFacts(combineFactIds []int64) (combineFacts []engine.Fact) { + for _, factId := range utils.RemoveDuplicates(combineFactIds) { + combineFact, found, err := fact.R().Get(factId) + if err != nil { + zap.L().Error("findCombineFacts cannot retrieve fact", zap.Int64("factID", factId), zap.Error(err)) + continue + } + if !found { + zap.L().Warn("findCombineFacts fact does not exist", zap.Int64("factID", factId)) + continue + } + combineFacts = append(combineFacts, combineFact) + } + return combineFacts +} diff --git a/internals/handlers/utils_test.go b/internals/handlers/utils_test.go index 401a5992..d7489f6d 100644 --- a/internals/handlers/utils_test.go +++ b/internals/handlers/utils_test.go @@ -139,23 +139,6 @@ func TestQueryParamToOptionalInt64Array(t *testing.T) { } -func TestRemoveDuplicate(t *testing.T) { - sample := []int64{1, 1, 1, 2, 2, 3, 4} - expectedResult := []int64{1, 2, 3, 4} - result := removeDuplicate(sample) - - if len(result) != len(expectedResult) { - t.FailNow() - } - - for i := 0; i < len(expectedResult); i++ { - if expectedResult[i] != result[i] { - t.FailNow() - } - } - -} - func TestHandleError(t *testing.T) { // response writer and request w := httptest.NewRecorder() diff --git a/internals/router/routes.go b/internals/router/routes.go index 1c694759..865bce4a 100644 --- a/internals/router/routes.go +++ b/internals/router/routes.go @@ -71,6 +71,7 @@ func engineRouter(services Services) http.Handler { r.Post("/facts/execute", handlers.ExecuteFactFromSource) // ?time=2019-05-10T12:00:00.000 debug= r.Get("/facts/{id}/hits", handlers.GetFactHits) // ?time=2019-05-10T12:00:00.000 debug= r.Get("/facts/{id}/es", handlers.FactToESQuery) + r.Get("/facts/streamedexport", handlers.ExportFactStreamed) r.Get("/situations", handlers.GetSituations) r.Get("/situations/{id}", handlers.GetSituation) @@ -173,13 +174,11 @@ func engineRouter(services Services) http.Handler { r.Get("/connector/{id}/executions/last", handlers.GetlastConnectorExecutionDateTime) - r.Get("/facts/{id}/streamedexport", handlers.ExportFactStreamed) - // exports r.Get("/exports", services.ExportHandler.GetExports) r.Get("/exports/{id}", services.ExportHandler.GetExport) r.Delete("/exports/{id}", services.ExportHandler.DeleteExport) - r.Post("/exports/fact/{id}", services.ExportHandler.ExportFact) + r.Post("/exports/fact", services.ExportHandler.ExportFact) r.Get("/variablesconfig", handlers.GetVariablesConfig) r.Get("/variablesconfig/{id}", handlers.GetVariableConfig) diff --git a/internals/tasker/situation_reporting.go b/internals/tasker/situation_reporting.go index 113291a3..d5a24bd1 100644 --- a/internals/tasker/situation_reporting.go +++ b/internals/tasker/situation_reporting.go @@ -30,18 +30,16 @@ func verifyCache(key string, timeout time.Duration) bool { // SituationReportingTask struct for close issues created in the current day from the BRMS type SituationReportingTask struct { - ID string `json:"id"` - IssueID string `json:"issueId"` - Subject string `json:"subject"` - BodyTemplate string `json:"bodyTemplate"` - To []string `json:"to"` - AttachmentFileNames []string `json:"attachmentFileNames"` - AttachmentFactIDs []int64 `json:"attachmentFactIds"` - Columns []string `json:"columns"` - FormatColumnsData map[string]string `json:"formateColumns"` - ColumnsLabel []string `json:"columnsLabel"` - Separator rune `json:"separator"` - Timeout string `json:"timeout"` + ID string `json:"id"` + IssueID string `json:"issueId"` + Subject string `json:"subject"` + BodyTemplate string `json:"bodyTemplate"` + To []string `json:"to"` + AttachmentFileNames []string `json:"attachmentFileNames"` + AttachmentFactIDs []int64 `json:"attachmentFactIds"` + Columns []export.Column `json:"columns"` + Separator rune `json:"separator"` + Timeout string `json:"timeout"` } func buildSituationReportingTask(parameters map[string]interface{}) (SituationReportingTask, error) { @@ -100,25 +98,43 @@ func buildSituationReportingTask(parameters map[string]interface{}) (SituationRe } if val, ok := parameters["columns"].(string); ok && val != "" { - task.Columns = strings.Split(val, ",") - } + columns := strings.Split(val, ",") + var columnsLabel []string + + if val, ok = parameters["columnsLabel"].(string); ok && val != "" { + columnsLabel = strings.Split(val, ",") + } - if val, ok := parameters["formateColumns"].(string); ok && val != "" { - formatColumnsData := strings.Split(val, ",") - task.FormatColumnsData = make(map[string]string) - for _, formatData := range formatColumnsData { - parts := strings.Split(formatData, ";") - if len(parts) != 2 { - continue + if len(columns) != len(columnsLabel) { + return task, errors.New("parameters 'columns' and 'columns label' have different length") + } + + formatColumnsDataMap := make(map[string]string) + + if val, ok = parameters["formateColumns"].(string); ok && val != "" { + formatColumnsData := strings.Split(val, ",") + for _, formatData := range formatColumnsData { + parts := strings.Split(formatData, ";") + if len(parts) != 2 { + continue + } + key := strings.TrimSpace(parts[0]) + formatColumnsDataMap[key] = parts[1] } - key := strings.TrimSpace(parts[0]) - task.FormatColumnsData[key] = parts[1] } - } + for i, column := range columns { + exportColumn := export.Column{ + Name: column, + Label: columnsLabel[i], + } + + if format, ok := formatColumnsDataMap[column]; ok { + exportColumn.Format = format + } - if val, ok := parameters["columnsLabel"].(string); ok && val != "" { - task.ColumnsLabel = strings.Split(val, ",") + task.Columns = append(task.Columns, exportColumn) + } } if val, ok := parameters["separator"].(string); ok && val != "" { @@ -127,10 +143,6 @@ func buildSituationReportingTask(parameters map[string]interface{}) (SituationRe task.Separator = ',' } - if len(task.Columns) != len(task.ColumnsLabel) { - return task, errors.New("parameters 'columns' and 'columns label' have different length") - } - if val, ok := parameters["timeout"].(string); ok && val != "" { task.Timeout = val } else { @@ -202,7 +214,7 @@ func (task SituationReportingTask) Perform(key string, context ContextData) erro return err } - csvAttachment, err := export.ConvertHitsToCSV(fullHits, task.Columns, task.ColumnsLabel, task.FormatColumnsData, task.Separator) + csvAttachment, err := export.ConvertHitsToCSV(fullHits, export.CSVParameters{Columns: task.Columns, Separator: task.Separator}, true) if err != nil { return err } diff --git a/internals/utils/utils.go b/internals/utils/utils.go index dae701c3..4cbfd750 100644 --- a/internals/utils/utils.go +++ b/internals/utils/utils.go @@ -1,12 +1,13 @@ package utils -func RemoveDuplicates(stringSlice []string) []string { - keys := make(map[string]bool) - list := []string{} - for _, entry := range stringSlice { - if _, value := keys[entry]; !value { - keys[entry] = true - list = append(list, entry) +// RemoveDuplicates remove duplicate values from a slice +func RemoveDuplicates[T string | int | int64](sliceList []T) []T { + allKeys := make(map[T]bool) + var list []T + for _, item := range sliceList { + if _, value := allKeys[item]; !value { + allKeys[item] = true + list = append(list, item) } } return list diff --git a/internals/utils/utils_test.go b/internals/utils/utils_test.go new file mode 100644 index 00000000..ab230f88 --- /dev/null +++ b/internals/utils/utils_test.go @@ -0,0 +1,51 @@ +package utils + +import "testing" + +func TestRemoveDuplicates_Int64(t *testing.T) { + sample := []int64{1, 1, 1, 2, 2, 3, 4} + expectedResult := []int64{1, 2, 3, 4} + result := RemoveDuplicates(sample) + + if len(result) != len(expectedResult) { + t.FailNow() + } + + for i := 0; i < len(expectedResult); i++ { + if expectedResult[i] != result[i] { + t.FailNow() + } + } +} + +func TestRemoveDuplicates_Int(t *testing.T) { + sample := []int{1, 1, 1, 2, 2, 3, 4} + expectedResult := []int{1, 2, 3, 4} + result := RemoveDuplicates(sample) + + if len(result) != len(expectedResult) { + t.FailNow() + } + + for i := 0; i < len(expectedResult); i++ { + if expectedResult[i] != result[i] { + t.FailNow() + } + } +} + +func TestRemoveDuplicates_String(t *testing.T) { + sample := []string{"a", "a", "a", "b", "b", "c", "d"} + expectedResult := []string{"a", "b", "c", "d"} + result := RemoveDuplicates(sample) + + if len(result) != len(expectedResult) { + t.FailNow() + } + + for i := 0; i < len(expectedResult); i++ { + if expectedResult[i] != result[i] { + t.FailNow() + } + } +}