Async export : streamed and zipped csv exports in filesystem

.github/workflows/go.yml

@@ -34,7 +34,7 @@ jobs:
 
     - name: Coverage
       uses: gwatts/go-coverage-action@v1
-      if: github.ref == 'refs/heads/master'
+    #  if: github.ref == 'refs/heads/master'
     #  continue-on-error: true
       with:
         add-comment: true

.gitignore

@@ -35,11 +35,11 @@ Gopkg.toml
 # Version file (used in CI)
 version
 
-#plugins
+# Myrtea Plugins
 /plugin/*.plugin
 
-/certs
-
-plugin/*.plugin
+# Exports directory (for local testing)
+exports/
 
+# GoLang Linter executable
 golangci-lint.exe

config/engine-api.toml

@@ -240,4 +240,33 @@ AUTHENTICATION_OIDC_FRONT_END_URL = "http://127.0.0.1:4200"
 # Note: The key length is critical for the AES encryption algorithm used for state encryption/decryption.
 # It must be exactly 16, 24 or 32 characters long.
 # Default value: "thisis24characterslongs." (24 characters)
-AUTHENTICATION_OIDC_ENCRYPTION_KEY = "thisis24characterslongs."
+AUTHENTICATION_OIDC_ENCRYPTION_KEY = "thisis24characterslongs."
+
+# NOTIFICATION_LIFETIME: The lifetime of a notification in the database.
+# Default value: "168h"
+NOTIFICATION_LIFETIME = "168h" # 168h = 7 days, available units are "ns", "us" (or "µs"), "ms", "s", "m", "h"
+
+# Path to directory where the resulting export files will be stored.
+# Default value: "exports/"
+EXPORT_BASE_PATH = "exports/"
+
+# Number of days before one export file will be auto deleted
+# Default value: 4
+EXPORT_DISK_RETENTION_DAYS = 4
+
+# Export queue max size, any export request that is made when queue is full will be refused.
+# Default value: 30
+EXPORT_QUEUE_MAX_SIZE = 30
+
+# Number of concurrent export workers
+# Default value: 4
+EXPORT_WORKERS_COUNT = 4
+
+# Whether download must be directly streamed through http or handled by an external web server
+# Default value: true
+EXPORT_DIRECT_DOWNLOAD = true
+
+# Reverse proxy like nginx, apache gives direct access to the exports directory at a specific path
+# Full URL to the exports directory
+# Default value: ""
+EXPORT_INDIRECT_DOWNLOAD_URL = ""

go.mod

@@ -4,7 +4,6 @@ go 1.20
 
 require (
 	github.com/Masterminds/squirrel v1.5.3
-	github.com/alexmullins/zip v0.0.0-20180717182244-4affb64b04d0
 	github.com/coreos/go-oidc/v3 v3.6.0
 	github.com/crewjam/saml v0.4.6
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
@@ -19,7 +18,7 @@ require (
 	github.com/gorilla/context v1.1.1
 	github.com/gorilla/websocket v1.5.0
 	github.com/hashicorp/go-hclog v1.3.1
-	github.com/hashicorp/go-plugin v1.3.0
+	github.com/hashicorp/go-plugin v1.5.2
 	github.com/jmoiron/sqlx v1.2.0
 	github.com/json-iterator/go v1.1.12
 	github.com/lestrrat-go/jwx v1.2.6
@@ -38,7 +37,7 @@ require (
 	golang.org/x/net v0.12.0
 	golang.org/x/oauth2 v0.6.0
 	google.golang.org/grpc v1.40.0
-	google.golang.org/protobuf v1.28.1
+	google.golang.org/protobuf v1.28.2-0.20230222093303-bc1253ad3743
 )
 
 require (

go.sum

@@ -43,15 +43,11 @@ github.com/PaesslerAG/gval v1.2.2 h1:Y7iBzhgE09IGTt5QgGQ2IdaYYYOU134YGHBThD+wm9E
 github.com/PaesslerAG/gval v1.2.2/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac=
 github.com/PaesslerAG/jsonpath v0.1.0 h1:gADYeifvlqK3R3i2cR5B4DGgxLXIPb3TRTH1mGi0jPI=
 github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
-github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/alexmullins/zip v0.0.0-20180717182244-4affb64b04d0 h1:BVts5dexXf4i+JX8tXlKT0aKoi38JwTXSe+3WUneX0k=
-github.com/alexmullins/zip v0.0.0-20180717182244-4affb64b04d0/go.mod h1:FDIQmoMNJJl5/k7upZEnGvgWVZfFeE6qHeN7iCMbCsA=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
@@ -62,6 +58,7 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -224,20 +221,18 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
-github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI=
 github.com/hashicorp/go-hclog v1.3.1 h1:vDwF1DFNZhntP4DAjuTpOw3uEgMUpXh1pB5fW9DqHpo=
 github.com/hashicorp/go-hclog v1.3.1/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
-github.com/hashicorp/go-plugin v1.3.0 h1:4d/wJojzvHV1I4i/rrjVaeuyxWrLzDE1mDCyDy8fXS8=
-github.com/hashicorp/go-plugin v1.3.0/go.mod h1:F9eH4LrE/ZsRdbwhfjs9k9HoDUwAHnYtXdgmf1AVNs0=
+github.com/hashicorp/go-plugin v1.5.2 h1:aWv8eimFqWlsEiMrYZdPYl+FdHaBJSN4AWwGWfT1G2Y=
+github.com/hashicorp/go-plugin v1.5.2/go.mod h1:w1sAEES3g3PuV/RzUrgow20W2uErMly84hhD3um1WL4=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M=
 github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE=
-github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
+github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c=
 github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
@@ -312,7 +307,6 @@ github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@@ -327,8 +321,6 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/myrteametrics/myrtea-sdk/v4 v4.4.5 h1:8dbIWpNLzvOq9/fQTiJUpZd56NRVAEoAmcRQoC+uD5c=
-github.com/myrteametrics/myrtea-sdk/v4 v4.4.5/go.mod h1:wa9nwNcFGpGbZeqXXqhTLp7sXERbCrRhhcASGY6H0QA=
 github.com/myrteametrics/myrtea-sdk/v4 v4.4.7 h1:cIn6+hCgzGAaWGjtAm0rFPdXX/cl6z4wWQuWI+KG9eQ=
 github.com/myrteametrics/myrtea-sdk/v4 v4.4.7/go.mod h1:wa9nwNcFGpGbZeqXXqhTLp7sXERbCrRhhcASGY6H0QA=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
@@ -494,7 +486,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
-golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -692,7 +683,6 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -725,7 +715,6 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4 h1:ysnBoUyeL/H6RCvNRhWHjKoDEmguI+mPU+qHgK8qv/w=
 google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -755,8 +744,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
-google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.28.2-0.20230222093303-bc1253ad3743 h1:yqElulDvOF26oZ2O+2/aoX7mQ8DY/6+p39neytrycd8=
+google.golang.org/protobuf v1.28.2-0.20230222093303-bc1253ad3743/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internals/app/app.go

@@ -5,7 +5,7 @@ import (
 	"github.com/spf13/viper"
 )
 
-// Init initialiaze all the app configuration and components
+// Init initialize all the app configuration and components
 func Init() {
 
 	docs.SwaggerInfo.Host = viper.GetString("SWAGGER_HOST")

internals/app/services.go

@@ -2,6 +2,7 @@ package app
 
 import (
 	"errors"
+	"github.com/myrteametrics/myrtea-engine-api/v5/internals/export"
 	"strings"
 
 	"github.com/myrteametrics/myrtea-engine-api/v5/internals/calendar"
@@ -78,6 +79,11 @@ func stopServices() {
 }
 
 func initNotifier() {
+	notificationLifetime := viper.GetDuration("NOTIFICATION_LIFETIME")
+	handler := notification.NewHandler(notificationLifetime)
+	handler.RegisterNotificationType(notification.MockNotification{})
+	handler.RegisterNotificationType(export.ExportNotification{})
+	notification.ReplaceHandlerGlobals(handler)
 	notifier.ReplaceGlobals(notifier.NewNotifier())
 }
 
@@ -90,7 +96,6 @@ func initScheduler() {
 		if viper.GetBool("ENABLE_CRONS_ON_START") {
 			scheduler.S().C.Start()
 		}
-
 	}
 }
 func initTasker() {
@@ -100,7 +105,6 @@ func initTasker() {
 
 func initCalendars() {
 	calendar.Init()
-
 }
 
 func initCoordinator() {
@@ -114,7 +118,7 @@ func initCoordinator() {
 
 	instanceName := viper.GetString("INSTANCE_NAME")
 	if err = coordinator.InitInstance(instanceName, models); err != nil {
-		zap.L().Fatal("Intialisation of coordinator master", zap.Error(err))
+		zap.L().Fatal("Initialization of coordinator master", zap.Error(err))
 	}
 	if viper.GetBool("ENABLE_CRONS_ON_START") {
 		for _, li := range coordinator.GetInstance().LogicalIndices {
@@ -127,13 +131,11 @@ func initCoordinator() {
 }
 
 func initEmailSender() {
-
 	username := viper.GetString("SMTP_USERNAME")
 	password := viper.GetString("SMTP_PASSWORD")
 	host := viper.GetString("SMTP_HOST")
 	port := viper.GetString("SMTP_PORT")
 	email.InitSender(username, password, host, port)
-
 }
 
 func initOidcAuthentication() {

internals/export/csv.go

@@ -6,36 +6,43 @@ import (
 	"fmt"
 	"strings"
 	"time"
+	"unicode/utf8"
 
 	"github.com/myrteametrics/myrtea-engine-api/v5/internals/reader"
 	"go.uber.org/zap"
 )
 
-func ConvertHitsToCSV(hits []reader.Hit, columns []string, columnsLabel []string, formatColumnsData map[string]string, separator rune) ([]byte, error) {
-	b := new(bytes.Buffer)
-	w := csv.NewWriter(b)
-	w.Comma = separator
+// WriteConvertHitsToCSV writes hits to CSV
+func WriteConvertHitsToCSV(w *csv.Writer, hits []reader.Hit, params CSVParameters, writeHeader bool) error {
+	if len(params.Separator) == 1 {
+		w.Comma, _ = utf8.DecodeRune([]byte(params.Separator))
+		if w.Comma == utf8.RuneError {
+			w.Comma = ','
+		}
+	} else {
+		w.Comma = ','
+	}
 
 	// avoid to print header when labels are empty
-	if len(columnsLabel) > 0 {
-		w.Write(columnsLabel)
+	if writeHeader && len(params.Columns) > 0 {
+		w.Write(params.GetColumnsLabel())
 	}
 
 	for _, hit := range hits {
 		record := make([]string, 0)
-		for _, column := range columns {
-			value, err := nestedMapLookup(hit.Fields, strings.Split(column, ".")...)
+		for _, column := range params.Columns {
+			value, err := nestedMapLookup(hit.Fields, strings.Split(column.Name, ".")...)
 			if err != nil {
 				value = ""
-			} else if format, ok := formatColumnsData[column]; ok {
+			} else if column.Format != "" {
 				if date, ok := value.(time.Time); ok {
-					value = date.Format(format)
+					value = date.Format(column.Format)
 				} else if dateStr, ok := value.(string); ok {
 					date, err := parseDate(dateStr)
 					if err != nil {
 						zap.L().Error("Failed to parse date string:", zap.Any(":", dateStr), zap.Error(err))
 					} else {
-						value = date.Format(format)
+						value = date.Format(column.Format)
 					}
 				}
 			}
@@ -45,12 +52,23 @@ func ConvertHitsToCSV(hits []reader.Hit, columns []string, columnsLabel []string
 	}
 
 	w.Flush()
-	if err := w.Error(); err != nil {
+	return w.Error()
+}
+
+// ConvertHitsToCSV converts hits to CSV
+func ConvertHitsToCSV(hits []reader.Hit, params CSVParameters, writeHeader bool) ([]byte, error) {
+	b := new(bytes.Buffer)
+	w := csv.NewWriter(b)
+	err := WriteConvertHitsToCSV(w, hits, params, writeHeader)
+
+	if err != nil {
 		return nil, err
 	}
+
 	return b.Bytes(), nil
 }
 
+// nestedMapLookup looks up a nested map item
 func nestedMapLookup(m map[string]interface{}, ks ...string) (rval interface{}, err error) {
 	var ok bool
 	if len(ks) == 0 {
@@ -67,6 +85,7 @@ func nestedMapLookup(m map[string]interface{}, ks ...string) (rval interface{},
 	}
 }
 
+// parseDate parses a date string
 func parseDate(dateStr string) (time.Time, error) {
 	formats := []string{
 		"2006-01-02T15:04:05.999",