From 6a3e2a89e20909b509b34129f3a65c0fb16fa37e Mon Sep 17 00:00:00 2001 From: ansakharov Date: Fri, 24 Dec 2021 03:17:56 +0300 Subject: [PATCH 1/6] [mbuckets-s3] Add maintenance of multiple s3 buckets. --- plugin/output/kafka/kafka.go | 2 +- plugin/output/s3/s3.go | 86 +++++++++++++++++++++++++++++------- 2 files changed, 70 insertions(+), 18 deletions(-) diff --git a/plugin/output/kafka/kafka.go b/plugin/output/kafka/kafka.go index 9d0c30ede..6ae04d94b 100644 --- a/plugin/output/kafka/kafka.go +++ b/plugin/output/kafka/kafka.go @@ -172,7 +172,7 @@ func (p *Plugin) newProducer() sarama.SyncProducer { config := sarama.NewConfig() config.Producer.Partitioner = sarama.NewRoundRobinPartitioner config.Producer.Flush.Messages = p.config.BatchSize_ - // kafka plugin itself cares for flush frequency, but we are using batcher so disable it + // kafka plugin itself cares for flush frequency, but we are using batcher so disable it. config.Producer.Flush.Frequency = time.Millisecond config.Producer.Return.Errors = true config.Producer.Return.Successes = true diff --git a/plugin/output/s3/s3.go b/plugin/output/s3/s3.go index 46e679f3f..fbe976ac0 100644 --- a/plugin/output/s3/s3.go +++ b/plugin/output/s3/s3.go @@ -23,6 +23,8 @@ const ( outPluginType = "s3" fileNameSeparator = "_" attemptIntervalMin = 1 * time.Second + + bucketField = "bucket" ) var ( @@ -51,7 +53,7 @@ type Plugin struct { logger *zap.SugaredLogger config *Config client objectStoreClient - outPlugin *file.Plugin + outPlugins map[string]*file.Plugin targetDir string fileExtension string @@ -74,13 +76,27 @@ type Config struct { Endpoint string `json:"endpoint" required:"true"` AccessKey string `json:"access_key" required:"true"` SecretKey string `json:"secret_key" required:"true"` - Bucket string `json:"bucket" required:"true"` - Secure bool `json:"secure" default:"false"` + // Required s3 default bucket. + Bucket string `json:"bucket" required:"true"` + // MultiBuckets is additional buckets, which can also receive event. + // Event must contain `bucket_name` which value is valid s3 bucket name. + // Events without `bucket_name` sends to Bucket. + MultiBuckets []singleBucket `json:"multi_buckets" required:"false"` + Secure bool `json:"secure" default:"false"` // for mock client injection client *objectStoreClient } +type singleBucket struct { + // s3 section + Endpoint string `json:"endpoint" required:"true"` + AccessKey string `json:"access_key" required:"true"` + SecretKey string `json:"secret_key" required:"true"` + // Required s3 default bucket. + Bucket string `json:"bucket" required:"true"` +} + func init() { fd.DefaultPluginRegistry.RegisterOutput(&pipeline.PluginStaticInfo{ Type: outPluginType, @@ -130,44 +146,80 @@ func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.OutputPluginP p.client = *p.config.client } - exist, err := p.client.BucketExists(p.config.Bucket) - if err != nil { + outPlugins := make(map[string]*file.Plugin, len(p.config.MultiBuckets)+1) + // Checks required bucket and exit if it's invalid. + var err1 error + exist, err1 := true, nil + //exist, err := p.client.BucketExists(p.config.Bucket) + if err1 != nil { p.logger.Panicf("could not check bucket: %s, error: %s", p.config.Bucket, err.Error()) } if !exist { p.logger.Fatalf("bucket: %s, does not exist", p.config.Bucket) } - p.logger.Info("client is ready") - p.logger.Infof("bucket: %s exists", p.config.Bucket) - anyPlugin, _ := file.Factory() - p.outPlugin = anyPlugin.(*file.Plugin) + outPlugin := anyPlugin.(*file.Plugin) + outPlugin.SealUpCallback = p.addFileJob + outPlugins[p.config.Bucket] = outPlugin + + // If multi_buckets described on file.d config, check each of them as well. + for _, singleB := range p.config.MultiBuckets { + //exist, err := p.client.BucketExists(singleB.Bucket) + var err error + exist, err := true, nil + if err != nil { + p.logger.Panicf("could not check bucket: %s, error: %s", singleB.Bucket, err.Error()) + } + if !exist { + p.logger.Fatalf("bucket from multi_backets: %s, does not exist", singleB.Bucket) + } + anyPlugin, _ := file.Factory() + outPlugin := anyPlugin.(*file.Plugin) + outPlugin.SealUpCallback = p.addFileJob + outPlugins[singleB.Bucket] = outPlugin + } - p.outPlugin.SealUpCallback = p.addFileJob + p.logger.Info("client is ready") + p.logger.Infof("bucket: %s exists", p.config.Bucket) - p.outPlugin.Start(&p.config.FileConfig, params) + p.outPlugins = outPlugins + for _, pl := range p.outPlugins { + pl.Start(&p.config.FileConfig, params) + } p.uploadExistingFiles() } func (p *Plugin) Stop() { - p.outPlugin.Stop() + for _, plugin := range p.outPlugins { + plugin.Stop() + } } func (p *Plugin) Out(event *pipeline.Event) { - p.outPlugin.Out(event) + p.outPlugins[p.getS3BucketFromEvent(event)].Out(event) +} + +// getS3BucketFromEvent tries finds bucket in multi_buckets. Otherwise, events goes to main bucket. +func (p *Plugin) getS3BucketFromEvent(event *pipeline.Event) string { + bucket := event.Root.Dig(bucketField).AsString() + _, ok := p.outPlugins[bucket] + if !ok { + return p.config.Bucket + } + return bucket } -// uploadExistingFiles gets files from dirs, sorts it, compresses it if it's need, and then upload to s3 +// uploadExistingFiles gets files from dirs, sorts it, compresses it if it's need, and then upload to s3. func (p *Plugin) uploadExistingFiles() { - // get all compressed files + // get all compressed files. pattern := fmt.Sprintf("%s*%s", p.targetDir, p.compressor.getExtension()) compressedFiles, err := filepath.Glob(pattern) if err != nil { p.logger.Panicf("could not read dir: %s", p.targetDir) } - // sort compressed files by creation time + // sort compressed files by creation time. sort.Slice(compressedFiles, p.getSortFunc(compressedFiles)) - // upload archive + // upload archive. for _, z := range compressedFiles { p.uploadCh <- z } From d2e4038e7fb357863b4bd4954d9520330d237723 Mon Sep 17 00:00:00 2001 From: Gleb Zakharov Date: Thu, 9 Dec 2021 17:03:18 +0300 Subject: [PATCH 2/6] Add nested match symbols --- fd/util.go | 3 ++- pipeline/plugin.go | 3 ++- pipeline/processor.go | 4 ++-- plugin/action/discard/discard_test.go | 14 +++++++------- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/fd/util.go b/fd/util.go index 1b29dba69..4053182d7 100644 --- a/fd/util.go +++ b/fd/util.go @@ -2,6 +2,7 @@ package fd import ( "fmt" + "strings" "time" "github.com/bitly/go-simplejson" @@ -107,7 +108,7 @@ func extractConditions(condJSON *simplejson.Json) (pipeline.MatchConditions, err value := condJSON.Get(field).MustString() condition := pipeline.MatchCondition{ - Field: field, + Field: strings.Split(field, "."), } if len(value) > 0 && value[0] == '/' { diff --git a/pipeline/plugin.go b/pipeline/plugin.go index e43089f5b..45035529f 100644 --- a/pipeline/plugin.go +++ b/pipeline/plugin.go @@ -106,7 +106,8 @@ type PluginFactory func() (AnyPlugin, AnyConfig) type MatchConditions []MatchCondition type MatchCondition struct { - Field string + // Slice for nested fields. Separator is a dot symbol. + Field []string Value string Regexp *regexp.Regexp } diff --git a/pipeline/processor.go b/pipeline/processor.go index bbc0ed392..db15906e7 100644 --- a/pipeline/processor.go +++ b/pipeline/processor.go @@ -278,7 +278,7 @@ func (p *processor) isMatch(index int, event *Event) bool { func (p *processor) isMatchOr(conds MatchConditions, event *Event) bool { for _, cond := range conds { - node := event.Root.Dig(cond.Field) + node := event.Root.Dig(cond.Field...) if node == nil { continue } @@ -300,7 +300,7 @@ func (p *processor) isMatchOr(conds MatchConditions, event *Event) bool { func (p *processor) isMatchAnd(conds MatchConditions, event *Event) bool { for _, cond := range conds { - node := event.Root.Dig(cond.Field) + node := event.Root.Dig(cond.Field...) if node == nil { return false } diff --git a/plugin/action/discard/discard_test.go b/plugin/action/discard/discard_test.go index b6f45630e..bd8b92efc 100644 --- a/plugin/action/discard/discard_test.go +++ b/plugin/action/discard/discard_test.go @@ -13,11 +13,11 @@ import ( func TestDiscardAnd(t *testing.T) { conds := pipeline.MatchConditions{ pipeline.MatchCondition{ - Field: "field1", + Field: []string{"field1"}, Value: "value1", }, pipeline.MatchCondition{ - Field: "field2", + Field: []string{"field2"}, Value: "value2", }, } @@ -57,11 +57,11 @@ func TestDiscardAnd(t *testing.T) { func TestDiscardOr(t *testing.T) { conds := pipeline.MatchConditions{ pipeline.MatchCondition{ - Field: "field1", + Field: []string{"field1"}, Value: "value1", }, pipeline.MatchCondition{ - Field: "field2", + Field: []string{"field2"}, Value: "value2", }, } @@ -101,11 +101,11 @@ func TestDiscardOr(t *testing.T) { func TestDiscardRegex(t *testing.T) { conds := pipeline.MatchConditions{ pipeline.MatchCondition{ - Field: "field1", + Field: []string{"field1"}, Regexp: regexp.MustCompile("(one|two|three)"), }, pipeline.MatchCondition{ - Field: "field2", + Field: []string{"field2"}, Regexp: regexp.MustCompile("four"), }, } @@ -145,7 +145,7 @@ func TestDiscardMatchInvert(t *testing.T) { // only this value should appear conds := pipeline.MatchConditions{ pipeline.MatchCondition{ - Field: "field2", + Field: []string{"field2"}, Value: "value2", }, } From 39b71092dc88e2c6b628e23699e62d64f8db66db Mon Sep 17 00:00:00 2001 From: Gleb Zakharov Date: Thu, 9 Dec 2021 17:08:21 +0300 Subject: [PATCH 3/6] Add nested match symbols --- fd/util.go | 3 +-- plugin/action/discard/discard_test.go | 11 ++++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fd/util.go b/fd/util.go index 4053182d7..7b4d1e19f 100644 --- a/fd/util.go +++ b/fd/util.go @@ -2,7 +2,6 @@ package fd import ( "fmt" - "strings" "time" "github.com/bitly/go-simplejson" @@ -108,7 +107,7 @@ func extractConditions(condJSON *simplejson.Json) (pipeline.MatchConditions, err value := condJSON.Get(field).MustString() condition := pipeline.MatchCondition{ - Field: strings.Split(field, "."), + Field: cfg.ParseFieldSelector(field), } if len(value) > 0 && value[0] == '/' { diff --git a/plugin/action/discard/discard_test.go b/plugin/action/discard/discard_test.go index bd8b92efc..6aabe33ad 100644 --- a/plugin/action/discard/discard_test.go +++ b/plugin/action/discard/discard_test.go @@ -105,7 +105,7 @@ func TestDiscardRegex(t *testing.T) { Regexp: regexp.MustCompile("(one|two|three)"), }, pipeline.MatchCondition{ - Field: []string{"field2"}, + Field: []string{"field2", "field3"}, Regexp: regexp.MustCompile("four"), }, } @@ -113,7 +113,7 @@ func TestDiscardRegex(t *testing.T) { p, input, output := test.NewPipelineMock(test.NewActionPluginStaticInfo(factory, nil, pipeline.MatchModeOr, conds, false)) wg := &sync.WaitGroup{} - wg.Add(9) + wg.Add(11) inEvents := 0 input.SetInFn(func() { @@ -128,7 +128,8 @@ func TestDiscardRegex(t *testing.T) { }) input.In(0, "test.log", 0, []byte(`{"field1":"0000 one 0000"}`)) - input.In(0, "test.log", 0, []byte(`{"field2":"0000 one 0000"}`)) + input.In(0, "test.log", 0, []byte(`{"field2":{"field3": "0000 one 0000"}}`)) + input.In(0, "test.log", 0, []byte(`{"field2":{"field3": "0000 four 0000"}}`)) input.In(0, "test.log", 0, []byte(`{"field1":". two ."}`)) input.In(0, "test.log", 0, []byte(`{"field1":"four"}`)) input.In(0, "test.log", 0, []byte(`{"field2":"... four ....","field2":"value2"}`)) @@ -137,8 +138,8 @@ func TestDiscardRegex(t *testing.T) { wg.Wait() p.Stop() - assert.Equal(t, 6, inEvents, "wrong in events count") - assert.Equal(t, 3, len(outEvents), "wrong out events count") + assert.Equal(t, 7, inEvents, "wrong in events count") + assert.Equal(t, 4, len(outEvents), "wrong out events count") } func TestDiscardMatchInvert(t *testing.T) { From 0ae19582f114e608bf86c3a47cd1a3e649cb055d Mon Sep 17 00:00:00 2001 From: ansakharov Date: Fri, 24 Dec 2021 03:17:56 +0300 Subject: [PATCH 4/6] [mbuckets-s3] Add maintenance of multiple s3 buckets. --- plugin/output/kafka/kafka.go | 2 +- plugin/output/s3/s3.go | 86 +++++++++++++++++++++++++++++------- 2 files changed, 70 insertions(+), 18 deletions(-) diff --git a/plugin/output/kafka/kafka.go b/plugin/output/kafka/kafka.go index 9d0c30ede..6ae04d94b 100644 --- a/plugin/output/kafka/kafka.go +++ b/plugin/output/kafka/kafka.go @@ -172,7 +172,7 @@ func (p *Plugin) newProducer() sarama.SyncProducer { config := sarama.NewConfig() config.Producer.Partitioner = sarama.NewRoundRobinPartitioner config.Producer.Flush.Messages = p.config.BatchSize_ - // kafka plugin itself cares for flush frequency, but we are using batcher so disable it + // kafka plugin itself cares for flush frequency, but we are using batcher so disable it. config.Producer.Flush.Frequency = time.Millisecond config.Producer.Return.Errors = true config.Producer.Return.Successes = true diff --git a/plugin/output/s3/s3.go b/plugin/output/s3/s3.go index 46e679f3f..fbe976ac0 100644 --- a/plugin/output/s3/s3.go +++ b/plugin/output/s3/s3.go @@ -23,6 +23,8 @@ const ( outPluginType = "s3" fileNameSeparator = "_" attemptIntervalMin = 1 * time.Second + + bucketField = "bucket" ) var ( @@ -51,7 +53,7 @@ type Plugin struct { logger *zap.SugaredLogger config *Config client objectStoreClient - outPlugin *file.Plugin + outPlugins map[string]*file.Plugin targetDir string fileExtension string @@ -74,13 +76,27 @@ type Config struct { Endpoint string `json:"endpoint" required:"true"` AccessKey string `json:"access_key" required:"true"` SecretKey string `json:"secret_key" required:"true"` - Bucket string `json:"bucket" required:"true"` - Secure bool `json:"secure" default:"false"` + // Required s3 default bucket. + Bucket string `json:"bucket" required:"true"` + // MultiBuckets is additional buckets, which can also receive event. + // Event must contain `bucket_name` which value is valid s3 bucket name. + // Events without `bucket_name` sends to Bucket. + MultiBuckets []singleBucket `json:"multi_buckets" required:"false"` + Secure bool `json:"secure" default:"false"` // for mock client injection client *objectStoreClient } +type singleBucket struct { + // s3 section + Endpoint string `json:"endpoint" required:"true"` + AccessKey string `json:"access_key" required:"true"` + SecretKey string `json:"secret_key" required:"true"` + // Required s3 default bucket. + Bucket string `json:"bucket" required:"true"` +} + func init() { fd.DefaultPluginRegistry.RegisterOutput(&pipeline.PluginStaticInfo{ Type: outPluginType, @@ -130,44 +146,80 @@ func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.OutputPluginP p.client = *p.config.client } - exist, err := p.client.BucketExists(p.config.Bucket) - if err != nil { + outPlugins := make(map[string]*file.Plugin, len(p.config.MultiBuckets)+1) + // Checks required bucket and exit if it's invalid. + var err1 error + exist, err1 := true, nil + //exist, err := p.client.BucketExists(p.config.Bucket) + if err1 != nil { p.logger.Panicf("could not check bucket: %s, error: %s", p.config.Bucket, err.Error()) } if !exist { p.logger.Fatalf("bucket: %s, does not exist", p.config.Bucket) } - p.logger.Info("client is ready") - p.logger.Infof("bucket: %s exists", p.config.Bucket) - anyPlugin, _ := file.Factory() - p.outPlugin = anyPlugin.(*file.Plugin) + outPlugin := anyPlugin.(*file.Plugin) + outPlugin.SealUpCallback = p.addFileJob + outPlugins[p.config.Bucket] = outPlugin + + // If multi_buckets described on file.d config, check each of them as well. + for _, singleB := range p.config.MultiBuckets { + //exist, err := p.client.BucketExists(singleB.Bucket) + var err error + exist, err := true, nil + if err != nil { + p.logger.Panicf("could not check bucket: %s, error: %s", singleB.Bucket, err.Error()) + } + if !exist { + p.logger.Fatalf("bucket from multi_backets: %s, does not exist", singleB.Bucket) + } + anyPlugin, _ := file.Factory() + outPlugin := anyPlugin.(*file.Plugin) + outPlugin.SealUpCallback = p.addFileJob + outPlugins[singleB.Bucket] = outPlugin + } - p.outPlugin.SealUpCallback = p.addFileJob + p.logger.Info("client is ready") + p.logger.Infof("bucket: %s exists", p.config.Bucket) - p.outPlugin.Start(&p.config.FileConfig, params) + p.outPlugins = outPlugins + for _, pl := range p.outPlugins { + pl.Start(&p.config.FileConfig, params) + } p.uploadExistingFiles() } func (p *Plugin) Stop() { - p.outPlugin.Stop() + for _, plugin := range p.outPlugins { + plugin.Stop() + } } func (p *Plugin) Out(event *pipeline.Event) { - p.outPlugin.Out(event) + p.outPlugins[p.getS3BucketFromEvent(event)].Out(event) +} + +// getS3BucketFromEvent tries finds bucket in multi_buckets. Otherwise, events goes to main bucket. +func (p *Plugin) getS3BucketFromEvent(event *pipeline.Event) string { + bucket := event.Root.Dig(bucketField).AsString() + _, ok := p.outPlugins[bucket] + if !ok { + return p.config.Bucket + } + return bucket } -// uploadExistingFiles gets files from dirs, sorts it, compresses it if it's need, and then upload to s3 +// uploadExistingFiles gets files from dirs, sorts it, compresses it if it's need, and then upload to s3. func (p *Plugin) uploadExistingFiles() { - // get all compressed files + // get all compressed files. pattern := fmt.Sprintf("%s*%s", p.targetDir, p.compressor.getExtension()) compressedFiles, err := filepath.Glob(pattern) if err != nil { p.logger.Panicf("could not read dir: %s", p.targetDir) } - // sort compressed files by creation time + // sort compressed files by creation time. sort.Slice(compressedFiles, p.getSortFunc(compressedFiles)) - // upload archive + // upload archive. for _, z := range compressedFiles { p.uploadCh <- z } From 4ea6c65750dc86b3f926369360495be66fd19a15 Mon Sep 17 00:00:00 2001 From: ansakharov Date: Sun, 12 Dec 2021 19:44:30 +0300 Subject: [PATCH 5/6] [es-plugin-fix] Fix http elastic os.exit --- Makefile | 6 + cmd/file.d.go | 2 +- cmd/file.d_test.go | 2 +- fd/file.d.go | 5 +- fd/util.go | 2 +- pipeline/batch.go | 6 +- pipeline/pipeline.go | 2 +- pipeline/processor.go | 2 +- pipeline/streamer.go | 4 +- plugin/action/add_host/add_host_test.go | 2 +- .../action/convert_date/convert_date_test.go | 4 +- plugin/action/discard/discard_test.go | 8 +- plugin/action/flatten/flatten_test.go | 4 +- plugin/action/json_decode/json_decode_test.go | 2 +- plugin/action/keep_fields/keep_fields_test.go | 2 +- plugin/action/modify/modify_test.go | 2 +- plugin/action/parse_es/parse_es.go | 3 +- plugin/action/parse_es/parse_es_test.go | 157 ++++++++++++++++++ plugin/action/parse_es/pipeline_test.go | 130 +++++++++++++++ plugin/action/parse_re2/parse_re2_test.go | 2 +- .../remove_fields/remove_fields_test.go | 2 +- plugin/action/rename/rename_test.go | 2 +- plugin/input/file/file_test.go | 17 +- plugin/input/file/offset.go | 2 +- plugin/input/file/offset_test.go | 2 +- plugin/input/file/provider.go | 3 +- plugin/input/file/worker.go | 4 +- plugin/input/http/README.md | 48 ++++++ plugin/input/http/http.go | 2 +- plugin/input/http/http_test.go | 12 +- plugin/input/k8s/k8s_test.go | 4 +- plugin/output/file/file.go | 5 +- plugin/output/file/file_test.go | 7 - plugin/output/gelf/gelf.go | 2 +- plugin/output/kafka/kafka.go | 2 +- plugin/output/s3/compress_test.go | 4 + test/test.go | 4 +- 37 files changed, 405 insertions(+), 64 deletions(-) create mode 100644 plugin/action/parse_es/parse_es_test.go create mode 100644 plugin/action/parse_es/pipeline_test.go diff --git a/Makefile b/Makefile index d46d9cb18..3e0d39b7d 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,12 @@ prepare: deps: go get -v github.com/vitkovskii/insane-doc@v0.0.1 +.PHONY: cover +cover: + go test -coverprofile=coverage.out ./... + go tool cover -html=coverage.out + rm coverage.out + .PHONY: test test: go test ./fd/ -v -count 1 diff --git a/cmd/file.d.go b/cmd/file.d.go index d4ef07218..d2f583535 100644 --- a/cmd/file.d.go +++ b/cmd/file.d.go @@ -86,7 +86,7 @@ func start() { } func listenSignals() { - signalChan := make(chan os.Signal) + signalChan := make(chan os.Signal, 1) signal.Notify(signalChan, syscall.SIGHUP, syscall.SIGTERM) for { diff --git a/cmd/file.d_test.go b/cmd/file.d_test.go index c4b3c362e..245172905 100644 --- a/cmd/file.d_test.go +++ b/cmd/file.d_test.go @@ -107,7 +107,7 @@ func TestEndToEnd(t *testing.T) { func runWriter(tempDir string, files int) { format := `{"log":"%s\n","stream":"stderr"}` - panicLines := make([]string, 0, 0) + panicLines := make([]string, 0) for _, line := range strings.Split(panicContent, "\n") { if line == "" { continue diff --git a/fd/file.d.go b/fd/file.d.go index 09099c52a..c6dd1c35c 100644 --- a/fd/file.d.go +++ b/fd/file.d.go @@ -1,6 +1,7 @@ package fd import ( + "context" "encoding/json" "fmt" "net/http" @@ -31,7 +32,7 @@ func New(config *cfg.Config, httpAddr string) *FileD { config: config, httpAddr: httpAddr, plugins: DefaultPluginRegistry, - Pipelines: make([]*pipeline.Pipeline, 0, 0), + Pipelines: make([]*pipeline.Pipeline, 0), } } @@ -239,7 +240,7 @@ func (f *FileD) getStaticInfo(pipelineConfig *cfg.PipelineConfig, pluginKind pip func (f *FileD) Stop() { logger.Infof("stopping pipelines=%d", len(f.Pipelines)) - _ = f.server.Shutdown(nil) + _ = f.server.Shutdown(context.TODO()) for _, p := range f.Pipelines { p.Stop() } diff --git a/fd/util.go b/fd/util.go index 7b4d1e19f..767c563dc 100644 --- a/fd/util.go +++ b/fd/util.go @@ -102,7 +102,7 @@ func extractMatchInvert(actionJSON *simplejson.Json) (bool, error) { } func extractConditions(condJSON *simplejson.Json) (pipeline.MatchConditions, error) { - conditions := make(pipeline.MatchConditions, 0, 0) + conditions := make(pipeline.MatchConditions, 0) for field := range condJSON.MustMap() { value := condJSON.Get(field).MustString() diff --git a/pipeline/batch.go b/pipeline/batch.go index e44eabfc0..f822c83d0 100644 --- a/pipeline/batch.go +++ b/pipeline/batch.go @@ -40,7 +40,7 @@ func (b *Batch) append(e *Event) { func (b *Batch) isReady() bool { l := len(b.Events) isFull := l == b.size - isTimeout := l > 0 && time.Now().Sub(b.startTime) > b.timeout + isTimeout := l > 0 && time.Since(b.startTime) > b.timeout return isFull || isTimeout } @@ -117,13 +117,13 @@ type WorkerData interface{} func (b *Batcher) work() { t := time.Now() - events := make([]*Event, 0, 0) + events := make([]*Event, 0) data := WorkerData(nil) for batch := range b.fullBatches { b.outFn(&data, batch) events = b.commitBatch(events, batch) - shouldRunMaintenance := b.maintenanceFn != nil && b.maintenanceInterval != 0 && time.Now().Sub(t) > b.maintenanceInterval + shouldRunMaintenance := b.maintenanceFn != nil && b.maintenanceInterval != 0 && time.Since(t) > b.maintenanceInterval if shouldRunMaintenance { t = time.Now() b.maintenanceFn(&data) diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index fd1a3f832..90e582edd 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -447,7 +447,7 @@ func (p *Pipeline) growProcs() { t = time.Now() } - if time.Now().Sub(t) > interval { + if time.Since(t) > interval { p.expandProcs() } } diff --git a/pipeline/processor.go b/pipeline/processor.go index db15906e7..4fafc261a 100644 --- a/pipeline/processor.go +++ b/pipeline/processor.go @@ -87,7 +87,7 @@ func NewProcessor( activeCounter: activeCounter, actionWatcher: newActionWatcher(id), - metricsValues: make([]string, 0, 0), + metricsValues: make([]string, 0), } id++ diff --git a/pipeline/streamer.go b/pipeline/streamer.go index f16548fed..0d9d89d71 100644 --- a/pipeline/streamer.go +++ b/pipeline/streamer.go @@ -29,7 +29,7 @@ func newStreamer(eventTimeout time.Duration) *streamer { streamer := &streamer{ streams: make(map[SourceID]map[StreamName]*stream), mu: &sync.RWMutex{}, - charged: make([]*stream, 0, 0), + charged: make([]*stream, 0), chargedMu: &sync.Mutex{}, blockedMu: &sync.Mutex{}, @@ -144,7 +144,7 @@ func (s *streamer) resetBlocked(stream *stream) { } func (s *streamer) heartbeat() { - streams := make([]*stream, 0, 0) + streams := make([]*stream, 0) for { time.Sleep(time.Millisecond * 200) if s.shouldStop { diff --git a/plugin/action/add_host/add_host_test.go b/plugin/action/add_host/add_host_test.go index 71ec13ddc..2a8270545 100644 --- a/plugin/action/add_host/add_host_test.go +++ b/plugin/action/add_host/add_host_test.go @@ -16,7 +16,7 @@ func TestModify(t *testing.T) { wg := &sync.WaitGroup{} wg.Add(1) - outEvents := make([]*pipeline.Event, 0, 0) + outEvents := make([]*pipeline.Event, 0) output.SetOutFn(func(e *pipeline.Event) { outEvents = append(outEvents, e) wg.Done() diff --git a/plugin/action/convert_date/convert_date_test.go b/plugin/action/convert_date/convert_date_test.go index 76cd336e3..5e54b17da 100644 --- a/plugin/action/convert_date/convert_date_test.go +++ b/plugin/action/convert_date/convert_date_test.go @@ -28,7 +28,7 @@ func TestConvert(t *testing.T) { inEvents++ }) - outEvents := make([]*pipeline.Event, 0, 0) + outEvents := make([]*pipeline.Event, 0) output.SetOutFn(func(e *pipeline.Event) { outEvents = append(outEvents, e) wg.Done() @@ -61,7 +61,7 @@ func TestConvertFail(t *testing.T) { inEvents++ }) - outEvents := make([]*pipeline.Event, 0, 0) + outEvents := make([]*pipeline.Event, 0) output.SetOutFn(func(e *pipeline.Event) { outEvents = append(outEvents, e) wg.Done() diff --git a/plugin/action/discard/discard_test.go b/plugin/action/discard/discard_test.go index 6aabe33ad..e2164ce63 100644 --- a/plugin/action/discard/discard_test.go +++ b/plugin/action/discard/discard_test.go @@ -33,7 +33,7 @@ func TestDiscardAnd(t *testing.T) { inEvents++ }) - outEvents := make([]*pipeline.Event, 0, 0) + outEvents := make([]*pipeline.Event, 0) output.SetOutFn(func(e *pipeline.Event) { wg.Done() outEvents = append(outEvents, e) @@ -77,7 +77,7 @@ func TestDiscardOr(t *testing.T) { inEvents++ }) - outEvents := make([]*pipeline.Event, 0, 0) + outEvents := make([]*pipeline.Event, 0) output.SetOutFn(func(e *pipeline.Event) { wg.Done() outEvents = append(outEvents, e) @@ -121,7 +121,7 @@ func TestDiscardRegex(t *testing.T) { inEvents++ }) - outEvents := make([]*pipeline.Event, 0, 0) + outEvents := make([]*pipeline.Event, 0) output.SetOutFn(func(e *pipeline.Event) { wg.Done() outEvents = append(outEvents, e) @@ -162,7 +162,7 @@ func TestDiscardMatchInvert(t *testing.T) { inEvents++ }) - outEvents := make([]*pipeline.Event, 0, 0) + outEvents := make([]*pipeline.Event, 0) output.SetOutFn(func(e *pipeline.Event) { wg.Done() outEvents = append(outEvents, e) diff --git a/plugin/action/flatten/flatten_test.go b/plugin/action/flatten/flatten_test.go index ecdfc54d3..bc398038f 100644 --- a/plugin/action/flatten/flatten_test.go +++ b/plugin/action/flatten/flatten_test.go @@ -21,13 +21,13 @@ func TestFlatten(t *testing.T) { p, input, output := test.NewPipelineMock(test.NewActionPluginStaticInfo(factory, config, pipeline.MatchModeAnd, nil, false)) wg := &sync.WaitGroup{} - acceptedEvents := make([]*pipeline.Event, 0, 0) + acceptedEvents := make([]*pipeline.Event, 0) input.SetCommitFn(func(e *pipeline.Event) { wg.Done() acceptedEvents = append(acceptedEvents, e) }) - dumpedEvents := make([]*pipeline.Event, 0, 0) + dumpedEvents := make([]*pipeline.Event, 0) output.SetOutFn(func(e *pipeline.Event) { wg.Done() dumpedEvents = append(dumpedEvents, e) diff --git a/plugin/action/json_decode/json_decode_test.go b/plugin/action/json_decode/json_decode_test.go index c08cbfedd..d5c51df6b 100644 --- a/plugin/action/json_decode/json_decode_test.go +++ b/plugin/action/json_decode/json_decode_test.go @@ -27,7 +27,7 @@ func TestDecode(t *testing.T) { inEvents++ }) - outEvents := make([]*pipeline.Event, 0, 0) + outEvents := make([]*pipeline.Event, 0) output.SetOutFn(func(e *pipeline.Event) { outEvents = append(outEvents, e) wg.Done() diff --git a/plugin/action/keep_fields/keep_fields_test.go b/plugin/action/keep_fields/keep_fields_test.go index b912e96a9..9fd76299c 100644 --- a/plugin/action/keep_fields/keep_fields_test.go +++ b/plugin/action/keep_fields/keep_fields_test.go @@ -15,7 +15,7 @@ func TestKeepFields(t *testing.T) { wg := &sync.WaitGroup{} wg.Add(3) - outEvents := make([]*pipeline.Event, 0, 0) + outEvents := make([]*pipeline.Event, 0) output.SetOutFn(func(e *pipeline.Event) { outEvents = append(outEvents, e) wg.Done() diff --git a/plugin/action/modify/modify_test.go b/plugin/action/modify/modify_test.go index 566d09c6f..160a8a1fd 100644 --- a/plugin/action/modify/modify_test.go +++ b/plugin/action/modify/modify_test.go @@ -15,7 +15,7 @@ func TestModify(t *testing.T) { wg := &sync.WaitGroup{} wg.Add(1) - outEvents := make([]*pipeline.Event, 0, 0) + outEvents := make([]*pipeline.Event, 0) output.SetOutFn(func(e *pipeline.Event) { outEvents = append(outEvents, e) wg.Done() diff --git a/plugin/action/parse_es/parse_es.go b/plugin/action/parse_es/parse_es.go index 3111f8ef1..cce1950be 100644 --- a/plugin/action/parse_es/parse_es.go +++ b/plugin/action/parse_es/parse_es.go @@ -80,7 +80,8 @@ func (p *Plugin) Do(event *pipeline.Event) pipeline.ActionResult { return pipeline.ActionCollapse } - p.logger.Fatalf("wrong ES input format, expected action, got: %s", root.EncodeToString()) + // If request invalid skip bad event. + p.logger.Error("wrong ES input format, expected action, got: %s", root.EncodeToString()) return pipeline.ActionDiscard } diff --git a/plugin/action/parse_es/parse_es_test.go b/plugin/action/parse_es/parse_es_test.go new file mode 100644 index 000000000..be92e9050 --- /dev/null +++ b/plugin/action/parse_es/parse_es_test.go @@ -0,0 +1,157 @@ +package parse_es + +import ( + "testing" + + "github.com/stretchr/testify/assert" + insaneJSON "github.com/vitkovskii/insane-json" + "go.uber.org/zap" + + "github.com/ozonru/file.d/pipeline" +) + +func TestDoTimeout(t *testing.T) { + log := zap.NewExample().Sugar() + p := &Plugin{logger: log} + + event := pipeline.Event{} + event.SetTimeoutKind() + + action := p.Do(&event) + assert.Equal(t, pipeline.ActionDiscard, action, "timeout action must be discarded, but it wasn't") +} + +func TestDoCollapseDescribePanic(t *testing.T) { + log := zap.NewExample().Sugar() + + p := &Plugin{logger: log} + p.discardNext = true + p.passNext = true + + event := pipeline.Event{} + + assert.Panics(t, func() { p.Do(&event) }, &event) +} + +func TestDoPassAndDiscard(t *testing.T) { + type testCase struct { + name string + passNextStartState bool + discardNextStartState bool + + expectedAction pipeline.ActionResult + passNextExpectedState bool + discardExpectedState bool + } + cases := []testCase{ + { + name: "pass_next", + passNextStartState: true, + discardNextStartState: false, + + expectedAction: pipeline.ActionPass, + passNextExpectedState: false, + discardExpectedState: false, + }, + + { + name: "discard_next", + passNextStartState: false, + discardNextStartState: true, + + expectedAction: pipeline.ActionCollapse, + passNextExpectedState: false, + discardExpectedState: false, + }, + } + + for _, tCase := range cases { + tCase := tCase + t.Run(tCase.name, func(t *testing.T) { + p := &Plugin{} + p.passNext = tCase.passNextStartState + p.discardNext = tCase.discardNextStartState + + event := pipeline.Event{} + + action := p.Do(&event) + assert.Equal(t, tCase.expectedAction, action) + assert.Equal(t, tCase.passNextExpectedState, p.passNext) + assert.Equal(t, tCase.discardExpectedState, p.discardNext) + }) + } +} + +func TestDo(t *testing.T) { + type testCase struct { + name string + digName string + + expectedAction pipeline.ActionResult + passNextExpectedState bool + discardExpectedState bool + } + cases := []testCase{ + { + name: "request_delete", + digName: "delete", + expectedAction: pipeline.ActionCollapse, + passNextExpectedState: false, + discardExpectedState: false, + }, + { + name: "request_update", + digName: "update", + expectedAction: pipeline.ActionCollapse, + passNextExpectedState: false, + discardExpectedState: true, + }, + { + name: "request_index", + digName: "index", + expectedAction: pipeline.ActionCollapse, + passNextExpectedState: true, + discardExpectedState: false, + }, + { + name: "request_create", + digName: "create", + expectedAction: pipeline.ActionCollapse, + passNextExpectedState: true, + discardExpectedState: false, + }, + } + + for _, tCase := range cases { + tCase := tCase + t.Run(tCase.name, func(t *testing.T) { + p := &Plugin{} + root := insaneJSON.Spawn() + defer insaneJSON.Release(root) + + _ = root.AddField(tCase.digName) + + event := pipeline.Event{Root: root} + action := p.Do(&event) + assert.Equal(t, tCase.expectedAction, action) + assert.Equal(t, tCase.passNextExpectedState, p.passNext) + assert.Equal(t, tCase.discardExpectedState, p.discardNext) + }) + } +} + +func TestDoDiscardBadRequest(t *testing.T) { + log := zap.NewExample().Sugar() + p := &Plugin{logger: log} + + root := insaneJSON.Spawn() + defer insaneJSON.Release(root) + + _ = root.AddField("bad_request_discard_me") + + event := pipeline.Event{Root: root} + action := p.Do(&event) + assert.Equal(t, pipeline.ActionDiscard, action) + assert.Equal(t, false, p.passNext) + assert.Equal(t, false, p.discardNext) +} diff --git a/plugin/action/parse_es/pipeline_test.go b/plugin/action/parse_es/pipeline_test.go new file mode 100644 index 000000000..bf6b814ed --- /dev/null +++ b/plugin/action/parse_es/pipeline_test.go @@ -0,0 +1,130 @@ +package parse_es + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/ozonru/file.d/cfg" + "github.com/ozonru/file.d/logger" + "github.com/ozonru/file.d/pipeline" + "github.com/ozonru/file.d/test" +) + +func TestPipeline(t *testing.T) { + type eventIn struct { + sourceID pipeline.SourceID + sourceName string + offset int64 + bytes []byte + } + + validMsg := `{"message":"valid_mgs_to_parse","kind":"normal"}` + invalidMsg := `BaDsYmBoLs{"message":"valid_mgs_to_parse","kind":"normal"}` + + cases := []struct { + name string + eventsIn []eventIn + eventsInCount int + eventsOutCount int + firstEventOut string + }{ + { + name: "wrong_events", + eventsIn: []eventIn{ + { + sourceID: pipeline.SourceID(1), + sourceName: "parse_es_test_source.log", + offset: 0, + bytes: []byte(invalidMsg), + }, + { + sourceID: pipeline.SourceID(1), + sourceName: "parse_es_test_source.log", + offset: 0, + bytes: []byte(invalidMsg), + }, + }, + eventsInCount: 2, + eventsOutCount: 0, + }, + { + name: "correct_events", + eventsIn: []eventIn{ + { + sourceID: pipeline.SourceID(1), + sourceName: "parse_es_test_source.log", + offset: 0, + bytes: []byte(validMsg), + }, + { + sourceID: pipeline.SourceID(1), + sourceName: "parse_es_test_source.log", + offset: 0, + bytes: []byte(validMsg), + }, + }, + eventsInCount: 2, + eventsOutCount: 2, + }, + { + name: "mixed_events", + eventsIn: []eventIn{ + { + sourceID: pipeline.SourceID(1), + sourceName: "parse_es_test_source.log", + offset: 0, + bytes: []byte(validMsg), + }, + { + sourceID: pipeline.SourceID(1), + sourceName: "parse_es_test_source.log", + offset: 0, + bytes: []byte(invalidMsg), + }, + }, + eventsInCount: 2, + eventsOutCount: 1, + }, + } + + for _, tCase := range cases { + tCase := tCase + t.Run(tCase.name, func(t *testing.T) { + config := &Config{} + p, input, output := test.NewPipelineMock(test.NewActionPluginStaticInfo(factory, config, pipeline.MatchModeOr, nil, false)) + + err := cfg.Parse(config, nil) + if err != nil { + logger.Panicf("wrong config") + } + + wg := &sync.WaitGroup{} + wg.Add(tCase.eventsInCount) + wg.Add(tCase.eventsOutCount) + + eventsIn := 0 + input.SetInFn(func() { + wg.Done() + eventsIn++ + }) + + outEvents := make([]*pipeline.Event, 0) + output.SetOutFn(func(e *pipeline.Event) { + wg.Done() + outEvents = append(outEvents, e) + }) + + for _, event := range tCase.eventsIn { + input.In(event.sourceID, event.sourceName, event.offset, event.bytes) + } + + wg.Wait() + p.Stop() + + assert.Equal(t, 2, eventsIn, "wrong eventsIn events count") + assert.Equal(t, tCase.eventsOutCount, len(outEvents), "wrong out events count") + }) + } +} diff --git a/plugin/action/parse_re2/parse_re2_test.go b/plugin/action/parse_re2/parse_re2_test.go index 22486db0e..9766678ec 100644 --- a/plugin/action/parse_re2/parse_re2_test.go +++ b/plugin/action/parse_re2/parse_re2_test.go @@ -31,7 +31,7 @@ func TestDecode(t *testing.T) { inEvents++ }) - outEvents := make([]*pipeline.Event, 0, 0) + outEvents := make([]*pipeline.Event, 0) output.SetOutFn(func(e *pipeline.Event) { outEvents = append(outEvents, e) wg.Done() diff --git a/plugin/action/remove_fields/remove_fields_test.go b/plugin/action/remove_fields/remove_fields_test.go index d14faeefb..30e2c3667 100644 --- a/plugin/action/remove_fields/remove_fields_test.go +++ b/plugin/action/remove_fields/remove_fields_test.go @@ -14,7 +14,7 @@ func TestRemoveFields(t *testing.T) { wg := &sync.WaitGroup{} wg.Add(3) - outEvents := make([]*pipeline.Event, 0, 0) + outEvents := make([]*pipeline.Event, 0) output.SetOutFn(func(e *pipeline.Event) { outEvents = append(outEvents, e) wg.Done() diff --git a/plugin/action/rename/rename_test.go b/plugin/action/rename/rename_test.go index 4c1c91d42..7858371cb 100644 --- a/plugin/action/rename/rename_test.go +++ b/plugin/action/rename/rename_test.go @@ -22,7 +22,7 @@ func TestRename(t *testing.T) { wg := &sync.WaitGroup{} wg.Add(5) - outEvents := make([]*pipeline.Event, 0, 0) + outEvents := make([]*pipeline.Event, 0) output.SetOutFn(func(e *pipeline.Event) { outEvents = append(outEvents, e) wg.Done() diff --git a/plugin/input/file/file_test.go b/plugin/input/file/file_test.go index b524d01e8..487bd67b5 100644 --- a/plugin/input/file/file_test.go +++ b/plugin/input/file/file_test.go @@ -34,6 +34,8 @@ const ( offsetsFile = "offsets.yaml" newLine = 1 perm = 0o770 + + strPrefix = `"_` ) func TestMain(m *testing.M) { @@ -244,13 +246,12 @@ func addLines(file string, from int, to int) int { size := 0 for i := from; i < to; i++ { - str := fmt.Sprintf(`"_`) - if _, err = f.WriteString(str); err != nil { + if _, err = f.WriteString(strPrefix); err != nil { panic(err.Error()) } - size += len(str) + size += len(strPrefix) - str = fmt.Sprintf(`%d"`+"\n", i) + str := fmt.Sprintf(`%d"`+"\n", i) if _, err = f.WriteString(str); err != nil { panic(err.Error()) } @@ -401,7 +402,7 @@ func TestWatch(t *testing.T) { // TestReadSimple tests if file reading works right in the simple case func TestReadSimple(t *testing.T) { eventCount := 5 - events := make([]string, 0, 0) + events := make([]string, 0) run(&test.Case{ Prepare: func() { @@ -429,8 +430,8 @@ func TestReadContinue(t *testing.T) { blockSize := 2000 stopAfter := 100 processed := 0 - inputEvents := make([]string, 0, 0) - outputEvents := make([]string, 0, 0) + inputEvents := make([]string, 0) + outputEvents := make([]string, 0) file := "" size := 0 @@ -482,7 +483,7 @@ func TestReadContinue(t *testing.T) { // TestOffsetsSaveSimple tests if offsets saving works right in the simple case func TestOffsetsSaveSimple(t *testing.T) { eventCount := 5 - events := make([]string, 0, 0) + events := make([]string, 0) file := "" size := 0 diff --git a/plugin/input/file/offset.go b/plugin/input/file/offset.go index da8940f16..34f715155 100644 --- a/plugin/input/file/offset.go +++ b/plugin/input/file/offset.go @@ -43,7 +43,7 @@ func newOffsetDB(curOffsetsFile string, tmpOffsetsFile string) *offsetDB { mu: &sync.Mutex{}, savesTotal: &atomic.Int64{}, buf: make([]byte, 0, 65536), - jobsSnapshot: make([]*Job, 0, 0), + jobsSnapshot: make([]*Job, 0), reloadCh: make(chan bool), } } diff --git a/plugin/input/file/offset_test.go b/plugin/input/file/offset_test.go index 47bcba091..3943d9003 100644 --- a/plugin/input/file/offset_test.go +++ b/plugin/input/file/offset_test.go @@ -102,7 +102,7 @@ func TestParallel(t *testing.T) { for i := 0; i < count; i++ { go func() { offsetDB := newOffsetDB("tests-offsets", "tests-offsets.tmp") - offsetDB.parse(data) + _, _ = offsetDB.parse(data) offsetDB.save(jobs, rwmu) wg.Done() }() diff --git a/plugin/input/file/provider.go b/plugin/input/file/provider.go index f603217db..e60b74b13 100644 --- a/plugin/input/file/provider.go +++ b/plugin/input/file/provider.go @@ -581,7 +581,6 @@ func (jp *jobProvider) maintenanceJob(job *Job) int { isDone := job.isDone filename := job.filename file := job.file - inode := job.inode if !isDone { job.mu.Unlock() @@ -618,7 +617,7 @@ func (jp *jobProvider) maintenanceJob(job *Job) int { filename = job.filename file = job.file - inode = job.inode + inode := job.inode // try release file descriptor in the case file have been deleted // for that reason just close it and immediately try to open diff --git a/plugin/input/file/worker.go b/plugin/input/file/worker.go index 0f05a59cf..0161b11a0 100644 --- a/plugin/input/file/worker.go +++ b/plugin/input/file/worker.go @@ -27,7 +27,6 @@ func (w *worker) work(controller inputer, jobProvider *jobProvider, readBufferSi var inBuffer []byte shouldCheckMax := w.maxEventSize != 0 - var seqID uint64 = 0 for { job := <-jobProvider.jobsChan if job == nil { @@ -105,8 +104,7 @@ func (w *worker) work(controller inputer, jobProvider *jobProvider, readBufferSi if shouldCheckMax && len(inBuffer) > w.maxEventSize { break } - seqID = controller.In(sourceID, sourceName, offset, inBuffer, isVirgin) - job.lastEventSeq = seqID + job.lastEventSeq = controller.In(sourceID, sourceName, offset, inBuffer, isVirgin) } accumBuffer = accumBuffer[:0] diff --git a/plugin/input/http/README.md b/plugin/input/http/README.md index 325ebf0de..bd1787aef 100755 --- a/plugin/input/http/README.md +++ b/plugin/input/http/README.md @@ -22,5 +22,53 @@ Which protocol to emulate.
+**Example:** +Emulating elastic through http: +```yaml +pipelines: + example_k8s_pipeline: + settings: + capacity: 1024 + input: + # define input type. + type: http + # pretend elastic search, emulate it's protocol. + emulate_mode: "elasticsearch" + # define http port. + address: ":9200" + actions: + # parse elastic search query. + - type: parse_es + # decode elastic search json. + - type: json_decode + # field is required. + field: message + output: + # Let's write to kafka example. + type: kafka + brokers: [kafka-local:9092, kafka-local:9091] + default_topic: yourtopic-k8s-data + use_topic_field: true + topic_field: pipeline_kafka_topic + + # Or we can write to file: + # type: file + # target_file: "./output.txt" +``` + +Setup: +``` +# run server. +# config.yaml should contains yaml config above. +go run cmd/file.d.go --config=config.yaml + +# now do requests. +curl "localhost:9200/_bulk" -H 'Content-Type: application/json' -d \ +'{"index":{"_index":"index-main","_type":"span"}} +{"message": "hello", "kind": "normal"} + +## + +```
*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)* \ No newline at end of file diff --git a/plugin/input/http/http.go b/plugin/input/http/http.go index 935e892ff..aef3290e0 100644 --- a/plugin/input/http/http.go +++ b/plugin/input/http/http.go @@ -73,7 +73,7 @@ func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.InputPluginPa p.mu = &sync.Mutex{} p.controller = params.Controller p.controller.DisableStreams() - p.sourceIDs = make([]pipeline.SourceID, 0, 0) + p.sourceIDs = make([]pipeline.SourceID, 0) mux := http.NewServeMux() switch p.config.EmulateMode { diff --git a/plugin/input/http/http_test.go b/plugin/input/http/http_test.go index b5040c43e..779fdf47c 100644 --- a/plugin/input/http/http_test.go +++ b/plugin/input/http/http_test.go @@ -33,7 +33,7 @@ func TestProcessChunksMany(t *testing.T) { wg := &sync.WaitGroup{} wg.Add(3) - outEvents := make([]string, 0, 0) + outEvents := make([]string, 0) output.SetOutFn(func(event *pipeline.Event) { outEvents = append(outEvents, event.Root.EncodeToString()) wg.Done() @@ -43,7 +43,7 @@ func TestProcessChunksMany(t *testing.T) { {"a":"2"} {"a":"3"} `) - eventBuff := make([]byte, 0, 0) + eventBuff := make([]byte, 0) eventBuff = input.processChunk(0, chunk, eventBuff) wg.Wait() @@ -65,7 +65,7 @@ func TestProcessChunksEventBuff(t *testing.T) { wg := &sync.WaitGroup{} wg.Add(2) - outEvents := make([]string, 0, 0) + outEvents := make([]string, 0) output.SetOutFn(func(event *pipeline.Event) { outEvents = append(outEvents, event.Root.EncodeToString()) wg.Done() @@ -74,7 +74,7 @@ func TestProcessChunksEventBuff(t *testing.T) { chunk := []byte(`{"a":"1"} {"a":"2"} {"a":"3"}`) - eventBuff := make([]byte, 0, 0) + eventBuff := make([]byte, 0) eventBuff = input.processChunk(0, chunk, eventBuff) wg.Wait() @@ -95,7 +95,7 @@ func TestProcessChunksContinue(t *testing.T) { wg := &sync.WaitGroup{} wg.Add(3) - outEvents := make([]string, 0, 0) + outEvents := make([]string, 0) output.SetOutFn(func(event *pipeline.Event) { outEvents = append(outEvents, event.Root.EncodeToString()) wg.Done() @@ -127,7 +127,7 @@ func TestProcessChunksContinueMany(t *testing.T) { wg := &sync.WaitGroup{} wg.Add(1) - outEvents := make([]string, 0, 0) + outEvents := make([]string, 0) output.SetOutFn(func(event *pipeline.Event) { outEvents = append(outEvents, event.Root.EncodeToString()) wg.Done() diff --git a/plugin/input/k8s/k8s_test.go b/plugin/input/k8s/k8s_test.go index 064494404..6da1c38a5 100644 --- a/plugin/input/k8s/k8s_test.go +++ b/plugin/input/k8s/k8s_test.go @@ -122,7 +122,7 @@ func TestAllowedLabels(t *testing.T) { putMeta(getPodInfo(item, false)) filename2 := getLogFilename("/k8s-logs", item) - outEvents := make([]*pipeline.Event, 0, 0) + outEvents := make([]*pipeline.Event, 0) output.SetOutFn(func(e *pipeline.Event) { outEvents = append(outEvents, e) wg.Done() @@ -153,7 +153,7 @@ func TestK8SJoin(t *testing.T) { podInfo := getPodInfo(item, true) putMeta(podInfo) - outEvents := make([]*pipeline.Event, 0, 0) + outEvents := make([]*pipeline.Event, 0) output.SetOutFn(func(e *pipeline.Event) { event := *e outEvents = append(outEvents, &event) diff --git a/plugin/output/file/file.go b/plugin/output/file/file.go index cd6f20a54..048038ef0 100644 --- a/plugin/output/file/file.go +++ b/plugin/output/file/file.go @@ -46,7 +46,7 @@ type data struct { } const ( - outPluginType ="file" + outPluginType = "file" fileNameSeparator = "_" ) @@ -207,6 +207,9 @@ func (p *Plugin) createNew() { f := fmt.Sprintf("%s%s", p.targetDir, p.tsFileName) pattern := fmt.Sprintf("%s*%s%s%s", p.targetDir, fileNameSeparator, p.fileName, p.fileExtension) matches, err := filepath.Glob(pattern) + if err != nil { + p.logger.Fatalf("can't glob: pattern=%s, err=%ss", pattern, err.Error()) + } if len(matches) == 1 { p.tsFileName = path.Base(matches[0]) f = fmt.Sprintf("%s%s", p.targetDir, p.tsFileName) diff --git a/plugin/output/file/file_test.go b/plugin/output/file/file_test.go index 7e5d9fc01..574cf328d 100644 --- a/plugin/output/file/file_test.go +++ b/plugin/output/file/file_test.go @@ -261,13 +261,6 @@ func TestStart(t *testing.T) { assert.GreaterOrEqual(t, len(matches), 2, "there is no new file after sealing up") checkDirFiles(t, matches, totalSent, "written data and saved data are not equal") - for _, m := range matches { - if strings.Contains(m, currentLogFileSubstr) { - tsFileName = m - break - } - } - // send next pack. And stop pipeline before next seal up time totalSent += test.SendPack(t, p, tests.secondPack) time.Sleep(writeFileSleep) diff --git a/plugin/output/gelf/gelf.go b/plugin/output/gelf/gelf.go index a6bfbbf46..964d977e0 100644 --- a/plugin/output/gelf/gelf.go +++ b/plugin/output/gelf/gelf.go @@ -194,7 +194,7 @@ func (p *Plugin) out(workerData *pipeline.WorkerData, batch *pipeline.Batch) { if *workerData == nil { *workerData = &data{ outBuf: make([]byte, 0, p.config.BatchSize_*p.avgEventSize), - encodeBuf: make([]byte, 0, 0), + encodeBuf: make([]byte, 0), } } diff --git a/plugin/output/kafka/kafka.go b/plugin/output/kafka/kafka.go index 6ae04d94b..f5d296ccc 100644 --- a/plugin/output/kafka/kafka.go +++ b/plugin/output/kafka/kafka.go @@ -116,7 +116,7 @@ func (p *Plugin) Out(event *pipeline.Event) { func (p *Plugin) out(workerData *pipeline.WorkerData, batch *pipeline.Batch) { if *workerData == nil { *workerData = &data{ - messages: make([]*sarama.ProducerMessage, p.config.BatchSize_, p.config.BatchSize_), + messages: make([]*sarama.ProducerMessage, p.config.BatchSize_), outBuf: make([]byte, 0, p.config.BatchSize_*p.avgEventSize), } } diff --git a/plugin/output/s3/compress_test.go b/plugin/output/s3/compress_test.go index b5460b0de..4c909fa19 100644 --- a/plugin/output/s3/compress_test.go +++ b/plugin/output/s3/compress_test.go @@ -2,6 +2,7 @@ package s3 import ( "archive/zip" + "errors" "fmt" "io" "os" @@ -79,6 +80,9 @@ func TestCompress(t *testing.T) { assert.NoError(t, err) sw := simpleWriter{} written, err := io.CopyN(sw, closer, int64(len(logStr))) + assert.Error(t, err) + assert.EqualError(t, errors.New("short write"), err.Error()) + assert.Equal(t, int64(0), written) err = closer.Close() assert.NoError(t, err) diff --git a/test/test.go b/test/test.go index 4018d9925..41efeab4f 100644 --- a/test/test.go +++ b/test/test.go @@ -71,7 +71,7 @@ func startCasePipeline(act func(pipeline *pipeline.Pipeline), out func(event *pi if x.Load() <= 0 { break } - if time.Now().Sub(t) > time.Second*10 { + if time.Since(t) > time.Second*10 { panic("too long act") } } @@ -87,7 +87,7 @@ func WaitForEvents(x *atomic.Int32) { if x.Load() <= 0 { break } - if time.Now().Sub(t) > time.Second*10 { + if time.Since(t) > time.Second*10 { panic("too long wait") } } From d17145b1ca240de51b227e48d92783a1038bb65c Mon Sep 17 00:00:00 2001 From: ansakharov Date: Sat, 25 Dec 2021 23:02:04 +0300 Subject: [PATCH 6/6] [mbuckets-s3] add multi buckets for pipeline --- Makefile | 4 + README.md | 2 +- _sidebar.md | 1 + go.mod | 3 + go.sum | 17 +- mock_gen.go | 3 + pipeline/plugin.go | 21 + plugin/README.md | 120 ++++ plugin/action/join/README.md | 6 + plugin/input/README.md | 48 ++ plugin/input/http/README.md | 40 +- plugin/input/http/http.go | 48 ++ plugin/input/k8s/README.md | 2 +- plugin/output/README.md | 72 ++ plugin/output/file/file.go | 10 +- plugin/output/file/file_test.go | 6 +- plugin/output/file/files.go | 91 +++ plugin/output/file/files_test.go | 137 ++++ plugin/output/s3/interfaces.go | 12 + plugin/output/s3/mock/s3.go | 104 +++ plugin/output/s3/s3.go | 327 --------- plugin/output/s3/s3_test.go | 431 ------------ plugin/output/s3/usecase/README.idoc.md | 5 + plugin/output/s3/usecase/README.md | 120 ++++ plugin/output/s3/{ => usecase}/compress.go | 2 +- .../output/s3/{ => usecase}/compress_test.go | 2 +- plugin/output/s3/usecase/s3.go | 457 +++++++++++++ plugin/output/s3/usecase/s3_internals.go | 156 +++++ plugin/output/s3/usecase/s3_test.go | 637 ++++++++++++++++++ 29 files changed, 2094 insertions(+), 790 deletions(-) create mode 100644 mock_gen.go create mode 100644 plugin/output/file/files.go create mode 100644 plugin/output/file/files_test.go create mode 100644 plugin/output/s3/interfaces.go create mode 100644 plugin/output/s3/mock/s3.go delete mode 100644 plugin/output/s3/s3.go delete mode 100644 plugin/output/s3/s3_test.go create mode 100644 plugin/output/s3/usecase/README.idoc.md create mode 100755 plugin/output/s3/usecase/README.md rename plugin/output/s3/{ => usecase}/compress.go (99%) rename plugin/output/s3/{ => usecase}/compress_test.go (99%) create mode 100644 plugin/output/s3/usecase/s3.go create mode 100644 plugin/output/s3/usecase/s3_internals.go create mode 100644 plugin/output/s3/usecase/s3_test.go diff --git a/Makefile b/Makefile index 3e0d39b7d..2e9b8d803 100644 --- a/Makefile +++ b/Makefile @@ -63,3 +63,7 @@ push-images-all: push-images-version push-images-latest lint: # installation: https://golangci-lint.run/usage/install/#local-installation golangci-lint run --new-from-rev=${UPSTREAM_BRANCH} + +.PHONY: mock +mock: + go generate mock_gen.go diff --git a/README.md b/README.md index 05aa59518..c7d6f83e6 100755 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ TBD: throughput on production servers. **Action**: [add_host](plugin/action/add_host/README.md), [convert_date](plugin/action/convert_date/README.md), [debug](plugin/action/debug/README.md), [discard](plugin/action/discard/README.md), [flatten](plugin/action/flatten/README.md), [join](plugin/action/join/README.md), [json_decode](plugin/action/json_decode/README.md), [keep_fields](plugin/action/keep_fields/README.md), [mask](plugin/action/mask/README.md), [modify](plugin/action/modify/README.md), [parse_es](plugin/action/parse_es/README.md), [parse_re2](plugin/action/parse_re2/README.md), [remove_fields](plugin/action/remove_fields/README.md), [rename](plugin/action/rename/README.md), [throttle](plugin/action/throttle/README.md) -**Output**: [devnull](plugin/output/devnull/README.md), [elasticsearch](plugin/output/elasticsearch/README.md), [gelf](plugin/output/gelf/README.md), [kafka](plugin/output/kafka/README.md), [splunk](plugin/output/splunk/README.md), [stdout](plugin/output/stdout/README.md) +**Output**: [devnull](plugin/output/devnull/README.md), [elasticsearch](plugin/output/elasticsearch/README.md), [gelf](plugin/output/gelf/README.md), [kafka](plugin/output/kafka/README.md), [s3](plugin/output/s3/README.md), [splunk](plugin/output/splunk/README.md), [stdout](plugin/output/stdout/README.md) ## What's next * [Quick start](/docs/quick-start.md) diff --git a/_sidebar.md b/_sidebar.md index 3a0f591e1..af4e71a7d 100644 --- a/_sidebar.md +++ b/_sidebar.md @@ -43,6 +43,7 @@ - [elasticsearch](plugin/output/elasticsearch/README.md) - [gelf](plugin/output/gelf/README.md) - [kafka](plugin/output/kafka/README.md) + - [s3](plugin/output/s3/usecase/README.md) - [splunk](plugin/output/splunk/README.md) - [stdout](plugin/output/stdout/README.md) diff --git a/go.mod b/go.mod index 1bc5da226..542e4466d 100644 --- a/go.mod +++ b/go.mod @@ -37,6 +37,8 @@ require ( k8s.io/utils v0.0.0-20190829053155-3a4a5477acf8 // indirect ) +require github.com/golang/mock v1.6.0 + require ( github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -86,6 +88,7 @@ require ( golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect golang.org/x/text v0.3.6 // indirect golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect + golang.org/x/tools v0.1.1 // indirect google.golang.org/appengine v1.4.0 // indirect google.golang.org/protobuf v1.25.0 // indirect gopkg.in/square/go-jose.v2 v2.5.1 // indirect diff --git a/go.sum b/go.sum index 1f26c646b..eff63d980 100644 --- a/go.sum +++ b/go.sum @@ -116,6 +116,8 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc h1:55rEp52jU6bkyslZ1+C/7NGfpQsEc6pxGLAGDOctqbw= github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -316,8 +318,6 @@ github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLk github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w8= -github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM= github.com/rjeczalik/notify v0.9.3-0.20210809113154-3472d85e95cd h1:LHLg0gdpRUCvujg2Zol6e2Uknq5vHycLxqEzYwxt1vY= github.com/rjeczalik/notify v0.9.3-0.20210809113154-3472d85e95cd/go.mod h1:gF3zSOrafR9DQEWSE8TjfI9NkooDxbyT4UgRGKZA0lc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -355,6 +355,7 @@ github.com/vitkovskii/insane-json v0.1.0 h1:EY22tUdAaMpXDxaw4P1Ojs9jEtsJjvLFgWis github.com/vitkovskii/insane-json v0.1.0/go.mod h1:xQyYcnFJ8ElboaEZG805SrQ7I4QupForGkm0/TnRaZ8= github.com/xdg/scram v1.0.3/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -371,6 +372,7 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI= @@ -382,6 +384,8 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -401,6 +405,7 @@ golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -412,6 +417,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -435,7 +441,9 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -461,9 +469,12 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/mock_gen.go b/mock_gen.go new file mode 100644 index 000000000..bc474ee16 --- /dev/null +++ b/mock_gen.go @@ -0,0 +1,3 @@ +package main + +//go:generate mockgen -source=plugin/output/s3/interfaces.go -destination=plugin/output/s3/mock/s3.go diff --git a/pipeline/plugin.go b/pipeline/plugin.go index 45035529f..57b43e53e 100644 --- a/pipeline/plugin.go +++ b/pipeline/plugin.go @@ -33,6 +33,12 @@ type OutputPlugin interface { Out(*Event) } +type PluginsStarterData struct { + Config AnyConfig + Params *OutputPluginParams +} +type PluginsStarterMap map[string]PluginsStarterData + type PluginDefaultParams struct { PipelineName string PipelineSettings *Settings @@ -119,3 +125,18 @@ const ( MatchModeOr MatchMode = 1 MatchModeUnknown MatchMode = 2 ) + +// PluginSelector only valid value now is ByNameSelector +// and only value is string type. If required in future it can be expanded with custom type. +type PluginSelector struct { + CondType ConditionType + CondValue string +} + +type ConditionType int + +const ( + // UnknownSelector value is default, therefore it's safer to user it as default unknown value. + UnknownSelector ConditionType = iota + ByNameSelector +) diff --git a/plugin/README.md b/plugin/README.md index 59f63519c..5d8c578c0 100755 --- a/plugin/README.md +++ b/plugin/README.md @@ -53,6 +53,54 @@ So you can use Elasticsearch filebeat output plugin to send data to `file.d`. > Plugin answers with HTTP code `OK 200` right after it has read all the request body. > It doesn't wait until events are committed. +**Example:** +Emulating elastic through http: +```yaml +pipelines: + example_k8s_pipeline: + settings: + capacity: 1024 + input: + # define input type. + type: http + # pretend elastic search, emulate it's protocol. + emulate_mode: "elasticsearch" + # define http port. + address: ":9200" + actions: + # parse elastic search query. + - type: parse_es + # decode elastic search json. + - type: json_decode + # field is required. + field: message + output: + # Let's write to kafka example. + type: kafka + brokers: [kafka-local:9092, kafka-local:9091] + default_topic: yourtopic-k8s-data + use_topic_field: true + topic_field: pipeline_kafka_topic + + # Or we can write to file: + # type: file + # target_file: "./output.txt" +``` + +Setup: +``` +# run server. +# config.yaml should contains yaml config above. +go run cmd/file.d.go --config=config.yaml + +# now do requests. +curl "localhost:9200/_bulk" -H 'Content-Type: application/json' -d \ +'{"index":{"_index":"index-main","_type":"span"}} +{"message": "hello", "kind": "normal"} + +## + + [More details...](plugin/input/http/README.md) ## journalctl Reads `journalctl` output. @@ -299,6 +347,78 @@ Allowed characters in field names are letters, numbers, underscores, dashes, and It sends the event batches to kafka brokers using `sarama` lib. [More details...](plugin/output/kafka/README.md) +## s3 +Sends events to s3 output of one or multiple buckets. +`bucket` is default bucket for events. Addition buckets can be described in `multi_buckets` section, example down here. +Field "bucket_field_in_event" is filed name, that will be searched in event. +If appears we try to send event to this bucket instead of described here. + +**Example** +Standard example: +```yaml +pipelines: + mkk: + settings: + capacity: 128 + # input plugin is not important in this case, let's emulate http input. + input: + type: http + emulate_mode: "no" + address: ":9200" + actions: + - type: json_decode + field: message + output: + type: s3 + file_plugin: + retention_interval: 10s + # endpoint, access_key, secret_key, bucket are required. + endpoint: "s3.fake_host.org:80" + access_key: "access_key1" + secret_key: "secret_key2" + bucket: "bucket-logs" + bucket_field_in_event: "bucket_name" +``` + +Example with fan-out buckets: +```yaml +pipelines: + mkk: + settings: + capacity: 128 + # input plugin is not important in this case, let's emulate http input. + input: + type: http + emulate_mode: "no" + address: ":9200" + actions: + - type: json_decode + field: message + output: + type: s3 + file_plugin: + retention_interval: 10s + # endpoint, access_key, secret_key, bucket are required. + endpoint: "s3.fake_host.org:80" + access_key: "access_key1" + secret_key: "secret_key2" + bucket: "bucket-logs" + # bucket_field_in_event - event with such field will be sent to bucket with its value + # if such exists: {"bucket_name": "secret", "message": 123} to bucket "secret". + bucket_field_in_event: "bucket_name" + # multi_buckets is optional, contains array of buckets. + multi_buckets: + - endpoint: "otherS3.fake_host.org:80" + access_key: "access_key2" + secret_key: "secret_key2" + bucket: "bucket-logs-2" + - endpoint: "yet_anotherS3.fake_host.ru:80" + access_key: "access_key3" + secret_key: "secret_key3" + bucket: "bucket-logs-3" +``` + +[More details...](plugin/output/s3/README.md) ## splunk It sends events to splunk. diff --git a/plugin/action/join/README.md b/plugin/action/join/README.md index 90f2ccf94..7d56a7e61 100755 --- a/plugin/action/join/README.md +++ b/plugin/action/join/README.md @@ -40,6 +40,12 @@ A regexp which will continue the join sequence.
+**`max_event_size`** *`int`* *`default=0`* + +Max size of the resulted event. If it is set and the event exceeds the limit, the event will be truncated. + +
+ ### Understanding start/continue regexps **No joining:** diff --git a/plugin/input/README.md b/plugin/input/README.md index b7cd5d482..19c960c6f 100755 --- a/plugin/input/README.md +++ b/plugin/input/README.md @@ -52,6 +52,54 @@ So you can use Elasticsearch filebeat output plugin to send data to `file.d`. > Plugin answers with HTTP code `OK 200` right after it has read all the request body. > It doesn't wait until events are committed. +**Example:** +Emulating elastic through http: +```yaml +pipelines: + example_k8s_pipeline: + settings: + capacity: 1024 + input: + # define input type. + type: http + # pretend elastic search, emulate it's protocol. + emulate_mode: "elasticsearch" + # define http port. + address: ":9200" + actions: + # parse elastic search query. + - type: parse_es + # decode elastic search json. + - type: json_decode + # field is required. + field: message + output: + # Let's write to kafka example. + type: kafka + brokers: [kafka-local:9092, kafka-local:9091] + default_topic: yourtopic-k8s-data + use_topic_field: true + topic_field: pipeline_kafka_topic + + # Or we can write to file: + # type: file + # target_file: "./output.txt" +``` + +Setup: +``` +# run server. +# config.yaml should contains yaml config above. +go run cmd/file.d.go --config=config.yaml + +# now do requests. +curl "localhost:9200/_bulk" -H 'Content-Type: application/json' -d \ +'{"index":{"_index":"index-main","_type":"span"}} +{"message": "hello", "kind": "normal"} + +## + + [More details...](plugin/input/http/README.md) ## journalctl Reads `journalctl` output. diff --git a/plugin/input/http/README.md b/plugin/input/http/README.md index bd1787aef..04680ccf5 100755 --- a/plugin/input/http/README.md +++ b/plugin/input/http/README.md @@ -9,20 +9,7 @@ So you can use Elasticsearch filebeat output plugin to send data to `file.d`. > Plugin answers with HTTP code `OK 200` right after it has read all the request body. > It doesn't wait until events are committed. -### Config params -**`address`** *`string`* *`default=:9200`* - -An address to listen to. Omit ip/host to listen all network interfaces. E.g. `:88` - -
- -**`emulate_mode`** *`string`* *`default=no`* *`options=no|elasticsearch`* - -Which protocol to emulate. - -
- -**Example:** +**Example:** Emulating elastic through http: ```yaml pipelines: @@ -50,16 +37,16 @@ pipelines: default_topic: yourtopic-k8s-data use_topic_field: true topic_field: pipeline_kafka_topic - - # Or we can write to file: + + # Or we can write to file: # type: file # target_file: "./output.txt" ``` Setup: ``` -# run server. -# config.yaml should contains yaml config above. +# run server. +# config.yaml should contains yaml config above. go run cmd/file.d.go --config=config.yaml # now do requests. @@ -68,7 +55,20 @@ curl "localhost:9200/_bulk" -H 'Content-Type: application/json' -d \ {"message": "hello", "kind": "normal"} ## - -``` + + +### Config params +**`address`** *`string`* *`default=:9200`* + +An address to listen to. Omit ip/host to listen all network interfaces. E.g. `:88` + +
+ +**`emulate_mode`** *`string`* *`default=no`* *`options=no|elasticsearch`* + +Which protocol to emulate. + +
+
*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)* \ No newline at end of file diff --git a/plugin/input/http/http.go b/plugin/input/http/http.go index aef3290e0..2b0427a4c 100644 --- a/plugin/input/http/http.go +++ b/plugin/input/http/http.go @@ -21,6 +21,54 @@ So you can use Elasticsearch filebeat output plugin to send data to `file.d`. > âš  Currently event commitment mechanism isn't implemented for this plugin. > Plugin answers with HTTP code `OK 200` right after it has read all the request body. > It doesn't wait until events are committed. + +**Example:** +Emulating elastic through http: +```yaml +pipelines: + example_k8s_pipeline: + settings: + capacity: 1024 + input: + # define input type. + type: http + # pretend elastic search, emulate it's protocol. + emulate_mode: "elasticsearch" + # define http port. + address: ":9200" + actions: + # parse elastic search query. + - type: parse_es + # decode elastic search json. + - type: json_decode + # field is required. + field: message + output: + # Let's write to kafka example. + type: kafka + brokers: [kafka-local:9092, kafka-local:9091] + default_topic: yourtopic-k8s-data + use_topic_field: true + topic_field: pipeline_kafka_topic + + # Or we can write to file: + # type: file + # target_file: "./output.txt" +``` + +Setup: +``` +# run server. +# config.yaml should contains yaml config above. +go run cmd/file.d.go --config=config.yaml + +# now do requests. +curl "localhost:9200/_bulk" -H 'Content-Type: application/json' -d \ +'{"index":{"_index":"index-main","_type":"span"}} +{"message": "hello", "kind": "normal"} + +## + }*/ type Plugin struct { diff --git a/plugin/input/k8s/README.md b/plugin/input/k8s/README.md index f1b142398..63596390a 100755 --- a/plugin/input/k8s/README.md +++ b/plugin/input/k8s/README.md @@ -70,4 +70,4 @@ Under the hood this plugin uses [file plugin](/plugin/input/file/README.md) to c
-
*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)* +
*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)* \ No newline at end of file diff --git a/plugin/output/README.md b/plugin/output/README.md index 7dba36706..4d997fe40 100755 --- a/plugin/output/README.md +++ b/plugin/output/README.md @@ -32,6 +32,78 @@ Allowed characters in field names are letters, numbers, underscores, dashes, and It sends the event batches to kafka brokers using `sarama` lib. [More details...](plugin/output/kafka/README.md) +## s3 +Sends events to s3 output of one or multiple buckets. +`bucket` is default bucket for events. Addition buckets can be described in `multi_buckets` section, example down here. +Field "bucket_field_in_event" is filed name, that will be searched in event. +If appears we try to send event to this bucket instead of described here. + +**Example** +Standard example: +```yaml +pipelines: + mkk: + settings: + capacity: 128 + # input plugin is not important in this case, let's emulate http input. + input: + type: http + emulate_mode: "no" + address: ":9200" + actions: + - type: json_decode + field: message + output: + type: s3 + file_plugin: + retention_interval: 10s + # endpoint, access_key, secret_key, bucket are required. + endpoint: "s3.fake_host.org:80" + access_key: "access_key1" + secret_key: "secret_key2" + bucket: "bucket-logs" + bucket_field_in_event: "bucket_name" +``` + +Example with fan-out buckets: +```yaml +pipelines: + mkk: + settings: + capacity: 128 + # input plugin is not important in this case, let's emulate http input. + input: + type: http + emulate_mode: "no" + address: ":9200" + actions: + - type: json_decode + field: message + output: + type: s3 + file_plugin: + retention_interval: 10s + # endpoint, access_key, secret_key, bucket are required. + endpoint: "s3.fake_host.org:80" + access_key: "access_key1" + secret_key: "secret_key2" + bucket: "bucket-logs" + # bucket_field_in_event - event with such field will be sent to bucket with its value + # if such exists: {"bucket_name": "secret", "message": 123} to bucket "secret". + bucket_field_in_event: "bucket_name" + # multi_buckets is optional, contains array of buckets. + multi_buckets: + - endpoint: "otherS3.fake_host.org:80" + access_key: "access_key2" + secret_key: "secret_key2" + bucket: "bucket-logs-2" + - endpoint: "yet_anotherS3.fake_host.ru:80" + access_key: "access_key3" + secret_key: "secret_key3" + bucket: "bucket-logs-3" +``` + +[More details...](plugin/output/s3/README.md) ## splunk It sends events to splunk. diff --git a/plugin/output/file/file.go b/plugin/output/file/file.go index 048038ef0..d8bf94545 100644 --- a/plugin/output/file/file.go +++ b/plugin/output/file/file.go @@ -19,6 +19,12 @@ import ( "golang.org/x/net/context" ) +type PluginInterface interface { + Start(config pipeline.AnyConfig, params *pipeline.OutputPluginParams) + Out(event *pipeline.Event) + Stop() +} + type Plugin struct { controller pipeline.OutputPluginController logger *zap.SugaredLogger @@ -53,6 +59,7 @@ const ( type Config struct { //> File name for log file. + // defaultTargetFileName = TargetFile default value TargetFile string `json:"target_file" default:"/var/log/file-d.log"` //* //> Interval of creation new file @@ -203,7 +210,7 @@ func (p *Plugin) write(data []byte) { func (p *Plugin) createNew() { p.tsFileName = fmt.Sprintf("%d%s%s%s", time.Now().Unix(), fileNameSeparator, p.fileName, p.fileExtension) - logger.Errorf("tsFileName in createNew=%s", p.tsFileName) + logger.Infof("tsFileName in createNew=%s", p.tsFileName) f := fmt.Sprintf("%s%s", p.targetDir, p.tsFileName) pattern := fmt.Sprintf("%s*%s%s%s", p.targetDir, fileNameSeparator, p.fileName, p.fileExtension) matches, err := filepath.Glob(pattern) @@ -242,7 +249,6 @@ func (p *Plugin) sealUp() { if err := oldFile.Close(); err != nil { p.logger.Panicf("could not close file: %s, error: %s", oldFile.Name(), err.Error()) } - logger.Errorf("sealing in %d, newFile: %s", time.Now().Unix(), newFileName) if p.SealUpCallback != nil { longpanic.Go(func() { p.SealUpCallback(newFileName) }) diff --git a/plugin/output/file/file_test.go b/plugin/output/file/file_test.go index 574cf328d..c18a18da3 100644 --- a/plugin/output/file/file_test.go +++ b/plugin/output/file/file_test.go @@ -66,7 +66,7 @@ func TestGetStartIdx(t *testing.T) { fileName: file[0 : len(file)-len(extension)], } - // create files + // create files. files := make([]*os.File, len(tc.filesName)) createDir(t, p.targetDir) for _, f := range tc.filesName { @@ -76,7 +76,7 @@ func TestGetStartIdx(t *testing.T) { idx := p.getStartIdx() assert.EqualValues(t, tc.expectedIdx, idx) - // close files + // close files. for _, f := range files { f.Close() } @@ -309,7 +309,7 @@ func TestStart(t *testing.T) { // check seal up for third time.Sleep(sealUpFileSleep) matches = test.GetMatches(t, generalPattern) - assert.GreaterOrEqual(t, len(matches), 4, "there is no new files after sealing up third pack") + assert.GreaterOrEqual(t, len(matches), 4, "there is no new plugins after sealing up third pack") checkDirFiles(t, matches, totalSent, "lost data for third pack") for _, m := range matches { if strings.Contains(m, currentLogFileSubstr) { diff --git a/plugin/output/file/files.go b/plugin/output/file/files.go new file mode 100644 index 000000000..121465902 --- /dev/null +++ b/plugin/output/file/files.go @@ -0,0 +1,91 @@ +package file + +import ( + "sync" + + "github.com/ozonru/file.d/logger" + "github.com/ozonru/file.d/pipeline" +) + +// Plugins is an abstraction upon multiple file.Plugin, which helps reuse it. +type Plugins struct { + // plugins contains plugs that exist from start of work. + plugins map[string]PluginInterface + // dynamicPlugins contains plugs, that created dynamically during execution. + // they separated from plugins to avoid races and reduce locking complexity. + dynamicPlugins map[string]PluginInterface + mu sync.RWMutex +} + +func NewFilePlugins(plugins map[string]PluginInterface) *Plugins { + return &Plugins{plugins: plugins, dynamicPlugins: make(map[string]PluginInterface)} +} + +func (p *Plugins) Out(event *pipeline.Event, selector pipeline.PluginSelector) { + switch selector.CondType { + case pipeline.ByNameSelector: + func() { + if p.IsStatic(selector.CondValue) { + p.plugins[selector.CondValue].Out(event) + } else { + p.mu.RLock() + defer p.mu.RUnlock() + + p.dynamicPlugins[selector.CondValue].Out(event) + } + }() + default: + logger.Fatalf("PluginSelector type didn't set for event: %v, selector: %v", event, selector) + } +} + +// Start runs all plugins. +func (p *Plugins) Start(starterData pipeline.PluginsStarterMap) { + p.mu.Lock() + defer p.mu.Unlock() + + for plugName, plug := range p.plugins { + plug.Start(starterData[plugName].Config, starterData[plugName].Params) + } +} + +// Stop stops all plugins (very useful comment). +func (p *Plugins) Stop() { + p.mu.Lock() + defer p.mu.Unlock() + + for _, plug := range p.plugins { + plug.Stop() + } + for _, plug := range p.dynamicPlugins { + plug.Stop() + } +} + +// Add new plugin to plugs. +func (p *Plugins) Add(plugName string, plug PluginInterface) { + p.mu.Lock() + defer p.mu.Unlock() + + p.dynamicPlugins[plugName] = plug +} + +// Exists asks if such file.Plugin exists in Plugins. +func (p *Plugins) Exists(plugName string) (exists bool) { + return p.IsStatic(plugName) || p.IsDynamic(plugName) +} + +// IsStatic tells is plugin created from config. +func (p *Plugins) IsStatic(plugName string) bool { + _, ok := p.plugins[plugName] + return ok +} + +// IsDynamic tells is plugin created from events. +func (p *Plugins) IsDynamic(PlugName string) bool { + p.mu.RLock() + defer p.mu.RUnlock() + + _, ok := p.dynamicPlugins[PlugName] + return ok +} diff --git a/plugin/output/file/files_test.go b/plugin/output/file/files_test.go new file mode 100644 index 000000000..80e39c9cc --- /dev/null +++ b/plugin/output/file/files_test.go @@ -0,0 +1,137 @@ +package file + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPlugins_Exists(t *testing.T) { + cases := []struct { + name string + prepareFunc func() *Plugins + search string + expected bool + }{ + { + name: "not_exists", + prepareFunc: func() *Plugins { + return NewFilePlugins(nil) + }, + search: "somePlugin", + expected: false, + }, + { + name: "not_exists", + prepareFunc: func() *Plugins { + return NewFilePlugins(map[string]PluginInterface{ + "EXISTS": nil, + }) + }, + search: "EXISTS", + expected: true, + }, + } + + for _, tCase := range cases { + t.Run(tCase.name, func(t *testing.T) { + plugins := tCase.prepareFunc() + exists := plugins.Exists(tCase.search) + require.Equal(t, tCase.expected, exists) + }) + } +} + +func TestPlugins_IsStatic(t *testing.T) { + cases := []struct { + name string + prepareFunc func() *Plugins + search string + expected bool + }{ + { + name: "not_static", + prepareFunc: func() *Plugins { + return NewFilePlugins(nil) + }, + search: "kekw", + expected: false, + }, + { + name: "static", + prepareFunc: func() *Plugins { + return NewFilePlugins(map[string]PluginInterface{ + "plugname": nil, + }) + }, + search: "plugname", + expected: true, + }, + } + + for _, tCase := range cases { + t.Run(tCase.name, func(t *testing.T) { + plugins := tCase.prepareFunc() + exists := plugins.IsStatic(tCase.search) + require.Equal(t, tCase.expected, exists) + }) + } +} + +func TestPlugins_IsDynamic(t *testing.T) { + cases := []struct { + name string + prepareFunc func() *Plugins + search string + expected bool + }{ + { + name: "not_dynamic", + prepareFunc: func() *Plugins { + return NewFilePlugins(nil) + }, + search: "kekw", + expected: false, + }, + { + name: "dynamic", + prepareFunc: func() *Plugins { + plugins := NewFilePlugins(nil) + plugins.dynamicPlugins["dynamic_plug"] = nil + + return plugins + }, + search: "dynamic_plug", + expected: true, + }, + } + + for _, tCase := range cases { + t.Run(tCase.name, func(t *testing.T) { + plugins := tCase.prepareFunc() + exists := plugins.IsDynamic(tCase.search) + require.Equal(t, tCase.expected, exists) + }) + } +} + +func TestPlugins_ExistsIsStaticIsDynamic(t *testing.T) { + staticPlug := "abc" + dynamicPlug := "def" + plugins := NewFilePlugins(map[string]PluginInterface{ + staticPlug: nil, + }) + plugins.Add(dynamicPlug, nil) + + require.True(t, plugins.Exists(staticPlug)) + require.True(t, plugins.IsStatic(staticPlug)) + require.False(t, plugins.IsDynamic(staticPlug)) + require.True(t, plugins.Exists(dynamicPlug)) + require.True(t, plugins.IsDynamic(dynamicPlug)) + require.False(t, plugins.IsStatic(dynamicPlug)) + + imNotExist := "lol" + require.False(t, plugins.Exists(imNotExist)) + require.False(t, plugins.IsDynamic(imNotExist)) + require.False(t, plugins.IsDynamic(imNotExist)) +} diff --git a/plugin/output/s3/interfaces.go b/plugin/output/s3/interfaces.go new file mode 100644 index 000000000..e44012e8f --- /dev/null +++ b/plugin/output/s3/interfaces.go @@ -0,0 +1,12 @@ +package s3 + +import "github.com/minio/minio-go" + +type ObjStoreFabricInterface interface { + NewObjStoreClient(endpoint, accessKeyID, secretAccessKey string, secure bool) (ObjectStoreClient, error) +} + +type ObjectStoreClient interface { + BucketExists(bucketName string) (bool, error) + FPutObject(bucketName, objectName, filePath string, opts minio.PutObjectOptions) (n int64, err error) +} diff --git a/plugin/output/s3/mock/s3.go b/plugin/output/s3/mock/s3.go new file mode 100644 index 000000000..d4787725b --- /dev/null +++ b/plugin/output/s3/mock/s3.go @@ -0,0 +1,104 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: plugin/output/s3/interfaces.go + +// Package mock_s3 is a generated GoMock package. +package mock_s3 + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + minio "github.com/minio/minio-go" + s3 "github.com/ozonru/file.d/plugin/output/s3" +) + +// MockObjStoreFabricInterface is a mock of ObjStoreFabricInterface interface. +type MockObjStoreFabricInterface struct { + ctrl *gomock.Controller + recorder *MockObjStoreFabricInterfaceMockRecorder +} + +// MockObjStoreFabricInterfaceMockRecorder is the mock recorder for MockObjStoreFabricInterface. +type MockObjStoreFabricInterfaceMockRecorder struct { + mock *MockObjStoreFabricInterface +} + +// NewMockObjStoreFabricInterface creates a new mock instance. +func NewMockObjStoreFabricInterface(ctrl *gomock.Controller) *MockObjStoreFabricInterface { + mock := &MockObjStoreFabricInterface{ctrl: ctrl} + mock.recorder = &MockObjStoreFabricInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockObjStoreFabricInterface) EXPECT() *MockObjStoreFabricInterfaceMockRecorder { + return m.recorder +} + +// NewObjStoreClient mocks base method. +func (m *MockObjStoreFabricInterface) NewObjStoreClient(endpoint, accessKeyID, secretAccessKey string, secure bool) (s3.ObjectStoreClient, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NewObjStoreClient", endpoint, accessKeyID, secretAccessKey, secure) + ret0, _ := ret[0].(s3.ObjectStoreClient) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NewObjStoreClient indicates an expected call of NewObjStoreClient. +func (mr *MockObjStoreFabricInterfaceMockRecorder) NewObjStoreClient(endpoint, accessKeyID, secretAccessKey, secure interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewObjStoreClient", reflect.TypeOf((*MockObjStoreFabricInterface)(nil).NewObjStoreClient), endpoint, accessKeyID, secretAccessKey, secure) +} + +// MockObjectStoreClient is a mock of ObjectStoreClient interface. +type MockObjectStoreClient struct { + ctrl *gomock.Controller + recorder *MockObjectStoreClientMockRecorder +} + +// MockObjectStoreClientMockRecorder is the mock recorder for MockObjectStoreClient. +type MockObjectStoreClientMockRecorder struct { + mock *MockObjectStoreClient +} + +// NewMockObjectStoreClient creates a new mock instance. +func NewMockObjectStoreClient(ctrl *gomock.Controller) *MockObjectStoreClient { + mock := &MockObjectStoreClient{ctrl: ctrl} + mock.recorder = &MockObjectStoreClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockObjectStoreClient) EXPECT() *MockObjectStoreClientMockRecorder { + return m.recorder +} + +// BucketExists mocks base method. +func (m *MockObjectStoreClient) BucketExists(bucketName string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BucketExists", bucketName) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// BucketExists indicates an expected call of BucketExists. +func (mr *MockObjectStoreClientMockRecorder) BucketExists(bucketName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BucketExists", reflect.TypeOf((*MockObjectStoreClient)(nil).BucketExists), bucketName) +} + +// FPutObject mocks base method. +func (m *MockObjectStoreClient) FPutObject(bucketName, objectName, filePath string, opts minio.PutObjectOptions) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FPutObject", bucketName, objectName, filePath, opts) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FPutObject indicates an expected call of FPutObject. +func (mr *MockObjectStoreClientMockRecorder) FPutObject(bucketName, objectName, filePath, opts interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FPutObject", reflect.TypeOf((*MockObjectStoreClient)(nil).FPutObject), bucketName, objectName, filePath, opts) +} diff --git a/plugin/output/s3/s3.go b/plugin/output/s3/s3.go deleted file mode 100644 index b226bbc38..000000000 --- a/plugin/output/s3/s3.go +++ /dev/null @@ -1,327 +0,0 @@ -package s3 - -import ( - "fmt" - "math" - "math/rand" - "os" - "path" - "path/filepath" - "sort" - "strconv" - "time" - - "github.com/minio/minio-go" - "github.com/ozonru/file.d/fd" - "github.com/ozonru/file.d/longpanic" - "github.com/ozonru/file.d/pipeline" - "github.com/ozonru/file.d/plugin/output/file" - "go.uber.org/zap" -) - -const ( - outPluginType = "s3" - fileNameSeparator = "_" - attemptIntervalMin = 1 * time.Second - - bucketField = "bucket" -) - -var ( - attemptInterval = attemptIntervalMin - compressors = map[string]func(*zap.SugaredLogger) compressor{ - zipName: newZipCompressor, - } - - r = rand.New(rand.NewSource(time.Now().UnixNano())) -) - -type objectStoreClient interface { - BucketExists(bucketName string) (bool, error) - FPutObject(bucketName, objectName, filePath string, opts minio.PutObjectOptions) (n int64, err error) -} - -type compressor interface { - compress(archiveName, fileName string) - getObjectOptions() minio.PutObjectOptions - getExtension() string - getName(fileName string) string -} - -type Plugin struct { - controller pipeline.OutputPluginController - logger *zap.SugaredLogger - config *Config - client objectStoreClient - outPlugins map[string]*file.Plugin - - targetDir string - fileExtension string - fileName string - - compressCh chan string - uploadCh chan string - - compressor compressor -} - -type Config struct { - // Under the hood this plugin uses /plugin/output/file/ to collect logs - FileConfig file.Config `json:"file_config" child:"true"` - - // Compression type - CompressionType string `json:"compression_type" default:"zip" options:"zip"` - - // s3 section - Endpoint string `json:"endpoint" required:"true"` - AccessKey string `json:"access_key" required:"true"` - SecretKey string `json:"secret_key" required:"true"` - // Required s3 default bucket. - Bucket string `json:"bucket" required:"true"` - // MultiBuckets is additional buckets, which can also receive event. - // Event must contain `bucket_name` which value is valid s3 bucket name. - // Events without `bucket_name` sends to Bucket. - MultiBuckets []singleBucket `json:"multi_buckets" required:"false"` - Secure bool `json:"secure" default:"false"` - - // for mock client injection - client *objectStoreClient -} - -type singleBucket struct { - // s3 section - Endpoint string `json:"endpoint" required:"true"` - AccessKey string `json:"access_key" required:"true"` - SecretKey string `json:"secret_key" required:"true"` - // Required s3 default bucket. - Bucket string `json:"bucket" required:"true"` -} - -func init() { - fd.DefaultPluginRegistry.RegisterOutput(&pipeline.PluginStaticInfo{ - Type: outPluginType, - Factory: Factory, - }) -} - -func Factory() (pipeline.AnyPlugin, pipeline.AnyConfig) { - return &Plugin{}, &Config{} -} - -func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.OutputPluginParams) { - p.controller = params.Controller - p.logger = params.Logger - p.config = config.(*Config) - - // set up compression - newCompressor, ok := compressors[p.config.CompressionType] - if !ok { - p.logger.Fatalf("compression type: %s is not supported", p.config.CompressionType) - } - p.compressor = newCompressor(p.logger) - - dir, f := filepath.Split(p.config.FileConfig.TargetFile) - - p.targetDir = dir - p.fileExtension = filepath.Ext(f) - p.fileName = f[0 : len(f)-len(p.fileExtension)] - - p.uploadCh = make(chan string, p.config.FileConfig.WorkersCount_*4) - p.compressCh = make(chan string, p.config.FileConfig.WorkersCount_) - - for i := 0; i < p.config.FileConfig.WorkersCount_; i++ { - longpanic.Go(p.uploadWork) - longpanic.Go(p.compressWork) - } - - // initialize minio client object. - minioClient, err := minio.New(p.config.Endpoint, p.config.AccessKey, p.config.SecretKey, p.config.Secure) - if err != nil || minioClient == nil { - p.logger.Panicf("could not create minio client, error: %s", err.Error()) - } - p.client = minioClient - - if p.config.client != nil { - p.logger.Info("set mock client") - p.client = *p.config.client - } - - outPlugins := make(map[string]*file.Plugin, len(p.config.MultiBuckets)+1) - // Checks required bucket and exit if it's invalid. - exist, err := p.client.BucketExists(p.config.Bucket) - if err != nil { - p.logger.Panicf("could not check bucket: %s, error: %s", p.config.Bucket, err.Error()) - } - if !exist { - p.logger.Fatalf("bucket: %s, does not exist", p.config.Bucket) - } - anyPlugin, _ := file.Factory() - outPlugin := anyPlugin.(*file.Plugin) - outPlugin.SealUpCallback = p.addFileJob - outPlugins[p.config.Bucket] = outPlugin - - // If multi_buckets described on file.d config, check each of them as well. - for _, singleB := range p.config.MultiBuckets { - exist, err := p.client.BucketExists(singleB.Bucket) - if err != nil { - p.logger.Panicf("could not check bucket: %s, error: %s", singleB.Bucket, err.Error()) - } - if !exist { - p.logger.Fatalf("bucket from multi_backets: %s, does not exist", singleB.Bucket) - } - anyPlugin, _ := file.Factory() - outPlugin := anyPlugin.(*file.Plugin) - outPlugin.SealUpCallback = p.addFileJob - outPlugins[singleB.Bucket] = outPlugin - } - - // If multi_buckets described on file.d config, check each of them as well. - for _, singleB := range p.config.MultiBuckets { - exist, err := p.client.BucketExists(singleB.Bucket) - if err != nil { - p.logger.Panicf("could not check bucket: %s, error: %s", singleB.Bucket, err.Error()) - } - if !exist { - p.logger.Fatalf("bucket from multi_backets: %s, does not exist", singleB.Bucket) - } - anyPlugin, _ := file.Factory() - outPlugin := anyPlugin.(*file.Plugin) - outPlugin.SealUpCallback = p.addFileJob - outPlugins[singleB.Bucket] = outPlugin - } - - p.logger.Info("client is ready") - p.logger.Infof("bucket: %s exists", p.config.Bucket) - - p.outPlugins = outPlugins - for _, pl := range p.outPlugins { - pl.Start(&p.config.FileConfig, params) - } - p.uploadExistingFiles() -} - -func (p *Plugin) Stop() { - for _, plugin := range p.outPlugins { - plugin.Stop() - } -} - -func (p *Plugin) Out(event *pipeline.Event) { - p.outPlugins[p.getS3BucketFromEvent(event)].Out(event) -} - -// getS3BucketFromEvent tries finds bucket in multi_buckets. Otherwise, events goes to main bucket. -func (p *Plugin) getS3BucketFromEvent(event *pipeline.Event) string { - bucket := event.Root.Dig(bucketField).AsString() - _, ok := p.outPlugins[bucket] - if !ok { - return p.config.Bucket - } - return bucket -} - -// uploadExistingFiles gets files from dirs, sorts it, compresses it if it's need, and then upload to s3. -func (p *Plugin) uploadExistingFiles() { - // get all compressed files. - pattern := fmt.Sprintf("%s*%s", p.targetDir, p.compressor.getExtension()) - compressedFiles, err := filepath.Glob(pattern) - if err != nil { - p.logger.Panicf("could not read dir: %s", p.targetDir) - } - // sort compressed files by creation time. - sort.Slice(compressedFiles, p.getSortFunc(compressedFiles)) - // upload archive. - for _, z := range compressedFiles { - p.uploadCh <- z - } - - // compress all files that we have in the dir - p.compressFilesInDir() -} - -// compressFilesInDir compresses all files in dir -func (p *Plugin) compressFilesInDir() { - pattern := fmt.Sprintf("%s/%s%s*%s*%s", p.targetDir, p.fileName, fileNameSeparator, fileNameSeparator, p.fileExtension) - files, err := filepath.Glob(pattern) - if err != nil { - p.logger.Panicf("could not read dir: %s", p.targetDir) - } - // sort files by creation time - sort.Slice(files, p.getSortFunc(files)) - for _, f := range files { - p.compressCh <- f - } -} - -// getSortFunc return func that sorts files by mod time -func (p *Plugin) getSortFunc(files []string) func(i, j int) bool { - return func(i, j int) bool { - InfoI, err := os.Stat(files[i]) - if err != nil { - p.logger.Panicf("could not get info about file: %s, error: %s", files[i], err.Error()) - } - InfoJ, err := os.Stat(files[j]) - if err != nil { - p.logger.Panicf("could not get info about file: %s, error: %s", files[j], err.Error()) - } - return InfoI.ModTime().Before(InfoJ.ModTime()) - } -} - -func (p *Plugin) addFileJob(fileName string) { - p.compressCh <- fileName -} - -// uploadWork uploads compressed files from channel to s3 and then delete compressed file -// in case error worker will attempt sending with an exponential time interval -func (p *Plugin) uploadWork() { - for compressed := range p.uploadCh { - sleepTime := attemptInterval - for { - err := p.uploadToS3(compressed) - if err == nil { - p.logger.Infof("successfully uploaded object: %s", compressed) - // delete archive after uploading - err = os.Remove(compressed) - if err != nil { - p.logger.Panicf("could not delete file: %s, err: %s", compressed, err.Error()) - } - break - } - sleepTime += sleepTime - p.logger.Errorf("could not upload object: %s, next attempt in %s, error: %s", compressed, sleepTime.String(), err.Error()) - time.Sleep(sleepTime) - } - } -} - -// compressWork compress file from channel and then delete source file -func (p *Plugin) compressWork() { - for f := range p.compressCh { - compressedName := p.compressor.getName(f) - p.compressor.compress(compressedName, f) - // delete old file - if err := os.Remove(f); err != nil { - p.logger.Panicf("could not delete file: %s, error: %s", f, err.Error()) - } - p.uploadCh <- compressedName - } -} - -// uploadToS3 uploads compressed file to s3 -func (p *Plugin) uploadToS3(name string) error { - _, err := p.client.FPutObject(p.config.Bucket, p.generateObjectName(name), name, p.compressor.getObjectOptions()) - if err != nil { - return fmt.Errorf("could not upload file: %s into bucket: %s, error: %s", name, p.config.Bucket, err.Error()) - } - return nil -} - -// generateObjectName generates object name by compressed file name -func (p *Plugin) generateObjectName(name string) string { - n := strconv.FormatInt(r.Int63n(math.MaxInt64), 16) - n = n[len(n)-8:] - objectName := path.Base(name) - objectName = objectName[0 : len(objectName)-len(p.compressor.getExtension())] - return fmt.Sprintf("%s.%s%s", objectName, n, p.compressor.getExtension()) -} diff --git a/plugin/output/s3/s3_test.go b/plugin/output/s3/s3_test.go deleted file mode 100644 index ea746472d..000000000 --- a/plugin/output/s3/s3_test.go +++ /dev/null @@ -1,431 +0,0 @@ -package s3 - -import ( - "bufio" - "fmt" - "net/http" - "os" - "path/filepath" - "testing" - "time" - - "github.com/minio/minio-go" - "github.com/ozonru/file.d/cfg" - "github.com/ozonru/file.d/logger" - "github.com/ozonru/file.d/pipeline" - "github.com/ozonru/file.d/plugin/input/fake" - "github.com/ozonru/file.d/plugin/output/file" - "github.com/ozonru/file.d/test" - "github.com/prometheus/client_golang/prometheus" - "github.com/stretchr/testify/assert" - "golang.org/x/net/context" -) - -const targetFile = "filetests/log.log" - -var ( - dir, _ = filepath.Split(targetFile) - fileName = "" -) - -type mockClient struct{} - -func newMockClient() objectStoreClient { - return mockClient{} -} - -func (m mockClient) BucketExists(bucketName string) (bool, error) { - return true, nil -} - -func (m mockClient) FPutObject(bucketName, objectName, filePath string, opts minio.PutObjectOptions) (n int64, err error) { - logger.Infof("put object: %s, %s, %s", bucketName, objectName, filePath) - targetDir := fmt.Sprintf("./%s", bucketName) - if _, err := os.Stat(targetDir); os.IsNotExist(err) { - if err := os.MkdirAll(targetDir, os.ModePerm); err != nil { - logger.Fatalf("could not create target dir: %s, error: %s", targetDir, err.Error()) - } - } - fileName = fmt.Sprintf("%s/%s", bucketName, "mockLog.txt") - file, err := os.OpenFile(fileName, os.O_CREATE|os.O_APPEND|os.O_RDWR, os.FileMode(0o777)) - if err != nil { - logger.Panicf("could not open or create file: %s, error: %s", fileName, err.Error()) - } - defer file.Close() - - if _, err := file.WriteString(fmt.Sprintf("%s | from '%s' to b: `%s` as obj: `%s`\n", time.Now().String(), filePath, bucketName, objectName)); err != nil { - return 0, fmt.Errorf(err.Error()) - } - - return 1, nil -} - -func TestStart(t *testing.T) { - if testing.Short() { - t.Skip("skip long tests in short mode") - } - tests := struct { - firstPack []test.Msg - secondPack []test.Msg - thirdPack []test.Msg - }{ - firstPack: []test.Msg{ - test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), - test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), - test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), - }, - secondPack: []test.Msg{ - test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_12","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), - test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_12","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), - test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_12","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), - }, - thirdPack: []test.Msg{ - test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_123","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), - test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_123","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), - test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_123","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), - }, - } - - // pattern for parent log file - pattern := fmt.Sprintf("%s/*.log", dir) - - test.ClearDir(t, dir) - s3MockClient := newMockClient() - fileConfig := file.Config{ - TargetFile: targetFile, - RetentionInterval: "300ms", - Layout: "01", - BatchFlushTimeout: "100ms", - } - config := &Config{ - FileConfig: fileConfig, - CompressionType: "zip", - Endpoint: "some", - AccessKey: "some", - SecretKey: "some", - Bucket: "some", - Secure: false, - client: &s3MockClient, - } - test.ClearDir(t, dir) - defer test.ClearDir(t, dir) - test.ClearDir(t, config.Bucket) - defer test.ClearDir(t, config.Bucket) - - err := cfg.Parse(config, map[string]int{"gomaxprocs": 1, "capacity": 64}) - assert.NoError(t, err) - - p := newPipeline(t, config) - assert.NotNil(t, p, "could not create new pipeline") - p.Start() - time.Sleep(300 * time.Microsecond) - - test.SendPack(t, p, tests.firstPack) - time.Sleep(time.Second) - size1 := test.CheckNotZero(t, fileName, "s3 data is missed after first pack") - - // check deletion upload log files - match := test.GetMatches(t, pattern) - assert.Equal(t, 1, len(match)) - test.CheckZero(t, match[0], "log file is not nil") - - // initial sending the second pack - // no special situations - test.SendPack(t, p, tests.secondPack) - time.Sleep(time.Second) - - match = test.GetMatches(t, pattern) - assert.Equal(t, 1, len(match)) - test.CheckZero(t, match[0], "log file is not empty") - - size2 := test.CheckNotZero(t, fileName, "s3 data missed after second pack") - assert.True(t, size2 > size1) - - // failed during writing - test.SendPack(t, p, tests.thirdPack) - time.Sleep(200 * time.Millisecond) - p.Stop() - - // check log file not empty - match = test.GetMatches(t, pattern) - assert.Equal(t, 1, len(match)) - test.CheckNotZero(t, match[0], "log file data missed") - // time.Sleep(sealUpFileSleep) - - // restart like after crash - p.Start() - - time.Sleep(time.Second) - - size3 := test.CheckNotZero(t, fileName, "s3 data missed after third pack") - assert.True(t, size3 > size2) -} - -func newPipeline(t *testing.T, configOutput *Config) *pipeline.Pipeline { - t.Helper() - settings := &pipeline.Settings{ - Capacity: 4096, - MaintenanceInterval: time.Second * 100000, - AntispamThreshold: 0, - AvgEventSize: 2048, - StreamField: "stream", - Decoder: "json", - } - - http.DefaultServeMux = &http.ServeMux{} - p := pipeline.New("test_pipeline", settings, prometheus.NewRegistry()) - p.DisableParallelism() - p.EnableEventLog() - - anyPlugin, _ := fake.Factory() - inputPlugin := anyPlugin.(*fake.Plugin) - p.SetInput(&pipeline.InputPluginInfo{ - PluginStaticInfo: &pipeline.PluginStaticInfo{ - Type: "fake", - }, - PluginRuntimeInfo: &pipeline.PluginRuntimeInfo{ - Plugin: inputPlugin, - }, - }) - - // output plugin - anyPlugin, _ = Factory() - outputPlugin := anyPlugin.(*Plugin) - p.SetOutput(&pipeline.OutputPluginInfo{ - PluginStaticInfo: &pipeline.PluginStaticInfo{ - Type: "s3", - Config: configOutput, - }, - PluginRuntimeInfo: &pipeline.PluginRuntimeInfo{ - Plugin: outputPlugin, - }, - }, - ) - return p -} - -func TestStartPanic(t *testing.T) { - test.ClearDir(t, dir) - fileConfig := file.Config{} - config := &Config{ - FileConfig: fileConfig, - CompressionType: "zip", - Endpoint: "some", - AccessKey: "some", - SecretKey: "some", - Bucket: "some", - Secure: false, - client: nil, - } - test.ClearDir(t, dir) - defer test.ClearDir(t, dir) - test.ClearDir(t, config.Bucket) - defer test.ClearDir(t, config.Bucket) - - err := cfg.Parse(config, map[string]int{"gomaxprocs": 1, "capacity": 64}) - assert.NoError(t, err) - p := newPipeline(t, config) - - assert.NotNil(t, p, "could not create new pipeline") - - assert.Panics(t, p.Start) -} - -type mockClientWIthSomeFails struct { - ctx context.Context - Cancel context.CancelFunc -} - -func newMockClientWIthSomeFails() objectStoreClient { - m := mockClientWIthSomeFails{} - ctx, cancel := context.WithCancel(context.Background()) - m.ctx = ctx - m.Cancel = cancel - return m -} - -func (m mockClientWIthSomeFails) BucketExists(bucketName string) (bool, error) { - return true, nil -} - -func (m mockClientWIthSomeFails) FPutObject(bucketName, objectName, filePath string, opts minio.PutObjectOptions) (n int64, err error) { - select { - case <-m.ctx.Done(): - fmt.Println("put object") - - targetDir := fmt.Sprintf("./%s", bucketName) - if _, err := os.Stat(targetDir); os.IsNotExist(err) { - if err := os.MkdirAll(targetDir, os.ModePerm); err != nil { - logger.Fatalf("could not create target dir: %s, error: %s", targetDir, err.Error()) - } - } - fileName = fmt.Sprintf("%s/%s", bucketName, "mockLog.txt") - file, err := os.OpenFile(fileName, os.O_CREATE|os.O_APPEND|os.O_RDWR, os.FileMode(0o777)) - if err != nil { - logger.Panicf("could not open or create file: %s, error: %s", fileName, err.Error()) - } - - if _, err := file.WriteString(fmt.Sprintf("%s | from '%s' to b: `%s` as obj: `%s`\n", time.Now().String(), filePath, bucketName, objectName)); err != nil { - return 0, fmt.Errorf(err.Error()) - } - return 1, nil - default: - return 0, fmt.Errorf("fake could not sent") - } -} - -func TestStartWithSendProblems(t *testing.T) { - if testing.Short() { - t.Skip("skip long tests in short mode") - } - tests := struct { - firstPack []test.Msg - secondPack []test.Msg - thirdPack []test.Msg - }{ - firstPack: []test.Msg{ - test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), - test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), - test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), - }, - secondPack: []test.Msg{ - test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_12","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), - test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_12","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), - test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_12","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), - }, - thirdPack: []test.Msg{ - test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_123","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), - test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_123","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), - test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_123","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), - }, - } - - // pattern for parent log file - pattern := fmt.Sprintf("%s/*.log", dir) - zipPattern := fmt.Sprintf("%s/*.zip", dir) - - writeFileSleep := 100*time.Millisecond + 100*time.Millisecond - sealUpFileSleep := 2*200*time.Millisecond + 500*time.Millisecond - test.ClearDir(t, dir) - s3MockClient := newMockClientWIthSomeFails() - - fileConfig := file.Config{ - TargetFile: targetFile, - RetentionInterval: "300ms", - Layout: "01", - BatchFlushTimeout: "100ms", - } - config := &Config{ - FileConfig: fileConfig, - CompressionType: "zip", - Endpoint: "some", - AccessKey: "some", - SecretKey: "some", - Bucket: "some", - Secure: false, - client: &s3MockClient, - } - test.ClearDir(t, dir) - defer test.ClearDir(t, dir) - test.ClearDir(t, fmt.Sprintf("%s/", config.Bucket)) - defer test.ClearDir(t, config.Bucket) - err := cfg.Parse(config, map[string]int{"gomaxprocs": 1, "capacity": 64}) - assert.NoError(t, err) - p := newPipeline(t, config) - - assert.NotNil(t, p, "could not create new pipeline") - - p.Start() - time.Sleep(300 * time.Microsecond) - test.SendPack(t, p, tests.firstPack) - time.Sleep(writeFileSleep) - time.Sleep(sealUpFileSleep) - - noSentToS3(t) - - matches := test.GetMatches(t, zipPattern) - - assert.Equal(t, 1, len(matches)) - test.CheckNotZero(t, matches[0], "zip file after seal up and compress is not ok") - - matches = test.GetMatches(t, pattern) - assert.Equal(t, 1, len(matches)) - test.CheckZero(t, matches[0], "log file is not empty") - - // initial sending the second pack - // no special situations - test.SendPack(t, p, tests.secondPack) - time.Sleep(writeFileSleep) - time.Sleep(sealUpFileSleep) - - matches = test.GetMatches(t, pattern) - assert.Equal(t, 1, len(matches)) - test.CheckZero(t, matches[0], "log file is not empty") - - // check not empty zips - matches = test.GetMatches(t, zipPattern) - assert.GreaterOrEqual(t, len(matches), 2) - for _, m := range matches { - test.CheckNotZero(t, m, "zip file is empty") - } - - noSentToS3(t) - - // allow sending to s3 - s3Client := s3MockClient.(mockClientWIthSomeFails) - s3Client.Cancel() - - test.SendPack(t, p, tests.thirdPack) - time.Sleep(writeFileSleep) - time.Sleep(sealUpFileSleep) - - maxTimeWait := 5 * sealUpFileSleep - sleep := sealUpFileSleep - for { - time.Sleep(sleep) - // wait deletion - matches = test.GetMatches(t, zipPattern) - if len(matches) == 0 { - logger.Infof("spent %f second of %f second of fake loading", sleep.Seconds(), maxTimeWait.Seconds()) - break - } - - if sleep > maxTimeWait { - logger.Infof("spent %f second of %f second of fake loading", sleep.Seconds(), maxTimeWait.Seconds()) - assert.Fail(t, "restart load is failed") - break - } - sleep += sleep - } - - // chek zip in dir - matches = test.GetMatches(t, zipPattern) - assert.Equal(t, 0, len(matches)) - - // check log file - matches = test.GetMatches(t, pattern) - assert.Equal(t, 1, len(matches)) - test.CheckZero(t, matches[0], "log file is not empty after restart sending") - - // check mock file is not empty and contains more than 3 raws - test.CheckNotZero(t, fileName, "s3 file is empty") - file, err := os.Open(fileName) - assert.NoError(t, err) - defer file.Close() - - scanner := bufio.NewScanner(file) - lineCounter := 0 - for scanner.Scan() { - fmt.Println(scanner.Text()) - lineCounter++ - } - assert.GreaterOrEqual(t, lineCounter, 3) -} - -func noSentToS3(t *testing.T) { - t.Helper() - // check no sent - _, err := os.Stat(fileName) - assert.Error(t, err) - assert.True(t, os.IsNotExist(err)) -} diff --git a/plugin/output/s3/usecase/README.idoc.md b/plugin/output/s3/usecase/README.idoc.md new file mode 100644 index 000000000..a2e3c7abb --- /dev/null +++ b/plugin/output/s3/usecase/README.idoc.md @@ -0,0 +1,5 @@ +# s3 output +@introduction + +### Config params +@config-params|description \ No newline at end of file diff --git a/plugin/output/s3/usecase/README.md b/plugin/output/s3/usecase/README.md new file mode 100755 index 000000000..d42af3d5e --- /dev/null +++ b/plugin/output/s3/usecase/README.md @@ -0,0 +1,120 @@ +# s3 output +Sends events to s3 output of one or multiple buckets. +`bucket` is default bucket for events. Addition buckets can be described in `multi_buckets` section, example down here. +Field "bucket_field_in_event" is filed name, that will be searched in event. +If appears we try to send event to this bucket instead of described here. + +**Example** +Standard example: +```yaml +pipelines: + mkk: + settings: + capacity: 128 + # input plugin is not important in this case, let's emulate http input. + input: + type: http + emulate_mode: "no" + address: ":9200" + actions: + - type: json_decode + field: message + output: + type: s3 + file_plugin: + retention_interval: 10s + # endpoint, access_key, secret_key, bucket are required. + endpoint: "s3.fake_host.org:80" + access_key: "access_key1" + secret_key: "secret_key2" + bucket: "bucket-logs" + bucket_field_in_event: "bucket_name" +``` + +Example with fan-out buckets: +```yaml +pipelines: + mkk: + settings: + capacity: 128 + # input plugin is not important in this case, let's emulate http input. + input: + type: http + emulate_mode: "no" + address: ":9200" + actions: + - type: json_decode + field: message + output: + type: s3 + file_plugin: + retention_interval: 10s + # endpoint, access_key, secret_key, bucket are required. + endpoint: "s3.fake_host.org:80" + access_key: "access_key1" + secret_key: "secret_key2" + bucket: "bucket-logs" + # bucket_field_in_event - event with such field will be sent to bucket with its value + # if such exists: {"bucket_name": "secret", "message": 123} to bucket "secret". + bucket_field_in_event: "bucket_name" + # multi_buckets is optional, contains array of buckets. + multi_buckets: + - endpoint: "otherS3.fake_host.org:80" + access_key: "access_key2" + secret_key: "secret_key2" + bucket: "bucket-logs-2" + - endpoint: "yet_anotherS3.fake_host.ru:80" + access_key: "access_key3" + secret_key: "secret_key3" + bucket: "bucket-logs-3" +``` + +### Config params +**`file_config`** *`file.Config`* +Under the hood this plugin uses /plugin/output/file/ to collect logs + +
+ +**`compression_type`** *`string`* *`default=zip`* *`options=zip`* +Compression type + +
+ +**`endpoint`** *`string`* *`required`* +Endpoint address of default bucket. + +
+ +**`access_key`** *`string`* *`required`* +s3 access key. + +
+ +**`secret_key`** *`string`* *`required`* +s3 secret key. + +
+ +**`bucket`** *`string`* *`required`* +s3 default bucket. + +
+ +**`multi_buckets`** *`[]singleBucketConfig`* +MultiBuckets is additional buckets, which can also receive event. +Event must contain `bucket_name` field which value is s3 bucket name. +Events without `bucket_name` sends to DefaultBucket. + +
+ +**`secure`** *`bool`* *`default=false`* +s3 connection secure option. + +
+ +**`bucket_field_in_event`** *`string`* *`default=bucket_name`* +BucketFieldInEvent field change destination bucket of event to fields value. + +
+ +
*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)* \ No newline at end of file diff --git a/plugin/output/s3/compress.go b/plugin/output/s3/usecase/compress.go similarity index 99% rename from plugin/output/s3/compress.go rename to plugin/output/s3/usecase/compress.go index 0f0b44647..42f619f8f 100644 --- a/plugin/output/s3/compress.go +++ b/plugin/output/s3/usecase/compress.go @@ -1,4 +1,4 @@ -package s3 +package usecase import ( "archive/zip" diff --git a/plugin/output/s3/compress_test.go b/plugin/output/s3/usecase/compress_test.go similarity index 99% rename from plugin/output/s3/compress_test.go rename to plugin/output/s3/usecase/compress_test.go index 4c909fa19..7c1aa1fef 100644 --- a/plugin/output/s3/compress_test.go +++ b/plugin/output/s3/usecase/compress_test.go @@ -1,4 +1,4 @@ -package s3 +package usecase import ( "archive/zip" diff --git a/plugin/output/s3/usecase/s3.go b/plugin/output/s3/usecase/s3.go new file mode 100644 index 000000000..41a3bee41 --- /dev/null +++ b/plugin/output/s3/usecase/s3.go @@ -0,0 +1,457 @@ +package usecase + +import ( + "errors" + "fmt" + "math" + "math/rand" + "os" + "path" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/ozonru/file.d/plugin/output/s3" + + "github.com/minio/minio-go" + + "github.com/ozonru/file.d/fd" + "github.com/ozonru/file.d/longpanic" + "github.com/ozonru/file.d/pipeline" + "github.com/ozonru/file.d/plugin/output/file" + "go.uber.org/zap" +) + +/*{ introduction +Sends events to s3 output of one or multiple buckets. +`bucket` is default bucket for events. Addition buckets can be described in `multi_buckets` section, example down here. +Field "bucket_field_in_event" is filed name, that will be searched in event. +If appears we try to send event to this bucket instead of described here. + +**Example** +Standard example: +```yaml +pipelines: + mkk: + settings: + capacity: 128 + # input plugin is not important in this case, let's emulate http input. + input: + type: http + emulate_mode: "no" + address: ":9200" + actions: + - type: json_decode + field: message + output: + type: s3 + file_plugin: + retention_interval: 10s + # endpoint, access_key, secret_key, bucket are required. + endpoint: "s3.fake_host.org:80" + access_key: "access_key1" + secret_key: "secret_key2" + bucket: "bucket-logs" + bucket_field_in_event: "bucket_name" +``` + +Example with fan-out buckets: +```yaml +pipelines: + mkk: + settings: + capacity: 128 + # input plugin is not important in this case, let's emulate http input. + input: + type: http + emulate_mode: "no" + address: ":9200" + actions: + - type: json_decode + field: message + output: + type: s3 + file_plugin: + retention_interval: 10s + # endpoint, access_key, secret_key, bucket are required. + endpoint: "s3.fake_host.org:80" + access_key: "access_key1" + secret_key: "secret_key2" + bucket: "bucket-logs" + # bucket_field_in_event - event with such field will be sent to bucket with its value + # if such exists: {"bucket_name": "secret", "message": 123} to bucket "secret". + bucket_field_in_event: "bucket_name" + # multi_buckets is optional, contains array of buckets. + multi_buckets: + - endpoint: "otherS3.fake_host.org:80" + access_key: "access_key2" + secret_key: "secret_key2" + bucket: "bucket-logs-2" + - endpoint: "yet_anotherS3.fake_host.ru:80" + access_key: "access_key3" + secret_key: "secret_key3" + bucket: "bucket-logs-3" +``` +}*/ + +const ( + outPluginType = "s3" + fileNameSeparator = "_" + attemptIntervalMin = 1 * time.Second + dirSep = "/" + StaticBucketDir = "static_buckets" + DynamicBucketDir = "dynamic_buckets" +) + +var ( + attemptInterval = attemptIntervalMin + compressors = map[string]func(*zap.SugaredLogger) compressor{ + zipName: newZipCompressor, + } + + r = rand.New(rand.NewSource(time.Now().UnixNano())) +) + +type compressor interface { + compress(archiveName, fileName string) + getObjectOptions() minio.PutObjectOptions + getExtension() string + getName(fileName string) string +} + +type Plugin struct { + controller pipeline.OutputPluginController + logger *zap.SugaredLogger + config *Config + params *pipeline.OutputPluginParams + fileExtension string + + // defaultClient separated from others to avoid locks with dynamic buckets. + defaultClient s3.ObjectStoreClient + clients map[string]s3.ObjectStoreClient + clientsFabric s3.ObjStoreFabricInterface + + outPlugins *file.Plugins + dynamicPlugCreationMu sync.Mutex + + compressCh chan fileDTO + uploadCh chan fileDTO + + compressor compressor +} + +type fileDTO struct { + fileName string + bucketName string +} + +//! config-params +//^ config-params +type Config struct { + //> @3@4@5@6 + //> Under the hood this plugin uses /plugin/output/file/ to collect logs + FileConfig file.Config `json:"file_config" child:"true"` //* + + //> @3@4@5@6 + //> Compression type + CompressionType string `json:"compression_type" default:"zip" options:"zip"` //* + + // s3 section + + //> @3@4@5@6 + //> Endpoint address of default bucket. + Endpoint string `json:"endpoint" required:"true"` //* + //> @3@4@5@6 + //> s3 access key. + AccessKey string `json:"access_key" required:"true"` //* + //> @3@4@5@6 + //> s3 secret key. + SecretKey string `json:"secret_key" required:"true"` //* + //> @3@4@5@6 + //> s3 default bucket. + DefaultBucket string `json:"bucket" required:"true"` //* + //> @3@4@5@6 + //> MultiBuckets is additional buckets, which can also receive event. + //> Event must contain `bucket_name` field which value is s3 bucket name. + //> Events without `bucket_name` sends to DefaultBucket. + MultiBuckets []singleBucketConfig `json:"multi_buckets" required:"false"` //* + //> @3@4@5@6 + //> s3 connection secure option. + Secure bool `json:"secure" default:"false"` //* + //> @3@4@5@6 + //> BucketFieldInEvent field change destination bucket of event to fields value. + // Fallback to DefaultBucket if BucketFieldInEvent bucket doesn't exist. + BucketFieldInEvent string `json:"bucket_field_in_event" default:"bucket_name"` //* +} + +type singleBucketConfig struct { + // s3 section + Endpoint string `json:"endpoint" required:"true"` + AccessKey string `json:"access_key" required:"true"` + SecretKey string `json:"secret_key" required:"true"` + Bucket string `json:"bucket" required:"true"` + Secure bool `json:"secure" default:"false"` + FilePluginInfo file.Config `json:"file_plugin" required:"true"` +} + +func init() { + fd.DefaultPluginRegistry.RegisterOutput(&pipeline.PluginStaticInfo{ + Type: outPluginType, + Factory: Factory, + }) +} + +func Factory() (pipeline.AnyPlugin, pipeline.AnyConfig) { + return &Plugin{}, &Config{} +} + +func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.OutputPluginParams) { + p.StartWithMinio(config, params, p.minioClientsFactory) +} + +func (p *Plugin) StartWithMinio(config pipeline.AnyConfig, params *pipeline.OutputPluginParams, factory objStoreFactory) { + p.controller = params.Controller + p.logger = params.Logger + p.config = config.(*Config) + p.params = params + + // outPlugCount is defaultBucket + multi_buckets count, use to set maps size. + outPlugCount := len(p.config.MultiBuckets) + 1 + + // set up compression. + newCompressor, ok := compressors[p.config.CompressionType] + if !ok { + p.logger.Fatalf("compression type: %s is not supported", p.config.CompressionType) + } + p.compressor = newCompressor(p.logger) + + // dir for all bucket files. + targetDirs := p.getDirs(outPlugCount) + // file for each bucket. + fileNames := p.getFileNames(outPlugCount) + + p.uploadCh = make(chan fileDTO, p.config.FileConfig.WorkersCount_*4) + p.compressCh = make(chan fileDTO, p.config.FileConfig.WorkersCount_) + + for i := 0; i < p.config.FileConfig.WorkersCount_; i++ { + longpanic.Go(p.uploadWork) + longpanic.Go(p.compressWork) + } + + // initialize minio clients. + defaultClient, clients, err := factory(p.config) + if err != nil { + p.logger.Panicf("could not create minio client, error: %s", err.Error()) + } + p.defaultClient = defaultClient + p.clients = clients + err = p.startPlugins(params, outPlugCount, targetDirs, fileNames) + if errors.Is(err, ErrCreateOutputPluginCantCheckBucket) { + p.logger.Panic(err.Error()) + } + if errors.Is(err, ErrCreateOutputPluginNoSuchBucket) { + p.logger.Fatal(err.Error()) + } + + p.uploadExistingFiles(targetDirs, fileNames) +} + +func (p *Plugin) Stop() { + p.outPlugins.Stop() +} + +func (p *Plugin) Out(event *pipeline.Event) { + p.outPlugins.Out(event, pipeline.PluginSelector{ + CondType: pipeline.ByNameSelector, + CondValue: p.decideReceiverBucket(event), + }) +} + +// decideReceiverBucket decide which s3 bucket shall receive event. +func (p *Plugin) decideReceiverBucket(event *pipeline.Event) string { + bucketName := event.Root.Dig(p.config.BucketFieldInEvent).AsString() + // no BucketFieldInEvent in message, it's DefaultBucket, showtime. + + if len(bucketName) == 0 { + return p.config.DefaultBucket + } + // Bucket exists. + if p.outPlugins.Exists(bucketName) { + return bucketName + } + + // Try to create dynamic bucketName. + if created := p.tryRunNewPlugin(bucketName); created { + // Succeed, return new bucketName. + return bucketName + } + + // Failed to create, fallback on DefaultBucket. + return p.config.DefaultBucket +} + +// creates new dynamic plugin if such s3 bucket exists. +func (p *Plugin) tryRunNewPlugin(bucketName string) (isCreated bool) { + // To avoid concurrent creation of bucketName plugin. + p.dynamicPlugCreationMu.Lock() + defer p.dynamicPlugCreationMu.Unlock() + // Probably other worker created plugin concurrently, need check dynamic bucket one more time. + if p.outPlugins.IsDynamic(bucketName) { + return true + } + + defaultBucketClient := p.defaultClient + exists, err := defaultBucketClient.BucketExists(bucketName) + // Fallback to DefaultBucket if we failed check bucket existence. + if err != nil { + p.logger.Errorf("couldn't check bucket from message existence: %s", err.Error()) + return false + } + if !exists { + p.logger.Infof("Bucket %s doesn't exist in s3, message sent to DefaultBucket", bucketName) + return false + } + + dir, _ := filepath.Split(p.config.FileConfig.TargetFile) + bucketDir := filepath.Join(dir, DynamicBucketDir, bucketName) + dirSep + // dynamic bucket share s3 credentials with DefaultBucket. + anyPlugin, _ := file.Factory() + outPlugin := anyPlugin.(*file.Plugin) + outPlugin.SealUpCallback = p.addFileJobWithBucket(bucketName) + + localBucketConfig := p.config.FileConfig + localBucketConfig.TargetFile = fmt.Sprintf("%s%s%s", bucketDir, bucketName, p.fileExtension) + outPlugin.Start(&localBucketConfig, p.params) + + p.outPlugins.Add(bucketName, outPlugin) + + return true +} + +// uploadExistingFiles gets files from dirs, sorts it, compresses it if it's need, and then upload to s3. +func (p *Plugin) uploadExistingFiles(targetDirs, fileNames map[string]string) { + for bucketName, dir := range targetDirs { + // get all compressed files. + pattern := fmt.Sprintf("%s*%s", dir, p.compressor.getExtension()) + compressedFiles, err := filepath.Glob(pattern) + if err != nil { + p.logger.Panicf("could not read dir: %s", dir) + } + // sort compressed files by creation time. + sort.Slice(compressedFiles, p.getSortFunc(compressedFiles)) + // upload archive. + for _, z := range compressedFiles { + splitPath := strings.Split(z, "/") + // multi_buckets are subdirectories of main bucket directory. + // If it's default: /var/log/file-d.log, then bucket "s3SpecialLogs" will have route: + // /var/log/static_buckets/s3SpecialLogs/12345_SpecialLogs.zip + // probableBucket will be found as in multi_buckets. + if len(splitPath) >= 2 { + probableBucket := fileNames[splitPath[len(splitPath)-2]] + _, ok := fileNames[probableBucket] + if ok { + p.uploadCh <- fileDTO{fileName: z, bucketName: probableBucket} + } + } + + // If probableBucket wasn't wound in p.fileNamesAndType, it's archive of main bucket + p.uploadCh <- fileDTO{fileName: z, bucketName: p.config.DefaultBucket} + } + // compress all files that we have in the dir + p.compressFilesInDir(bucketName, targetDirs, fileNames) + } +} + +// compressFilesInDir compresses all files in dir +func (p *Plugin) compressFilesInDir(bucketName string, targetDirs, fileNames map[string]string) { + pattern := fmt.Sprintf("%s/%s%s*%s*%s", targetDirs[bucketName], fileNames[bucketName], fileNameSeparator, fileNameSeparator, p.fileExtension) + files, err := filepath.Glob(pattern) + if err != nil { + p.logger.Panicf("could not read dir: %s", targetDirs[bucketName]) + } + // sort files by creation time. + sort.Slice(files, p.getSortFunc(files)) + for _, f := range files { + p.compressCh <- fileDTO{fileName: f, bucketName: bucketName} + } +} + +// getSortFunc return func that sorts files by mod time +func (p *Plugin) getSortFunc(files []string) func(i, j int) bool { + return func(i, j int) bool { + InfoI, err := os.Stat(files[i]) + if err != nil { + p.logger.Panicf("could not get info about file: %s, error: %s", files[i], err.Error()) + } + InfoJ, err := os.Stat(files[j]) + if err != nil { + p.logger.Panicf("could not get info about file: %s, error: %s", files[j], err.Error()) + } + return InfoI.ModTime().Before(InfoJ.ModTime()) + } +} + +func (p *Plugin) addFileJobWithBucket(bucketName string) func(filename string) { + return func(filename string) { + p.compressCh <- fileDTO{fileName: filename, bucketName: bucketName} + } +} + +func (p *Plugin) uploadWork() { + for compressed := range p.uploadCh { + sleepTime := attemptInterval + for { + err := p.uploadToS3(compressed) + if err == nil { + p.logger.Infof("successfully uploaded object: %s", compressed) + // delete archive after uploading + err = os.Remove(compressed.fileName) + if err != nil { + p.logger.Panicf("could not delete file: %s, err: %s", compressed, err.Error()) + } + break + } + sleepTime += sleepTime + p.logger.Errorf("could not upload object: %s, next attempt in %s, error: %s", compressed, sleepTime.String(), err.Error()) + time.Sleep(sleepTime) + } + } +} + +// compressWork compress file from channel and then delete source file +func (p *Plugin) compressWork() { + for DTO := range p.compressCh { + compressedName := p.compressor.getName(DTO.fileName) + p.compressor.compress(compressedName, DTO.fileName) + // delete old file + if err := os.Remove(DTO.fileName); err != nil { + p.logger.Panicf("could not delete file: %s, error: %s", DTO, err.Error()) + } + DTO.fileName = compressedName + p.uploadCh <- fileDTO{fileName: DTO.fileName, bucketName: DTO.bucketName} + } +} + +func (p *Plugin) uploadToS3(compressedDTO fileDTO) error { + _, err := p.clients[compressedDTO.bucketName].FPutObject( + compressedDTO.bucketName, p.generateObjectName(compressedDTO.fileName), + compressedDTO.fileName, + p.compressor.getObjectOptions(), + ) + if err != nil { + return fmt.Errorf("could not upload file: %s into bucket: %s, error: %s", compressedDTO.fileName, compressedDTO.bucketName, err.Error()) + } + return nil +} + +// generateObjectName generates object name by compressed file name +func (p *Plugin) generateObjectName(name string) string { + n := strconv.FormatInt(r.Int63n(math.MaxInt64), 16) + n = n[len(n)-8:] + objectName := path.Base(name) + objectName = objectName[0 : len(objectName)-len(p.compressor.getExtension())] + return fmt.Sprintf("%s.%s%s", objectName, n, p.compressor.getExtension()) +} diff --git a/plugin/output/s3/usecase/s3_internals.go b/plugin/output/s3/usecase/s3_internals.go new file mode 100644 index 000000000..b8ac6a748 --- /dev/null +++ b/plugin/output/s3/usecase/s3_internals.go @@ -0,0 +1,156 @@ +package usecase + +import ( + "errors" + "fmt" + "io/ioutil" + "path/filepath" + + "github.com/minio/minio-go" + + "github.com/ozonru/file.d/plugin/output/s3" + + "github.com/ozonru/file.d/pipeline" + "github.com/ozonru/file.d/plugin/output/file" +) + +var ( + ErrCreateOutputPluginCantCheckBucket = errors.New("could not check bucket") + ErrCreateOutputPluginNoSuchBucket = errors.New("bucket doesn't exist") +) + +type objStoreFactory func(cfg *Config) (s3.ObjectStoreClient, map[string]s3.ObjectStoreClient, error) + +func (p *Plugin) minioClientsFactory(cfg *Config) (s3.ObjectStoreClient, map[string]s3.ObjectStoreClient, error) { + minioClients := make(map[string]s3.ObjectStoreClient) + // initialize minio clients object for main bucket. + defaultClient, err := minio.New(cfg.Endpoint, cfg.AccessKey, cfg.SecretKey, cfg.Secure) + if err != nil { + return nil, nil, err + } + + for _, singleBucket := range cfg.MultiBuckets { + client, err := minio.New(singleBucket.Endpoint, singleBucket.AccessKey, singleBucket.SecretKey, singleBucket.Secure) + if err != nil { + return nil, nil, err + } + minioClients[singleBucket.Bucket] = client + } + + minioClients[cfg.DefaultBucket] = defaultClient + return defaultClient, minioClients, nil +} + +func (p *Plugin) getDirs(outPlugCount int) map[string]string { + // dir for all bucket files. + dir, _ := filepath.Split(p.config.FileConfig.TargetFile) + + targetDirs := make(map[string]string, outPlugCount) + targetDirs[p.config.DefaultBucket] = dir + // multi_buckets from config are sub dirs on in Config.FileConfig.TargetFile dir. + for _, singleBucket := range p.config.MultiBuckets { + targetDirs[singleBucket.Bucket] = filepath.Join(dir, StaticBucketDir, singleBucket.Bucket) + dirSep + } + return targetDirs +} + +func (p *Plugin) getFileNames(outPlugCount int) map[string]string { + fileNames := make(map[string]string, outPlugCount) + + _, f := filepath.Split(p.config.FileConfig.TargetFile) + p.fileExtension = filepath.Ext(f) + + mainFileName := f[0 : len(f)-len(p.fileExtension)] + fileNames[p.config.DefaultBucket] = mainFileName + for _, singleB := range p.config.MultiBuckets { + fileNames[singleB.Bucket] = singleB.Bucket + } + return fileNames +} + +// Try to create buckets from dirs lying in dynamic_dirs route +func (p *Plugin) createPlugsFromDynamicBucketArtifacts(targetDirs map[string]string) { + dynamicDirsPath := filepath.Join(targetDirs[p.config.DefaultBucket], DynamicBucketDir) + dynamicDir, err := ioutil.ReadDir(dynamicDirsPath) + if err != nil { + p.logger.Infof("%s doesn't exist, willn't restore dynamic s3 buckets", err.Error()) + return + } + for _, f := range dynamicDir { + if !f.IsDir() { + continue + } + created := p.tryRunNewPlugin(f.Name()) + if created { + p.logger.Infof("dynamic bucket '%s' retrieved from artifacts", f.Name()) + } else { + p.logger.Infof("can't retrieve dynamic bucket from artifacts: %s", err.Error()) + } + } +} + +func (p *Plugin) createOutPlugin(bucketName string) (*file.Plugin, error) { + exists, err := p.clients[bucketName].BucketExists(bucketName) + if err != nil { + return nil, fmt.Errorf("%w %s with error: %s", ErrCreateOutputPluginCantCheckBucket, bucketName, err.Error()) + } + if !exists { + return nil, fmt.Errorf("%w %s ", ErrCreateOutputPluginNoSuchBucket, bucketName) + } + + anyPlugin, _ := file.Factory() + outPlugin := anyPlugin.(*file.Plugin) + outPlugin.SealUpCallback = p.addFileJobWithBucket(bucketName) + + return outPlugin, nil +} + +func (p *Plugin) startPlugins(Params *pipeline.OutputPluginParams, outPlugCount int, targetDirs, fileNames map[string]string) error { + outPlugins := make(map[string]file.PluginInterface, outPlugCount) + outPlugin, err := p.createOutPlugin(p.config.DefaultBucket) + if err != nil { + return err + } + outPlugins[p.config.DefaultBucket] = outPlugin + p.logger.Infof("bucket %s exists", p.config.DefaultBucket) + + p.createPlugsFromDynamicBucketArtifacts(targetDirs) + // If multi_buckets described on file.d config, check each of them as well. + for _, singleBucket := range p.config.MultiBuckets { + outPlugin, err := p.createOutPlugin(singleBucket.Bucket) + if err != nil { + return err + } + outPlugins[singleBucket.Bucket] = outPlugin + p.logger.Infof("bucket %s exists", singleBucket.Bucket) + } + + p.logger.Info("outPlugins are ready") + p.outPlugins = file.NewFilePlugins(outPlugins) + + starterMap := make(pipeline.PluginsStarterMap, outPlugCount) + for bucketName := range outPlugins { + var starterData pipeline.PluginsStarterData + // for defaultBucket set it's config. + if bucketName == p.config.DefaultBucket { + starterData = pipeline.PluginsStarterData{ + Config: &p.config.FileConfig, + Params: Params, + } + } else { + // For multi_buckets copy main config and replace file path with bucket sub dir path. + // Example /var/log/file.d.log => /var/log/static_bucket/bucketName/bucketName.log + localBucketConfig := p.config.FileConfig + localBucketConfig.TargetFile = fmt.Sprintf("%s%s%s", targetDirs[bucketName], fileNames[bucketName], p.fileExtension) + starterData = pipeline.PluginsStarterData{ + Config: &localBucketConfig, + Params: Params, + } + } + + starterMap[bucketName] = starterData + } + p.outPlugins.Start(starterMap) + + return nil +} diff --git a/plugin/output/s3/usecase/s3_test.go b/plugin/output/s3/usecase/s3_test.go new file mode 100644 index 000000000..d5ad103ab --- /dev/null +++ b/plugin/output/s3/usecase/s3_test.go @@ -0,0 +1,637 @@ +package usecase + +import ( + "bufio" + "fmt" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ozonru/file.d/plugin/output/s3" + + "github.com/golang/mock/gomock" + mock_s3 "github.com/ozonru/file.d/plugin/output/s3/mock" + + "github.com/minio/minio-go" + "github.com/ozonru/file.d/cfg" + "github.com/ozonru/file.d/logger" + "github.com/ozonru/file.d/pipeline" + "github.com/ozonru/file.d/plugin/input/fake" + "github.com/ozonru/file.d/plugin/output/file" + "github.com/ozonru/file.d/test" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" + "golang.org/x/net/context" +) + +const targetFile = "filetests/log.log" + +var ( + dir, _ = filepath.Split(targetFile) + fileName = "" +) + +func testFactory(objStoreF objStoreFactory) (pipeline.AnyPlugin, pipeline.AnyConfig) { + return &testS3Plugin{objStoreF: objStoreF}, &Config{} +} + +type testS3Plugin struct { + Plugin + objStoreF objStoreFactory +} + +func (p *testS3Plugin) Start(config pipeline.AnyConfig, params *pipeline.OutputPluginParams) { + p.StartWithMinio(config, params, p.objStoreF) +} + +func fPutObjectOk(bucketName, objectName, filePath string, opts minio.PutObjectOptions) (n int64, err error) { + logger.Infof("put object: %s, %s, %s", bucketName, objectName, filePath) + targetDir := fmt.Sprintf("./%s", bucketName) + if _, err := os.Stat(targetDir); os.IsNotExist(err) { + if err := os.MkdirAll(targetDir, os.ModePerm); err != nil { + logger.Fatalf("could not create target dir: %s, error: %s", targetDir, err.Error()) + } + } + fileName = fmt.Sprintf("%s/%s", bucketName, "mockLog.txt") + f, err := os.OpenFile(fileName, os.O_CREATE|os.O_APPEND|os.O_RDWR, os.FileMode(0o777)) + if err != nil { + logger.Panicf("could not open or create file: %s, error: %s", fileName, err.Error()) + } + defer f.Close() + + if _, err := f.WriteString(fmt.Sprintf("%s | from '%s' to b: `%s` as obj: `%s`\n", time.Now().String(), filePath, bucketName, objectName)); err != nil { + return 0, fmt.Errorf(err.Error()) + } + + return 1, nil +} + +type putWithErr struct { + ctx context.Context + cancel context.CancelFunc +} + +func (put *putWithErr) fPutObjectErr(bucketName, objectName, filePath string, opts minio.PutObjectOptions) (n int64, err error) { + select { + case <-put.ctx.Done(): + fmt.Println("put object") + + targetDir := fmt.Sprintf("./%s", bucketName) + if _, err := os.Stat(targetDir); os.IsNotExist(err) { + if err := os.MkdirAll(targetDir, os.ModePerm); err != nil { + logger.Fatalf("could not create target dir: %s, error: %s", targetDir, err.Error()) + } + } + fileName = fmt.Sprintf("%s/%s", bucketName, "mockLog.txt") + file, err := os.OpenFile(fileName, os.O_CREATE|os.O_APPEND|os.O_RDWR, os.FileMode(0o777)) + if err != nil { + logger.Panicf("could not open or create file: %s, error: %s", fileName, err.Error()) + } + + if _, err := file.WriteString(fmt.Sprintf("%s | from '%s' to b: `%s` as obj: `%s`\n", time.Now().String(), filePath, bucketName, objectName)); err != nil { + return 0, fmt.Errorf(err.Error()) + } + return 1, nil + default: + return 0, fmt.Errorf("fake could not sent") + } +} + +func TestStart(t *testing.T) { + if testing.Short() { + t.Skip("skip long tests in short mode") + } + + bucketName := "some" + tests := struct { + firstPack []test.Msg + secondPack []test.Msg + thirdPack []test.Msg + }{ + firstPack: []test.Msg{ + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + }, + secondPack: []test.Msg{ + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_12","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_12","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_12","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + }, + thirdPack: []test.Msg{ + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_123","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_123","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_123","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + }, + } + + // pattern for parent log file + pattern := fmt.Sprintf("%s/*.log", dir) + + test.ClearDir(t, dir) + + ctl := gomock.NewController(t) + defer ctl.Finish() + s3GomockClient := mock_s3.NewMockObjectStoreClient(ctl) + s3GomockClient.EXPECT().BucketExists(bucketName).Return(true, nil).AnyTimes() + s3GomockClient.EXPECT().FPutObject(bucketName, gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(fPutObjectOk).AnyTimes() + + fileConfig := file.Config{ + TargetFile: targetFile, + RetentionInterval: "300ms", + Layout: "01", + BatchFlushTimeout: "100ms", + } + config := &Config{ + FileConfig: fileConfig, + CompressionType: "zip", + Endpoint: bucketName, + AccessKey: bucketName, + SecretKey: bucketName, + DefaultBucket: bucketName, + Secure: false, + } + test.ClearDir(t, dir) + defer test.ClearDir(t, dir) + test.ClearDir(t, config.DefaultBucket) + defer test.ClearDir(t, config.DefaultBucket) + + err := cfg.Parse(config, map[string]int{"gomaxprocs": 1, "capacity": 64}) + assert.NoError(t, err) + + p := newPipeline(t, config, func(cfg *Config) (s3.ObjectStoreClient, map[string]s3.ObjectStoreClient, error) { + return s3GomockClient, map[string]s3.ObjectStoreClient{ + bucketName: s3GomockClient, + }, nil + }) + assert.NotNil(t, p, "could not create new pipeline") + p.Start() + time.Sleep(300 * time.Microsecond) + + test.SendPack(t, p, tests.firstPack) + time.Sleep(time.Second) + size1 := test.CheckNotZero(t, fileName, "s3 data is missed after first pack") + + // check deletion upload log files + match := test.GetMatches(t, pattern) + assert.Equal(t, 1, len(match)) + test.CheckZero(t, match[0], "log file is not nil") + + // initial sending the second pack + // no special situations + test.SendPack(t, p, tests.secondPack) + time.Sleep(time.Second) + + match = test.GetMatches(t, pattern) + assert.Equal(t, 1, len(match)) + test.CheckZero(t, match[0], "log file is not empty") + + size2 := test.CheckNotZero(t, fileName, "s3 data missed after second pack") + assert.True(t, size2 > size1) + + // failed during writing + test.SendPack(t, p, tests.thirdPack) + time.Sleep(200 * time.Millisecond) + p.Stop() + + // check log file not empty + match = test.GetMatches(t, pattern) + assert.Equal(t, 1, len(match)) + test.CheckNotZero(t, match[0], "log file data missed") + // time.Sleep(sealUpFileSleep) + + // restart like after crash + p.Start() + + time.Sleep(time.Second) + + size3 := test.CheckNotZero(t, fileName, "s3 data missed after third pack") + assert.True(t, size3 > size2) +} + +func TestStartWithMultiBuckets(t *testing.T) { + if testing.Short() { + t.Skip("skip long tests in short mode") + } + // will be created. + dynamicBucket := "FAKE_BUCKET" + + buckets := []string{"main", "multi_bucket1", "multi_bucket2", dynamicBucket} + tests := struct { + firstPack []test.Msg + secondPack []test.Msg + thirdPack []test.Msg + }{ + firstPack: []test.Msg{ + // msg to main bucket. + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + // msg to first of multi_buckets. + test.Msg(fmt.Sprintf(`{"bucket_name": "%s", "level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`, buckets[1])), + // msg to second of multi_buckets. + test.Msg(fmt.Sprintf(`{"bucket_name": "%s", "level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`, buckets[2])), + // msg to not exist multi_bucket, will send to main bucket. + test.Msg(fmt.Sprintf(`{"bucket_name": "%s", "level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`, dynamicBucket)), + // msg to defaultBucket. + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + // msg to first of multi_buckets. + test.Msg(fmt.Sprintf(`{"bucket_name": "%s", "level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`, buckets[1])), + // msg to second of multi_buckets. + test.Msg(fmt.Sprintf(`{"bucket_name": "%s", "level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`, buckets[2])), + }, + secondPack: []test.Msg{ + // msg to main bucket. + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + // msg to first of multi_buckets. + test.Msg(fmt.Sprintf(`{"bucket_name": "%s", "level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`, buckets[1])), + // msg to second of multi_buckets. + test.Msg(fmt.Sprintf(`{"bucket_name": "%s", "level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`, buckets[2])), + // msg to not exist multi_bucket, will send to main bucket. + test.Msg(fmt.Sprintf(`{"bucket_name": "%s", "level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`, dynamicBucket)), + // msg to defaultBucket. + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + // msg to first of multi_buckets. + test.Msg(fmt.Sprintf(`{"bucket_name": "%s", "level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`, buckets[1])), + // msg to second of multi_buckets. + test.Msg(fmt.Sprintf(`{"bucket_name": "%s", "level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`, buckets[2])), + }, + thirdPack: []test.Msg{ + // msg to main bucket. + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + // msg to first of multi_buckets. + test.Msg(fmt.Sprintf(`{"bucket_name": "%s", "level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`, buckets[1])), + // msg to second of multi_buckets. + test.Msg(fmt.Sprintf(`{"bucket_name": "%s", "level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`, buckets[2])), + // msg to not exist multi_bucket, will send to main bucket. + test.Msg(fmt.Sprintf(`{"bucket_name": "%s", "level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`, dynamicBucket)), + // msg to defaultBucket. + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + // msg to first of multi_buckets. + test.Msg(fmt.Sprintf(`{"bucket_name": "%s", "level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`, buckets[1])), + // msg to second of multi_buckets. + test.Msg(fmt.Sprintf(`{"bucket_name": "%s", "level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`, buckets[2])), + }, + } + + // patternMain for parent log file + patternMain := fmt.Sprintf("%s/*.log", dir) + patternForMultiBucket1 := fmt.Sprintf("%s%s/%s/*.log", dir, StaticBucketDir, buckets[1]) + patternForMultiBucket2 := fmt.Sprintf("%s%s/%s/*.log", dir, StaticBucketDir, buckets[2]) + patterns := []string{} + patterns = append(append(append(patterns, patternMain), patternForMultiBucket1), patternForMultiBucket2) + + test.ClearDir(t, dir) + ctl := gomock.NewController(t) + defer ctl.Finish() + + s3MockClient := mock_s3.NewMockObjectStoreClient(ctl) + for _, bucketName := range append(append(buckets, dynamicBucket)) { + s3MockClient.EXPECT().BucketExists(bucketName).Return(true, nil).AnyTimes() + s3MockClient.EXPECT().FPutObject(bucketName, gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(fPutObjectOk).AnyTimes() + } + + fileConfig := file.Config{ + TargetFile: targetFile, + RetentionInterval: "300ms", + Layout: "01", + BatchFlushTimeout: "100ms", + } + config := &Config{ + FileConfig: fileConfig, + CompressionType: "zip", + Endpoint: buckets[0], + AccessKey: buckets[0], + SecretKey: buckets[0], + DefaultBucket: buckets[0], + Secure: false, + BucketFieldInEvent: "bucket_name", + MultiBuckets: []singleBucketConfig{ + { + Endpoint: buckets[1], + AccessKey: buckets[1], + SecretKey: buckets[1], + Bucket: buckets[1], + Secure: false, + FilePluginInfo: file.Config{}, + }, + { + Endpoint: buckets[2], + AccessKey: buckets[2], + SecretKey: buckets[2], + Bucket: buckets[2], + Secure: false, + FilePluginInfo: file.Config{}, + }, + }, + } + test.ClearDir(t, dir) + defer test.ClearDir(t, dir) + test.ClearDir(t, buckets[0]) + defer test.ClearDir(t, buckets[0]) + test.ClearDir(t, buckets[1]) + defer test.ClearDir(t, buckets[1]) + test.ClearDir(t, buckets[2]) + defer test.ClearDir(t, buckets[2]) + test.ClearDir(t, dynamicBucket) + defer test.ClearDir(t, dynamicBucket) + + err := cfg.Parse(config, map[string]int{"gomaxprocs": 1, "capacity": 64}) + assert.NoError(t, err) + + p := newPipeline(t, config, func(cfg *Config) (s3.ObjectStoreClient, map[string]s3.ObjectStoreClient, error) { + return s3MockClient, map[string]s3.ObjectStoreClient{ + buckets[0]: s3MockClient, + buckets[1]: s3MockClient, + buckets[2]: s3MockClient, + dynamicBucket: s3MockClient, + }, nil + }) + assert.NotNil(t, p, "could not create new pipeline") + p.Start() + time.Sleep(300 * time.Microsecond) + + test.SendPack(t, p, tests.firstPack) + time.Sleep(time.Second) + size1 := test.CheckNotZero(t, fileName, "s3 data is missed after first pack") + + // check deletion upload log files + for _, pattern := range patterns { + match := test.GetMatches(t, pattern) + assert.Equal(t, 1, len(match), "no matches for: ", pattern) + test.CheckZero(t, match[0], "log file is not nil") + } + + // initial sending the second pack + // no special situations + test.SendPack(t, p, tests.secondPack) + time.Sleep(time.Second) + + for _, pattern := range patterns { + match := test.GetMatches(t, pattern) + assert.Equal(t, 1, len(match)) + test.CheckZero(t, match[0], "log file is not empty") + } + + size2 := test.CheckNotZero(t, fileName, "s3 data missed after second pack") + assert.True(t, size2 > size1) + + // failed during writing + test.SendPack(t, p, tests.thirdPack) + time.Sleep(200 * time.Millisecond) + p.Stop() + + // check log file not empty + for _, pattern := range patterns { + match := test.GetMatches(t, pattern) + assert.Equal(t, 1, len(match)) + test.CheckNotZero(t, match[0], fmt.Sprintf("log file data missed for: %s", pattern)) + } + // time.Sleep(sealUpFileSleep) + + // restart like after crash + p.Start() + + time.Sleep(time.Second) + + size3 := test.CheckNotZero(t, fileName, "s3 data missed after third pack") + assert.True(t, size3 > size2) +} + +func newPipeline(t *testing.T, configOutput *Config, objStoreF objStoreFactory) *pipeline.Pipeline { + t.Helper() + settings := &pipeline.Settings{ + Capacity: 4096, + MaintenanceInterval: time.Second * 100000, + AntispamThreshold: 0, + AvgEventSize: 2048, + StreamField: "stream", + Decoder: "json", + } + + http.DefaultServeMux = &http.ServeMux{} + p := pipeline.New("test_pipeline", settings, prometheus.NewRegistry()) + p.DisableParallelism() + p.EnableEventLog() + + anyPlugin, _ := fake.Factory() + inputPlugin := anyPlugin.(*fake.Plugin) + p.SetInput(&pipeline.InputPluginInfo{ + PluginStaticInfo: &pipeline.PluginStaticInfo{ + Type: "fake", + }, + PluginRuntimeInfo: &pipeline.PluginRuntimeInfo{ + Plugin: inputPlugin, + }, + }) + + // output plugin + anyPlugin, _ = testFactory(objStoreF) + outputPlugin := anyPlugin.(*testS3Plugin) + p.SetOutput(&pipeline.OutputPluginInfo{ + PluginStaticInfo: &pipeline.PluginStaticInfo{ + Type: "s3", + Config: configOutput, + }, + PluginRuntimeInfo: &pipeline.PluginRuntimeInfo{ + Plugin: outputPlugin, + }, + }, + ) + return p +} + +func TestStartPanic(t *testing.T) { + test.ClearDir(t, dir) + fileConfig := file.Config{} + config := &Config{ + FileConfig: fileConfig, + CompressionType: "zip", + Endpoint: "some", + AccessKey: "some", + SecretKey: "some", + DefaultBucket: "some", + Secure: false, + } + test.ClearDir(t, dir) + defer test.ClearDir(t, dir) + test.ClearDir(t, config.DefaultBucket) + defer test.ClearDir(t, config.DefaultBucket) + + err := cfg.Parse(config, map[string]int{"gomaxprocs": 1, "capacity": 64}) + assert.NoError(t, err) + p := newPipeline(t, config, func(cfg *Config) (s3.ObjectStoreClient, map[string]s3.ObjectStoreClient, error) { + return nil, nil, nil + }) + + assert.NotNil(t, p, "could not create new pipeline") + + assert.Panics(t, p.Start) +} + +func TestStartWithSendProblems(t *testing.T) { + if testing.Short() { + t.Skip("skip long tests in short mode") + } + + bucketName := "some" + tests := struct { + firstPack []test.Msg + secondPack []test.Msg + thirdPack []test.Msg + }{ + firstPack: []test.Msg{ + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_1","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + }, + secondPack: []test.Msg{ + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_12","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_12","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_12","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + }, + thirdPack: []test.Msg{ + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_123","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_123","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + test.Msg(`{"level":"error","ts":"2019-08-21T11:43:25.865Z","message":"get_items_error_123","trace_id":"3ea4a6589d06bb3f","span_id":"deddd718684b10a","get_items_error":"product: error while consuming CoverImage: context canceled","get_items_error_option":"CoverImage","get_items_error_cause":"context canceled","get_items_error_cause_type":"context_cancelled"}`), + }, + } + + // pattern for parent log file + pattern := fmt.Sprintf("%s/*.log", dir) + zipPattern := fmt.Sprintf("%s/*.zip", dir) + + writeFileSleep := 100*time.Millisecond + 100*time.Millisecond + sealUpFileSleep := 2*200*time.Millisecond + 500*time.Millisecond + test.ClearDir(t, dir) + ctl := gomock.NewController(t) + defer ctl.Finish() + + s3GoMockClient := mock_s3.NewMockObjectStoreClient(ctl) + s3GoMockClient.EXPECT().BucketExists(bucketName).Return(true, nil).AnyTimes() + + ctx, cancel := context.WithCancel(context.Background()) + withErr := putWithErr{ctx: ctx, cancel: cancel} + s3GoMockClient.EXPECT().FPutObject(bucketName, gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(withErr.fPutObjectErr).AnyTimes() + + fileConfig := file.Config{ + TargetFile: targetFile, + RetentionInterval: "300ms", + Layout: "01", + BatchFlushTimeout: "100ms", + } + config := &Config{ + FileConfig: fileConfig, + CompressionType: "zip", + Endpoint: bucketName, + AccessKey: bucketName, + SecretKey: bucketName, + DefaultBucket: bucketName, + Secure: false, + } + test.ClearDir(t, dir) + defer test.ClearDir(t, dir) + test.ClearDir(t, fmt.Sprintf("%s/", config.DefaultBucket)) + defer test.ClearDir(t, config.DefaultBucket) + err := cfg.Parse(config, map[string]int{"gomaxprocs": 1, "capacity": 64}) + assert.NoError(t, err) + p := newPipeline(t, config, func(cfg *Config) (s3.ObjectStoreClient, map[string]s3.ObjectStoreClient, error) { + return s3GoMockClient, map[string]s3.ObjectStoreClient{ + bucketName: s3GoMockClient, + }, nil + }) + + assert.NotNil(t, p, "could not create new pipeline") + + p.Start() + time.Sleep(300 * time.Microsecond) + test.SendPack(t, p, tests.firstPack) + time.Sleep(writeFileSleep) + time.Sleep(sealUpFileSleep) + + noSentToS3(t) + + matches := test.GetMatches(t, zipPattern) + + assert.Equal(t, 1, len(matches)) + test.CheckNotZero(t, matches[0], "zip file after seal up and compress is not ok") + + matches = test.GetMatches(t, pattern) + assert.Equal(t, 1, len(matches)) + test.CheckZero(t, matches[0], "log file is not empty") + + // initial sending the second pack + // no special situations + test.SendPack(t, p, tests.secondPack) + time.Sleep(writeFileSleep) + time.Sleep(sealUpFileSleep) + + matches = test.GetMatches(t, pattern) + assert.Equal(t, 1, len(matches)) + test.CheckZero(t, matches[0], "log file is not empty") + + // check not empty zips + matches = test.GetMatches(t, zipPattern) + assert.GreaterOrEqual(t, len(matches), 2) + for _, m := range matches { + test.CheckNotZero(t, m, "zip file is empty") + } + + noSentToS3(t) + + cancel() + + test.SendPack(t, p, tests.thirdPack) + time.Sleep(writeFileSleep) + time.Sleep(sealUpFileSleep) + + maxTimeWait := 5 * sealUpFileSleep + sleep := sealUpFileSleep + for { + time.Sleep(sleep) + // wait deletion + matches = test.GetMatches(t, zipPattern) + if len(matches) == 0 { + logger.Infof("spent %f second of %f second of fake loading", sleep.Seconds(), maxTimeWait.Seconds()) + break + } + + if sleep > maxTimeWait { + logger.Infof("spent %f second of %f second of fake loading", sleep.Seconds(), maxTimeWait.Seconds()) + assert.Fail(t, "restart load is failed") + break + } + sleep += sleep + } + + // chek zip in dir + matches = test.GetMatches(t, zipPattern) + assert.Equal(t, 0, len(matches)) + + // check log file + matches = test.GetMatches(t, pattern) + assert.Equal(t, 1, len(matches)) + test.CheckZero(t, matches[0], "log file is not empty after restart sending") + + // check mock file is not empty and contains more than 3 raws + test.CheckNotZero(t, fileName, "s3 file is empty") + f, err := os.Open(fileName) + assert.NoError(t, err) + defer f.Close() + + scanner := bufio.NewScanner(f) + lineCounter := 0 + for scanner.Scan() { + fmt.Println(scanner.Text()) + lineCounter++ + } + assert.GreaterOrEqual(t, lineCounter, 3) +} + +func noSentToS3(t *testing.T) { + t.Helper() + // check no sent + _, err := os.Stat(fileName) + assert.Error(t, err) + assert.True(t, os.IsNotExist(err)) +}