[mbuckets-s3] multiple buckets

Makefile

@@ -1,4 +1,4 @@
-VERSION ?= 0.1.15
+VERSION ?= 0.2.2
 UPSTREAM_BRANCH ?= origin/master
 
 .PHONY: prepare
@@ -21,6 +21,12 @@ test:
 	go test ./pipeline/ -v -count 1
 	go test ./plugin/... -v -count 1
 
+.PHONY: test-short
+test-short:
+	go test ./fd/ -v -count 1 -short
+	go test ./pipeline/ -v -count 1 -short
+	go test ./plugin/... -v -count 1 -short
+
 .PHONY: test-e2e
 test-e2e:
 	go test ./cmd/ -v -count 1
@@ -66,4 +72,5 @@ lint:
 
 .PHONY: mock
 mock:
-	go generate mock_gen.go
+	go install github.com/golang/mock/mockgen
+	mockgen -source=plugin/output/s3/s3.go -destination=plugin/output/s3/mock/s3.go

README.idoc.md

@@ -1,10 +1,10 @@
 ![file.d](/static/file.d.png)
 
 # Overview
-[![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/ozonru/file.d/graphs/commit-activity)
+[![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/ozontech/file.d/graphs/commit-activity)
 [![CI](https://github.com/Snyssfx/file.d/actions/workflows/go.yml/badge.svg)](https://github.com/Snyssfx/file.d/actions/workflows/go.yml)
-[![GitHub go.mod Go version of a Go module](https://img.shields.io/github/go-mod/go-version/ozonru/file.d)](https://github.com/ozonru/file.d)
-[![GoReportCard example](https://goreportcard.com/badge/github.com/ozonru/file.d)](https://goreportcard.com/report/github.com/ozonru/file.d)
+[![GitHub go.mod Go version of a Go module](https://img.shields.io/github/go-mod/go-version/ozontech/file.d)](https://github.com/ozontech/file.d)
+[![GoReportCard example](https://goreportcard.com/badge/github.com/ozontech/file.d)](https://goreportcard.com/report/github.com/ozontech/file.d)
 
 `file.d` is a blazing fast tool for building data pipelines: read, process, and output events. Primarily developed to read from files, but also supports numerous input/action/output plugins. 
 

README.md

@@ -1,10 +1,10 @@
 ![file.d](/static/file.d.png)
 
 # Overview
-[![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/ozonru/file.d/graphs/commit-activity)
+[![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/ozontech/file.d/graphs/commit-activity)
 [![CI](https://github.com/Snyssfx/file.d/actions/workflows/go.yml/badge.svg)](https://github.com/Snyssfx/file.d/actions/workflows/go.yml)
-[![GitHub go.mod Go version of a Go module](https://img.shields.io/github/go-mod/go-version/ozonru/file.d)](https://github.com/ozonru/file.d)
-[![GoReportCard example](https://goreportcard.com/badge/github.com/ozonru/file.d)](https://goreportcard.com/report/github.com/ozonru/file.d)
+[![GitHub go.mod Go version of a Go module](https://img.shields.io/github/go-mod/go-version/ozontech/file.d)](https://github.com/ozontech/file.d)
+[![GoReportCard example](https://goreportcard.com/badge/github.com/ozontech/file.d)](https://goreportcard.com/report/github.com/ozontech/file.d)
 
 `file.d` is a blazing fast tool for building data pipelines: read, process, and output events. Primarily developed to read from files, but also supports numerous input/action/output plugins. 
 

_coverpage.md

@@ -2,7 +2,7 @@
 
 > A blazing fast tool for building data pipelines: read, process, and output events. 
 
-[GitHub](https://github.com/ozonru/file.d)
+[GitHub](https://github.com/ozontech/file.d)
 [Get started](/README.md)
 
 

_sidebar.md

@@ -43,7 +43,7 @@
     - [elasticsearch](plugin/output/elasticsearch/README.md)
     - [gelf](plugin/output/gelf/README.md)
     - [kafka](plugin/output/kafka/README.md)
-    - [s3](plugin/output/s3/usecase/README.md)
+    - [s3](plugin/output/s3/README.md)
     - [splunk](plugin/output/splunk/README.md)
     - [stdout](plugin/output/stdout/README.md)
 

cfg/config.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/bitly/go-simplejson"
 	"github.com/ghodss/yaml"
-	"github.com/ozonru/file.d/logger"
+	"github.com/ozontech/file.d/logger"
 )
 
 type Config struct {

cmd/file.d.go

@@ -7,44 +7,44 @@ import (
 	"syscall"
 
 	"github.com/alecthomas/kingpin"
-	"github.com/ozonru/file.d/cfg"
-	"github.com/ozonru/file.d/fd"
-	"github.com/ozonru/file.d/logger"
-	"github.com/ozonru/file.d/longpanic"
-	"github.com/ozonru/file.d/pipeline"
+	"github.com/ozontech/file.d/cfg"
+	"github.com/ozontech/file.d/fd"
+	"github.com/ozontech/file.d/logger"
+	"github.com/ozontech/file.d/longpanic"
+	"github.com/ozontech/file.d/pipeline"
 	insaneJSON "github.com/vitkovskii/insane-json"
 	"go.uber.org/automaxprocs/maxprocs"
 
-	_ "github.com/ozonru/file.d/plugin/action/add_host"
-	_ "github.com/ozonru/file.d/plugin/action/convert_date"
-	_ "github.com/ozonru/file.d/plugin/action/debug"
-	_ "github.com/ozonru/file.d/plugin/action/discard"
-	_ "github.com/ozonru/file.d/plugin/action/flatten"
-	_ "github.com/ozonru/file.d/plugin/action/join"
-	_ "github.com/ozonru/file.d/plugin/action/json_decode"
-	_ "github.com/ozonru/file.d/plugin/action/keep_fields"
-	_ "github.com/ozonru/file.d/plugin/action/mask"
-	_ "github.com/ozonru/file.d/plugin/action/modify"
-	_ "github.com/ozonru/file.d/plugin/action/parse_es"
-	_ "github.com/ozonru/file.d/plugin/action/parse_re2"
-	_ "github.com/ozonru/file.d/plugin/action/remove_fields"
-	_ "github.com/ozonru/file.d/plugin/action/rename"
-	_ "github.com/ozonru/file.d/plugin/action/throttle"
-	_ "github.com/ozonru/file.d/plugin/input/dmesg"
-	_ "github.com/ozonru/file.d/plugin/input/fake"
-	_ "github.com/ozonru/file.d/plugin/input/file"
-	_ "github.com/ozonru/file.d/plugin/input/http"
-	_ "github.com/ozonru/file.d/plugin/input/journalctl"
-	_ "github.com/ozonru/file.d/plugin/input/k8s"
-	_ "github.com/ozonru/file.d/plugin/input/kafka"
-	_ "github.com/ozonru/file.d/plugin/output/devnull"
-	_ "github.com/ozonru/file.d/plugin/output/elasticsearch"
-	_ "github.com/ozonru/file.d/plugin/output/file"
-	_ "github.com/ozonru/file.d/plugin/output/gelf"
-	_ "github.com/ozonru/file.d/plugin/output/kafka"
-	_ "github.com/ozonru/file.d/plugin/output/s3"
-	_ "github.com/ozonru/file.d/plugin/output/splunk"
-	_ "github.com/ozonru/file.d/plugin/output/stdout"
+	_ "github.com/ozontech/file.d/plugin/action/add_host"
+	_ "github.com/ozontech/file.d/plugin/action/convert_date"
+	_ "github.com/ozontech/file.d/plugin/action/debug"
+	_ "github.com/ozontech/file.d/plugin/action/discard"
+	_ "github.com/ozontech/file.d/plugin/action/flatten"
+	_ "github.com/ozontech/file.d/plugin/action/join"
+	_ "github.com/ozontech/file.d/plugin/action/json_decode"
+	_ "github.com/ozontech/file.d/plugin/action/keep_fields"
+	_ "github.com/ozontech/file.d/plugin/action/mask"
+	_ "github.com/ozontech/file.d/plugin/action/modify"
+	_ "github.com/ozontech/file.d/plugin/action/parse_es"
+	_ "github.com/ozontech/file.d/plugin/action/parse_re2"
+	_ "github.com/ozontech/file.d/plugin/action/remove_fields"
+	_ "github.com/ozontech/file.d/plugin/action/rename"
+	_ "github.com/ozontech/file.d/plugin/action/throttle"
+	_ "github.com/ozontech/file.d/plugin/input/dmesg"
+	_ "github.com/ozontech/file.d/plugin/input/fake"
+	_ "github.com/ozontech/file.d/plugin/input/file"
+	_ "github.com/ozontech/file.d/plugin/input/http"
+	_ "github.com/ozontech/file.d/plugin/input/journalctl"
+	_ "github.com/ozontech/file.d/plugin/input/k8s"
+	_ "github.com/ozontech/file.d/plugin/input/kafka"
+	_ "github.com/ozontech/file.d/plugin/output/devnull"
+	_ "github.com/ozontech/file.d/plugin/output/elasticsearch"
+	_ "github.com/ozontech/file.d/plugin/output/file"
+	_ "github.com/ozontech/file.d/plugin/output/gelf"
+	_ "github.com/ozontech/file.d/plugin/output/kafka"
+	_ "github.com/ozontech/file.d/plugin/output/s3"
+	_ "github.com/ozontech/file.d/plugin/output/splunk"
+	_ "github.com/ozontech/file.d/plugin/output/stdout"
 )
 
 var (

cmd/file.d_test.go

@@ -4,6 +4,8 @@ package main
 
 import (
 	"fmt"
+	"github.com/ozontech/file.d/pipeline"
+	"github.com/stretchr/testify/require"
 	"io/ioutil"
 	"math/rand"
 	"os"
@@ -13,18 +15,18 @@ import (
 	"testing"
 	"time"
 
-	"github.com/ozonru/file.d/cfg"
-	"github.com/ozonru/file.d/fd"
-	_ "github.com/ozonru/file.d/plugin/action/discard"
-	_ "github.com/ozonru/file.d/plugin/action/json_decode"
-	_ "github.com/ozonru/file.d/plugin/action/keep_fields"
-	_ "github.com/ozonru/file.d/plugin/action/rename"
-	_ "github.com/ozonru/file.d/plugin/action/throttle"
-	_ "github.com/ozonru/file.d/plugin/input/fake"
-	_ "github.com/ozonru/file.d/plugin/input/file"
-	k8s2 "github.com/ozonru/file.d/plugin/input/k8s"
-	_ "github.com/ozonru/file.d/plugin/output/devnull"
-	_ "github.com/ozonru/file.d/plugin/output/kafka"
+	"github.com/ozontech/file.d/cfg"
+	"github.com/ozontech/file.d/fd"
+	_ "github.com/ozontech/file.d/plugin/action/discard"
+	_ "github.com/ozontech/file.d/plugin/action/json_decode"
+	_ "github.com/ozontech/file.d/plugin/action/keep_fields"
+	_ "github.com/ozontech/file.d/plugin/action/rename"
+	_ "github.com/ozontech/file.d/plugin/action/throttle"
+	_ "github.com/ozontech/file.d/plugin/input/fake"
+	_ "github.com/ozontech/file.d/plugin/input/file"
+	k8s2 "github.com/ozontech/file.d/plugin/input/k8s"
+	_ "github.com/ozontech/file.d/plugin/output/devnull"
+	_ "github.com/ozontech/file.d/plugin/output/kafka"
 	uuid "github.com/satori/go.uuid"
 )
 
@@ -150,3 +152,61 @@ func runWriter(tempDir string, files int) {
 		}
 	}
 }
+
+/*
+Plugins registered automatically after importing by init() function:
+	_ "github.com/ozontech/file.d/plugin/output/devnull"
+	_ "github.com/ozontech/file.d/plugin/output/elasticsearch"
+Moving plugin in sub dir in plugin dir will quit registration quietly.
+To prevent this let's check that DefaultPluginRegistry contains all plugins.
+Plugins "dmesg", "journalctl" linux based, they contain tag: //go:build linux.
+We don't check them.
+*/
+func TestThatPluginsAreImported(t *testing.T) {
+	action := []string{
+		"add_host",
+		"debug",
+		"discard",
+		"flatten",
+		"json_decode",
+		"keep_fields",
+		"mask",
+		"modify",
+		"parse_es",
+		"parse_re2",
+		"remove_fields",
+		"rename",
+		"throttle",
+	}
+	for _, pluginName := range action {
+		pluginInfo := fd.DefaultPluginRegistry.Get(pipeline.PluginKindAction, pluginName)
+		require.NotNil(t, pluginInfo)
+	}
+
+	input := []string{
+		"fake",
+		"file",
+		"http",
+		"k8s",
+		"kafka",
+	}
+	for _, pluginName := range input {
+		pluginInfo := fd.DefaultPluginRegistry.Get(pipeline.PluginKindInput, pluginName)
+		require.NotNil(t, pluginInfo)
+	}
+
+	output := []string{
+		"devnull",
+		"elasticsearch",
+		"file",
+		"gelf",
+		"kafka",
+		"s3",
+		"splunk",
+		"stdout",
+	}
+	for _, pluginName := range output {
+		pluginInfo := fd.DefaultPluginRegistry.Get(pipeline.PluginKindOutput, pluginName)
+		require.NotNil(t, pluginInfo)
+	}
+}

docs/installation.md

@@ -2,10 +2,10 @@
 
 ## Docker
 
-Images are available on [docker hub](https://hub.docker.com/r/ozonru/file.d).
+Images are available on [docker hub](https://hub.docker.com/r/ozontech/file.d).
 
 Mount config from /my-config.yaml and start `file.d` with this config: <br>
-`docker run -v /my-config.yaml:/my-config.yaml ozonru/file.d:latest-linux-amd64 /file.d/file.d --config /my-config.yaml`
+`docker run -v /my-config.yaml:/my-config.yaml ozontech/file.d:latest-linux-amd64 /file.d/file.d --config /my-config.yaml`
 
 ## Downloads
 To be filled

docs/quick-start.md

@@ -7,13 +7,13 @@
 
 ## Steps
 1. Put configuration file on your HDD:<br> 
-`curl 'https://raw.githubusercontent.com/ozonru/file.d/master/testdata/config/welcome.yaml' > /tmp/welcome.yaml`
+`curl 'https://raw.githubusercontent.com/ozontech/file.d/master/testdata/config/welcome.yaml' > /tmp/welcome.yaml`
 
 2. Put test data file on your HDD:<br>
-`curl 'https://raw.githubusercontent.com/ozonru/file.d/master/testdata/json/welcome.json' > /tmp/welcome.json`
+`curl 'https://raw.githubusercontent.com/ozontech/file.d/master/testdata/json/welcome.json' > /tmp/welcome.json`
 
 3. Run `file.d`:<br>
-`docker run -v /tmp:/tmp ozonru/file.d:latest-linux-amd64 /file.d/file.d --config /tmp/welcome.yaml`
+`docker run -v /tmp:/tmp ozontech/file.d:latest-linux-amd64 /file.d/file.d --config /tmp/welcome.yaml`
 
 4. `file.d` will use data file as an input and your terminal as an output. So you'll see a welcome message.
 

fd/file.d.go

@@ -10,10 +10,10 @@ import (
 	"runtime/debug"
 
 	"github.com/bitly/go-simplejson"
-	"github.com/ozonru/file.d/cfg"
-	"github.com/ozonru/file.d/logger"
-	"github.com/ozonru/file.d/longpanic"
-	"github.com/ozonru/file.d/pipeline"
+	"github.com/ozontech/file.d/cfg"
+	"github.com/ozontech/file.d/logger"
+	"github.com/ozontech/file.d/longpanic"
+	"github.com/ozontech/file.d/pipeline"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )

fd/plugin_registry.go

@@ -1,8 +1,8 @@
 package fd
 
 import (
-	"github.com/ozonru/file.d/logger"
-	"github.com/ozonru/file.d/pipeline"
+	"github.com/ozontech/file.d/logger"
+	"github.com/ozontech/file.d/pipeline"
 )
 
 var DefaultPluginRegistry = &PluginRegistry{

fd/util.go

@@ -5,9 +5,9 @@ import (
 	"time"
 
 	"github.com/bitly/go-simplejson"
-	"github.com/ozonru/file.d/cfg"
-	"github.com/ozonru/file.d/logger"
-	"github.com/ozonru/file.d/pipeline"
+	"github.com/ozontech/file.d/cfg"
+	"github.com/ozontech/file.d/logger"
+	"github.com/ozontech/file.d/pipeline"
 )
 
 func extractPipelineParams(settings *simplejson.Json) *pipeline.Settings {

go.mod

@@ -1,4 +1,4 @@
-module github.com/ozonru/file.d
+module github.com/ozontech/file.d
 
 go 1.17
 
@@ -37,9 +37,8 @@ require (
 	k8s.io/utils v0.0.0-20190829053155-3a4a5477acf8 // indirect
 )
 
-require github.com/golang/mock v1.6.0
-
 require (
+	github.com/golang/mock v1.6.0
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v3 v3.0.0 // indirect
@@ -84,11 +83,13 @@ require (
 	github.com/spf13/pflag v1.0.3 // indirect
 	go.uber.org/multierr v1.5.0 // indirect
 	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect
+	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
 	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
 	golang.org/x/text v0.3.6 // indirect
 	golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect
 	golang.org/x/tools v0.1.1 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.4.0 // indirect
 	google.golang.org/protobuf v1.25.0 // indirect
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect

index.html

@@ -14,7 +14,7 @@
   <script>
     window.$docsify = {
       name: 'file.d',
-      repo: 'https://github.com/ozonru/file.d',
+      repo: 'https://github.com/ozontech/file.d',
       loadSidebar: true,
       loadNavbar: true,
       subMaxLevel: 2,

longpanic/longpanic.go

@@ -5,7 +5,7 @@ package longpanic
 import (
 	"time"
 
-	"github.com/ozonru/file.d/logger"
+	"github.com/ozontech/file.d/logger"
 	"go.uber.org/atomic"
 )
 

pipeline/antispamer.go

@@ -4,7 +4,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/ozonru/file.d/logger"
+	"github.com/ozontech/file.d/logger"
 	"go.uber.org/atomic"
 )