diff --git a/Makefile b/Makefile index 2e9b8d803..282534a0f 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -VERSION ?= 0.1.15 +VERSION ?= 0.2.2 UPSTREAM_BRANCH ?= origin/master .PHONY: prepare @@ -21,6 +21,12 @@ test: go test ./pipeline/ -v -count 1 go test ./plugin/... -v -count 1 +.PHONY: test-short +test-short: + go test ./fd/ -v -count 1 -short + go test ./pipeline/ -v -count 1 -short + go test ./plugin/... -v -count 1 -short + .PHONY: test-e2e test-e2e: go test ./cmd/ -v -count 1 @@ -66,4 +72,5 @@ lint: .PHONY: mock mock: - go generate mock_gen.go + go install github.com/golang/mock/mockgen + mockgen -source=plugin/output/s3/s3.go -destination=plugin/output/s3/mock/s3.go diff --git a/README.idoc.md b/README.idoc.md index 4de9e05a2..9e23c613a 100644 --- a/README.idoc.md +++ b/README.idoc.md @@ -1,10 +1,10 @@ ![file.d](/static/file.d.png) # Overview -[![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/ozonru/file.d/graphs/commit-activity) +[![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/ozontech/file.d/graphs/commit-activity) [![CI](https://github.com/Snyssfx/file.d/actions/workflows/go.yml/badge.svg)](https://github.com/Snyssfx/file.d/actions/workflows/go.yml) -[![GitHub go.mod Go version of a Go module](https://img.shields.io/github/go-mod/go-version/ozonru/file.d)](https://github.com/ozonru/file.d) -[![GoReportCard example](https://goreportcard.com/badge/github.com/ozonru/file.d)](https://goreportcard.com/report/github.com/ozonru/file.d) +[![GitHub go.mod Go version of a Go module](https://img.shields.io/github/go-mod/go-version/ozontech/file.d)](https://github.com/ozontech/file.d) +[![GoReportCard example](https://goreportcard.com/badge/github.com/ozontech/file.d)](https://goreportcard.com/report/github.com/ozontech/file.d) `file.d` is a blazing fast tool for building data pipelines: read, process, and output events. Primarily developed to read from files, but also supports numerous input/action/output plugins. diff --git a/README.md b/README.md index c7d6f83e6..4e3c5e809 100755 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ ![file.d](/static/file.d.png) # Overview -[![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/ozonru/file.d/graphs/commit-activity) +[![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/ozontech/file.d/graphs/commit-activity) [![CI](https://github.com/Snyssfx/file.d/actions/workflows/go.yml/badge.svg)](https://github.com/Snyssfx/file.d/actions/workflows/go.yml) -[![GitHub go.mod Go version of a Go module](https://img.shields.io/github/go-mod/go-version/ozonru/file.d)](https://github.com/ozonru/file.d) -[![GoReportCard example](https://goreportcard.com/badge/github.com/ozonru/file.d)](https://goreportcard.com/report/github.com/ozonru/file.d) +[![GitHub go.mod Go version of a Go module](https://img.shields.io/github/go-mod/go-version/ozontech/file.d)](https://github.com/ozontech/file.d) +[![GoReportCard example](https://goreportcard.com/badge/github.com/ozontech/file.d)](https://goreportcard.com/report/github.com/ozontech/file.d) `file.d` is a blazing fast tool for building data pipelines: read, process, and output events. Primarily developed to read from files, but also supports numerous input/action/output plugins. diff --git a/_coverpage.md b/_coverpage.md index 63a48597b..025f3d36e 100644 --- a/_coverpage.md +++ b/_coverpage.md @@ -2,7 +2,7 @@ > A blazing fast tool for building data pipelines: read, process, and output events. -[GitHub](https://github.com/ozonru/file.d) +[GitHub](https://github.com/ozontech/file.d) [Get started](/README.md) diff --git a/_sidebar.md b/_sidebar.md index af4e71a7d..b6b344dfb 100644 --- a/_sidebar.md +++ b/_sidebar.md @@ -43,7 +43,7 @@ - [elasticsearch](plugin/output/elasticsearch/README.md) - [gelf](plugin/output/gelf/README.md) - [kafka](plugin/output/kafka/README.md) - - [s3](plugin/output/s3/usecase/README.md) + - [s3](plugin/output/s3/README.md) - [splunk](plugin/output/splunk/README.md) - [stdout](plugin/output/stdout/README.md) diff --git a/cfg/config.go b/cfg/config.go index 523425dbd..6e32d77d2 100644 --- a/cfg/config.go +++ b/cfg/config.go @@ -12,7 +12,7 @@ import ( "github.com/bitly/go-simplejson" "github.com/ghodss/yaml" - "github.com/ozonru/file.d/logger" + "github.com/ozontech/file.d/logger" ) type Config struct { diff --git a/cmd/file.d.go b/cmd/file.d.go index d2f583535..0e9f5e504 100644 --- a/cmd/file.d.go +++ b/cmd/file.d.go @@ -7,44 +7,44 @@ import ( "syscall" "github.com/alecthomas/kingpin" - "github.com/ozonru/file.d/cfg" - "github.com/ozonru/file.d/fd" - "github.com/ozonru/file.d/logger" - "github.com/ozonru/file.d/longpanic" - "github.com/ozonru/file.d/pipeline" + "github.com/ozontech/file.d/cfg" + "github.com/ozontech/file.d/fd" + "github.com/ozontech/file.d/logger" + "github.com/ozontech/file.d/longpanic" + "github.com/ozontech/file.d/pipeline" insaneJSON "github.com/vitkovskii/insane-json" "go.uber.org/automaxprocs/maxprocs" - _ "github.com/ozonru/file.d/plugin/action/add_host" - _ "github.com/ozonru/file.d/plugin/action/convert_date" - _ "github.com/ozonru/file.d/plugin/action/debug" - _ "github.com/ozonru/file.d/plugin/action/discard" - _ "github.com/ozonru/file.d/plugin/action/flatten" - _ "github.com/ozonru/file.d/plugin/action/join" - _ "github.com/ozonru/file.d/plugin/action/json_decode" - _ "github.com/ozonru/file.d/plugin/action/keep_fields" - _ "github.com/ozonru/file.d/plugin/action/mask" - _ "github.com/ozonru/file.d/plugin/action/modify" - _ "github.com/ozonru/file.d/plugin/action/parse_es" - _ "github.com/ozonru/file.d/plugin/action/parse_re2" - _ "github.com/ozonru/file.d/plugin/action/remove_fields" - _ "github.com/ozonru/file.d/plugin/action/rename" - _ "github.com/ozonru/file.d/plugin/action/throttle" - _ "github.com/ozonru/file.d/plugin/input/dmesg" - _ "github.com/ozonru/file.d/plugin/input/fake" - _ "github.com/ozonru/file.d/plugin/input/file" - _ "github.com/ozonru/file.d/plugin/input/http" - _ "github.com/ozonru/file.d/plugin/input/journalctl" - _ "github.com/ozonru/file.d/plugin/input/k8s" - _ "github.com/ozonru/file.d/plugin/input/kafka" - _ "github.com/ozonru/file.d/plugin/output/devnull" - _ "github.com/ozonru/file.d/plugin/output/elasticsearch" - _ "github.com/ozonru/file.d/plugin/output/file" - _ "github.com/ozonru/file.d/plugin/output/gelf" - _ "github.com/ozonru/file.d/plugin/output/kafka" - _ "github.com/ozonru/file.d/plugin/output/s3" - _ "github.com/ozonru/file.d/plugin/output/splunk" - _ "github.com/ozonru/file.d/plugin/output/stdout" + _ "github.com/ozontech/file.d/plugin/action/add_host" + _ "github.com/ozontech/file.d/plugin/action/convert_date" + _ "github.com/ozontech/file.d/plugin/action/debug" + _ "github.com/ozontech/file.d/plugin/action/discard" + _ "github.com/ozontech/file.d/plugin/action/flatten" + _ "github.com/ozontech/file.d/plugin/action/join" + _ "github.com/ozontech/file.d/plugin/action/json_decode" + _ "github.com/ozontech/file.d/plugin/action/keep_fields" + _ "github.com/ozontech/file.d/plugin/action/mask" + _ "github.com/ozontech/file.d/plugin/action/modify" + _ "github.com/ozontech/file.d/plugin/action/parse_es" + _ "github.com/ozontech/file.d/plugin/action/parse_re2" + _ "github.com/ozontech/file.d/plugin/action/remove_fields" + _ "github.com/ozontech/file.d/plugin/action/rename" + _ "github.com/ozontech/file.d/plugin/action/throttle" + _ "github.com/ozontech/file.d/plugin/input/dmesg" + _ "github.com/ozontech/file.d/plugin/input/fake" + _ "github.com/ozontech/file.d/plugin/input/file" + _ "github.com/ozontech/file.d/plugin/input/http" + _ "github.com/ozontech/file.d/plugin/input/journalctl" + _ "github.com/ozontech/file.d/plugin/input/k8s" + _ "github.com/ozontech/file.d/plugin/input/kafka" + _ "github.com/ozontech/file.d/plugin/output/devnull" + _ "github.com/ozontech/file.d/plugin/output/elasticsearch" + _ "github.com/ozontech/file.d/plugin/output/file" + _ "github.com/ozontech/file.d/plugin/output/gelf" + _ "github.com/ozontech/file.d/plugin/output/kafka" + _ "github.com/ozontech/file.d/plugin/output/s3" + _ "github.com/ozontech/file.d/plugin/output/splunk" + _ "github.com/ozontech/file.d/plugin/output/stdout" ) var ( diff --git a/cmd/file.d_test.go b/cmd/file.d_test.go index 245172905..5bcb2abea 100644 --- a/cmd/file.d_test.go +++ b/cmd/file.d_test.go @@ -4,6 +4,8 @@ package main import ( "fmt" + "github.com/ozontech/file.d/pipeline" + "github.com/stretchr/testify/require" "io/ioutil" "math/rand" "os" @@ -13,18 +15,18 @@ import ( "testing" "time" - "github.com/ozonru/file.d/cfg" - "github.com/ozonru/file.d/fd" - _ "github.com/ozonru/file.d/plugin/action/discard" - _ "github.com/ozonru/file.d/plugin/action/json_decode" - _ "github.com/ozonru/file.d/plugin/action/keep_fields" - _ "github.com/ozonru/file.d/plugin/action/rename" - _ "github.com/ozonru/file.d/plugin/action/throttle" - _ "github.com/ozonru/file.d/plugin/input/fake" - _ "github.com/ozonru/file.d/plugin/input/file" - k8s2 "github.com/ozonru/file.d/plugin/input/k8s" - _ "github.com/ozonru/file.d/plugin/output/devnull" - _ "github.com/ozonru/file.d/plugin/output/kafka" + "github.com/ozontech/file.d/cfg" + "github.com/ozontech/file.d/fd" + _ "github.com/ozontech/file.d/plugin/action/discard" + _ "github.com/ozontech/file.d/plugin/action/json_decode" + _ "github.com/ozontech/file.d/plugin/action/keep_fields" + _ "github.com/ozontech/file.d/plugin/action/rename" + _ "github.com/ozontech/file.d/plugin/action/throttle" + _ "github.com/ozontech/file.d/plugin/input/fake" + _ "github.com/ozontech/file.d/plugin/input/file" + k8s2 "github.com/ozontech/file.d/plugin/input/k8s" + _ "github.com/ozontech/file.d/plugin/output/devnull" + _ "github.com/ozontech/file.d/plugin/output/kafka" uuid "github.com/satori/go.uuid" ) @@ -150,3 +152,61 @@ func runWriter(tempDir string, files int) { } } } + +/* +Plugins registered automatically after importing by init() function: + _ "github.com/ozontech/file.d/plugin/output/devnull" + _ "github.com/ozontech/file.d/plugin/output/elasticsearch" +Moving plugin in sub dir in plugin dir will quit registration quietly. +To prevent this let's check that DefaultPluginRegistry contains all plugins. +Plugins "dmesg", "journalctl" linux based, they contain tag: //go:build linux. +We don't check them. +*/ +func TestThatPluginsAreImported(t *testing.T) { + action := []string{ + "add_host", + "debug", + "discard", + "flatten", + "json_decode", + "keep_fields", + "mask", + "modify", + "parse_es", + "parse_re2", + "remove_fields", + "rename", + "throttle", + } + for _, pluginName := range action { + pluginInfo := fd.DefaultPluginRegistry.Get(pipeline.PluginKindAction, pluginName) + require.NotNil(t, pluginInfo) + } + + input := []string{ + "fake", + "file", + "http", + "k8s", + "kafka", + } + for _, pluginName := range input { + pluginInfo := fd.DefaultPluginRegistry.Get(pipeline.PluginKindInput, pluginName) + require.NotNil(t, pluginInfo) + } + + output := []string{ + "devnull", + "elasticsearch", + "file", + "gelf", + "kafka", + "s3", + "splunk", + "stdout", + } + for _, pluginName := range output { + pluginInfo := fd.DefaultPluginRegistry.Get(pipeline.PluginKindOutput, pluginName) + require.NotNil(t, pluginInfo) + } +} diff --git a/docs/installation.md b/docs/installation.md index 97b189c82..aa814daaf 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -2,10 +2,10 @@ ## Docker -Images are available on [docker hub](https://hub.docker.com/r/ozonru/file.d). +Images are available on [docker hub](https://hub.docker.com/r/ozontech/file.d). Mount config from /my-config.yaml and start `file.d` with this config:
-`docker run -v /my-config.yaml:/my-config.yaml ozonru/file.d:latest-linux-amd64 /file.d/file.d --config /my-config.yaml` +`docker run -v /my-config.yaml:/my-config.yaml ozontech/file.d:latest-linux-amd64 /file.d/file.d --config /my-config.yaml` ## Downloads To be filled diff --git a/docs/quick-start.md b/docs/quick-start.md index c6c1938d7..695931703 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -7,13 +7,13 @@ ## Steps 1. Put configuration file on your HDD:
-`curl 'https://raw.githubusercontent.com/ozonru/file.d/master/testdata/config/welcome.yaml' > /tmp/welcome.yaml` +`curl 'https://raw.githubusercontent.com/ozontech/file.d/master/testdata/config/welcome.yaml' > /tmp/welcome.yaml` 2. Put test data file on your HDD:
-`curl 'https://raw.githubusercontent.com/ozonru/file.d/master/testdata/json/welcome.json' > /tmp/welcome.json` +`curl 'https://raw.githubusercontent.com/ozontech/file.d/master/testdata/json/welcome.json' > /tmp/welcome.json` 3. Run `file.d`:
-`docker run -v /tmp:/tmp ozonru/file.d:latest-linux-amd64 /file.d/file.d --config /tmp/welcome.yaml` +`docker run -v /tmp:/tmp ozontech/file.d:latest-linux-amd64 /file.d/file.d --config /tmp/welcome.yaml` 4. `file.d` will use data file as an input and your terminal as an output. So you'll see a welcome message. diff --git a/fd/file.d.go b/fd/file.d.go index c6dd1c35c..a38f649b4 100644 --- a/fd/file.d.go +++ b/fd/file.d.go @@ -10,10 +10,10 @@ import ( "runtime/debug" "github.com/bitly/go-simplejson" - "github.com/ozonru/file.d/cfg" - "github.com/ozonru/file.d/logger" - "github.com/ozonru/file.d/longpanic" - "github.com/ozonru/file.d/pipeline" + "github.com/ozontech/file.d/cfg" + "github.com/ozontech/file.d/logger" + "github.com/ozontech/file.d/longpanic" + "github.com/ozontech/file.d/pipeline" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) diff --git a/fd/plugin_registry.go b/fd/plugin_registry.go index 9db78e52c..7c1e9933a 100644 --- a/fd/plugin_registry.go +++ b/fd/plugin_registry.go @@ -1,8 +1,8 @@ package fd import ( - "github.com/ozonru/file.d/logger" - "github.com/ozonru/file.d/pipeline" + "github.com/ozontech/file.d/logger" + "github.com/ozontech/file.d/pipeline" ) var DefaultPluginRegistry = &PluginRegistry{ diff --git a/fd/util.go b/fd/util.go index 767c563dc..3c29efda0 100644 --- a/fd/util.go +++ b/fd/util.go @@ -5,9 +5,9 @@ import ( "time" "github.com/bitly/go-simplejson" - "github.com/ozonru/file.d/cfg" - "github.com/ozonru/file.d/logger" - "github.com/ozonru/file.d/pipeline" + "github.com/ozontech/file.d/cfg" + "github.com/ozontech/file.d/logger" + "github.com/ozontech/file.d/pipeline" ) func extractPipelineParams(settings *simplejson.Json) *pipeline.Settings { diff --git a/go.mod b/go.mod index 542e4466d..fe2900c6a 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/ozonru/file.d +module github.com/ozontech/file.d go 1.17 @@ -37,9 +37,8 @@ require ( k8s.io/utils v0.0.0-20190829053155-3a4a5477acf8 // indirect ) -require github.com/golang/mock v1.6.0 - require ( + github.com/golang/mock v1.6.0 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v3 v3.0.0 // indirect @@ -84,11 +83,13 @@ require ( github.com/spf13/pflag v1.0.3 // indirect go.uber.org/multierr v1.5.0 // indirect golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect + golang.org/x/mod v0.4.2 // indirect golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect golang.org/x/text v0.3.6 // indirect golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect golang.org/x/tools v0.1.1 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.4.0 // indirect google.golang.org/protobuf v1.25.0 // indirect gopkg.in/square/go-jose.v2 v2.5.1 // indirect diff --git a/index.html b/index.html index b81b3de5c..c2fa8350b 100644 --- a/index.html +++ b/index.html @@ -14,7 +14,7 @@