From 66e981eee4faccb3920a4259f2fa4ec32f748992 Mon Sep 17 00:00:00 2001 From: Devin Collins Date: Tue, 14 Nov 2023 09:39:40 -0800 Subject: [PATCH] Initial commit --- .github/workflows/release.yaml | 13 +++ .github/workflows/test.yaml | 18 ++++ README.md | 50 +++++++++++ config.go | 8 ++ config_test.go | 13 +++ doc.go | 3 + factory.go | 31 +++++++ factory_test.go | 29 ++++++ go.mod | 44 +++++++++ go.sum | 125 ++++++++++++++++++++++++++ internal/errwrap/errwrap.go | 28 ++++++ internal/metadata/generated_status.go | 12 +++ metadata.yaml | 9 ++ processor.go | 71 +++++++++++++++ processor_test.go | 70 +++++++++++++++ 15 files changed, 524 insertions(+) create mode 100644 .github/workflows/release.yaml create mode 100644 .github/workflows/test.yaml create mode 100644 README.md create mode 100644 config.go create mode 100644 config_test.go create mode 100644 doc.go create mode 100644 factory.go create mode 100644 factory_test.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/errwrap/errwrap.go create mode 100644 internal/metadata/generated_status.go create mode 100644 metadata.yaml create mode 100644 processor.go create mode 100644 processor_test.go diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..65e1659 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,13 @@ +name: Release + +on: + push: + branches: + - main + +jobs: + release: + uses: jupiterone/.github/.github/workflows/provision_only_release.yml@v3 + secrets: + NPM_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} + AUTO_GITHUB_PAT_TOKEN: ${{ secrets.AUTO_GITHUB_PAT_TOKEN }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..f56315d --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,18 @@ +name: Go Test + +on: [pull_request] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: "1.21.3" + - name: Install dependencies + run: go get . + - name: Run tests + run: go test -v ./... diff --git a/README.md b/README.md new file mode 100644 index 0000000..3cd65dc --- /dev/null +++ b/README.md @@ -0,0 +1,50 @@ +# OpenTelemetry JSON Log Flattener +NewRelic doesn't support nested JSON attributes in OpenTelemetry logs. To help alleviate these issues, this processor will flatten JSON at the top level. It turns messages like this: +```json +{ + "name": "Test User", + "address": { + "street": "First Ave", + "house": 1234 + }, + "occupants": [ + "Test User", + "Test User 2", + "Test User 3" + ] +} +``` +into +```json +{ + "name": "Test User", + "address.street": "First Ave", + "address.house": 1234, + "occupants": "[\"Test User\", \"Test User 2\", \"Test User 3\"]" +} +``` + + +## Using this processor +To add this processor to your OpenTelemetry Collector, follow the instructions for building a collector here: https://opentelemetry.io/docs/collector/custom-collector/ + +In your config, add the following: +```yaml +processors: + - github.com/JupiterOne/otel-jsonlogflattenerprocessor +``` + +In your collector config, add the following processor: +```yaml +receivers: + ... +exporters: + ... +processors: + - oteljsonlogflattenerprocessor +pipelines: + logs: + receivers: [...] + processors: [oteljsonlogflattenerprocessor] + exporters: [...] +``` \ No newline at end of file diff --git a/config.go b/config.go new file mode 100644 index 0000000..6ba1870 --- /dev/null +++ b/config.go @@ -0,0 +1,8 @@ +package oteljsonlogflattenerprocessor + +// Config is very basic and takes no options +type Config struct{} + +func (cfg *Config) Validate() error { + return nil +} diff --git a/config_test.go b/config_test.go new file mode 100644 index 0000000..bedca5b --- /dev/null +++ b/config_test.go @@ -0,0 +1,13 @@ +package oteljsonlogflattenerprocessor + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestValidateConfig(t *testing.T) { + cfg := &Config{} + err := cfg.Validate() + require.NoError(t, err) +} diff --git a/doc.go b/doc.go new file mode 100644 index 0000000..0685a69 --- /dev/null +++ b/doc.go @@ -0,0 +1,3 @@ +//go:generate mdatagen metadata.yaml + +package oteljsonlogflattenerprocessor // import "github.com/jupiterone/oteljsonlogflattenerprocessor" diff --git a/factory.go b/factory.go new file mode 100644 index 0000000..9d0316d --- /dev/null +++ b/factory.go @@ -0,0 +1,31 @@ +package oteljsonlogflattenerprocessor + +import ( + "context" + "errors" + + "github.com/jupiterone/oteljsonlogflattenerprocessor/internal/metadata" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/processor" +) + +func NewFactory() processor.Factory { + return processor.NewFactory( + metadata.Type, + createDefaultConfig, + processor.WithLogs(createLogsProcessor, metadata.LogsStability), + ) +} + +func createDefaultConfig() component.Config { + return &Config{} +} + +func createLogsProcessor(_ context.Context, set processor.CreateSettings, cfg component.Config, nextConsumer consumer.Logs) (processor.Logs, error) { + pCfg, ok := cfg.(*Config) + if !ok { + return nil, errors.New("could not initialize logs transform processor") + } + return newProcessor(pCfg, nextConsumer, set.Logger) +} diff --git a/factory_test.go b/factory_test.go new file mode 100644 index 0000000..62c3f3c --- /dev/null +++ b/factory_test.go @@ -0,0 +1,29 @@ +package oteljsonlogflattenerprocessor + +import ( + "testing" + + "github.com/jupiterone/oteljsonlogflattenerprocessor/internal/metadata" + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" +) + +func TestFactoryType(t *testing.T) { + factory := NewFactory() + assert.Equal(t, factory.Type(), component.Type(metadata.Type)) +} + +func TestFactoryCreateDefaultConfig(t *testing.T) { + factory := NewFactory() + cfg := factory.CreateDefaultConfig() + assert.Equal(t, cfg, &Config{}) + assert.NoError(t, componenttest.CheckConfigStruct(cfg)) +} + +func TestFactoryCreateProcessorEmpty(t *testing.T) { + factory := NewFactory() + cfg := factory.CreateDefaultConfig() + err := component.ValidateConfig(cfg) + assert.NoError(t, err) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..582f7fa --- /dev/null +++ b/go.mod @@ -0,0 +1,44 @@ +module github.com/jupiterone/oteljsonlogflattenerprocessor + +go 1.21.3 + +require ( + github.com/stretchr/testify v1.8.4 + go.opentelemetry.io/collector/component v0.89.0 + go.opentelemetry.io/collector/consumer v0.89.0 + go.opentelemetry.io/collector/pdata v1.0.0-rcv0018 + go.opentelemetry.io/collector/processor v0.89.0 + go.uber.org/zap v1.26.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.0.1 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.opentelemetry.io/collector v0.89.0 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.89.0 // indirect + go.opentelemetry.io/collector/confmap v0.89.0 // indirect + go.opentelemetry.io/collector/featuregate v1.0.0-rcv0018 // indirect + go.opentelemetry.io/otel v1.20.0 // indirect + go.opentelemetry.io/otel/metric v1.20.0 // indirect + go.opentelemetry.io/otel/trace v1.20.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.13.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/grpc v1.59.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..79dcf70 --- /dev/null +++ b/go.sum @@ -0,0 +1,125 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.0.1 h1:1dYGITt1I23x8cfx8ZnldtezdyaZtfAuRtIFOiRzK7g= +github.com/knadh/koanf/v2 v2.0.1/go.mod h1:ZeiIlIDXTE7w1lMT6UVcNiRAS2/rCeLn/GdLNvY1Dus= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 h1:BpfhmLKZf+SjVanKKhCgf3bg+511DmU9eDQTen7LLbY= +github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/collector v0.89.0 h1:lzpfD9NTHh+1M+qzcoYUH+i2rOgFSox3bGQFUI5BPJg= +go.opentelemetry.io/collector v0.89.0/go.mod h1:UZUtmQ3kai0CLPWvPmHKpmwqqEoo50n1bwzYYhXX0eA= +go.opentelemetry.io/collector/component v0.89.0 h1:PoQJX86BpaSZhzx0deQXHh3QMuW6XKVmolSdTKE506c= +go.opentelemetry.io/collector/component v0.89.0/go.mod h1:ZZncnMVaNs++JIbAMiemUIWLZrZ3PMEzI3S3K8pnkws= +go.opentelemetry.io/collector/config/configtelemetry v0.89.0 h1:NtRknYDfMgP1r8mnByo6qQQK8IBw/lF9Qke5f7VhGZ0= +go.opentelemetry.io/collector/config/configtelemetry v0.89.0/go.mod h1:+LAXM5WFMW/UbTlAuSs6L/W72WC+q8TBJt/6z39FPOU= +go.opentelemetry.io/collector/confmap v0.89.0 h1:N5Vg1+FXEFBHHlGIPg4OSlM9uTHjCI7RlWWrKjtOzWQ= +go.opentelemetry.io/collector/confmap v0.89.0/go.mod h1:D8FMPvuihtVxwXaz/qp5q9X2lq9l97QyjfsdZD1spmc= +go.opentelemetry.io/collector/consumer v0.89.0 h1:MteKhkudX2L1ylbtdpSazO8SwyHSxl6fUEElc0rRLDQ= +go.opentelemetry.io/collector/consumer v0.89.0/go.mod h1:aOaoi6R0qVvfHu0pEPCzSE74gIPNJoCQM8Ml4Bc9NHE= +go.opentelemetry.io/collector/featuregate v1.0.0-rcv0018 h1:iK4muX3KIMqKk0xwKcRzu4ravgCtUdzsvuxxdz6A27g= +go.opentelemetry.io/collector/featuregate v1.0.0-rcv0018/go.mod h1:xGbRuw+GbutRtVVSEy3YR2yuOlEyiUMhN2M9DJljgqY= +go.opentelemetry.io/collector/pdata v1.0.0-rcv0018 h1:a2IHOZKphRzPagcvOHQHHUE0DlITFSKlIBwaWhPZpl4= +go.opentelemetry.io/collector/pdata v1.0.0-rcv0018/go.mod h1:oNIcTRyEJYIfMcRYyyh5lquDU0Vl+ktTL6ka+p+dYvg= +go.opentelemetry.io/collector/processor v0.89.0 h1:ypMnoOqBYbXgbDnAm9/Cb4uN3kxvmI05Vf6o4u/riBU= +go.opentelemetry.io/collector/processor v0.89.0/go.mod h1:HzMQ2VbxaECk7Oy1mHtug4qsl4acAW4XP1hpTgQKv84= +go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc= +go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= +go.opentelemetry.io/otel/metric v1.20.0 h1:ZlrO8Hu9+GAhnepmRGhSU7/VkpjrNowxRN9GyKR4wzA= +go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= +go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ= +go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/errwrap/errwrap.go b/internal/errwrap/errwrap.go new file mode 100644 index 0000000..f085936 --- /dev/null +++ b/internal/errwrap/errwrap.go @@ -0,0 +1,28 @@ +package errwrap + +import ( + "errors" + "fmt" +) + +type ErrWrap struct { + err error + werr error +} + +func NewErrWrap(err error, werr error) error { + return &ErrWrap{err, werr} +} + +func (ew *ErrWrap) Error() string { + return fmt.Sprintf("%s: %s", ew.err, ew.werr) +} + +func (ew *ErrWrap) Unwrap() error { + return ew.werr +} + +func (ew *ErrWrap) Is(target error) bool { + return errors.Is(ew.err, target) || + errors.Is(ew.werr, target) +} diff --git a/internal/metadata/generated_status.go b/internal/metadata/generated_status.go new file mode 100644 index 0000000..b3e8f03 --- /dev/null +++ b/internal/metadata/generated_status.go @@ -0,0 +1,12 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/component" +) + +const ( + Type = "jsonlogflattener" + LogsStability = component.StabilityLevelAlpha +) diff --git a/metadata.yaml b/metadata.yaml new file mode 100644 index 0000000..6c286cb --- /dev/null +++ b/metadata.yaml @@ -0,0 +1,9 @@ +type: jsonlogflattener + +status: + class: processor + stability: + alpha: [logs] + distributions: [] + codeowners: + active: ["@JupiterOne/sre"] diff --git a/processor.go b/processor.go new file mode 100644 index 0000000..982dcb4 --- /dev/null +++ b/processor.go @@ -0,0 +1,71 @@ +package oteljsonlogflattenerprocessor + +import ( + "context" + "encoding/json" + "reflect" + + "github.com/jupiterone/oteljsonlogflattenerprocessor/internal/errwrap" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/processor" + "go.uber.org/zap" +) + +var _ processor.Logs = (*jsonlogflattenerProcessor)(nil) + +type jsonlogflattenerProcessor struct { + config *Config + next consumer.Logs + logger *zap.Logger +} + +func newProcessor(config *Config, next consumer.Logs, logger *zap.Logger) (*jsonlogflattenerProcessor, error) { + return &jsonlogflattenerProcessor{ + config: config, + logger: logger, + next: next, + }, nil +} + +func (jlp *jsonlogflattenerProcessor) Capabilities() consumer.Capabilities { + return consumer.Capabilities{MutatesData: true} +} + +func (jlp *jsonlogflattenerProcessor) ConsumeLogs(ctx context.Context, ld plog.Logs) error { + if err := jlp.flatten(&ld); err != nil { + jlp.logger.Error(err.Error()) + } + return jlp.next.ConsumeLogs(ctx, ld) +} + +func (jlp *jsonlogflattenerProcessor) Start(_ context.Context, host component.Host) error { + return nil +} + +func (jlp *jsonlogflattenerProcessor) Shutdown(context.Context) error { + return nil +} + +func (jlp *jsonlogflattenerProcessor) flatten(ld *plog.Logs) error { + var rootErr error + for i := 0; i < ld.ResourceLogs().Len(); i++ { + for j := 0; j < ld.ResourceLogs().At(i).ScopeLogs().Len(); j++ { + for k := 0; k < ld.ResourceLogs().At(i).ScopeLogs().At(j).LogRecords().Len(); k++ { + for l, m := range ld.ResourceLogs().At(i).ScopeLogs().At(j).LogRecords().At(k).Attributes().AsRaw() { + if reflect.TypeOf(m).Kind() == reflect.Map || reflect.TypeOf(m).Kind() == reflect.Slice { + payload, err := json.Marshal(m) + if err != nil { + jlp.logger.Error(err.Error()) + rootErr = errwrap.NewErrWrap(rootErr, err) + continue + } + ld.ResourceLogs().At(i).ScopeLogs().At(j).LogRecords().At(k).Attributes().PutStr(l, string(payload)) + } + } + } + } + } + return rootErr +} diff --git a/processor_test.go b/processor_test.go new file mode 100644 index 0000000..d530948 --- /dev/null +++ b/processor_test.go @@ -0,0 +1,70 @@ +package oteljsonlogflattenerprocessor + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/processor/processortest" +) + +func TestFlatten(t *testing.T) { + factory := NewFactory() + tln := new(consumertest.LogsSink) + jlfp, err := factory.CreateLogsProcessor(context.Background(), processortest.NewNopCreateSettings(), &Config{}, tln) + require.NoError(t, err) + assert.True(t, jlfp.Capabilities().MutatesData) + + err = jlfp.Start(context.Background(), nil) + require.NoError(t, err) + + ld := plog.NewLogs() + ld.ResourceLogs().AppendEmpty() + scope := ld.ResourceLogs().At(0).ScopeLogs().AppendEmpty() + log := scope.LogRecords().AppendEmpty() + + log.Attributes().PutStr("foo-string", "bar") + log.Attributes().PutDouble("foo-double", 1.0) + log.Attributes().PutInt("foo-int", 1) + log.Attributes().PutBool("foo-bool", true) + inputFooMap := log.Attributes().PutEmptyMap("foo-map") + inputFooMap.PutStr("foo-map-string", "bar") + inputFooMap.PutDouble("foo-map-double", 1.0) + inputFooMap.PutInt("foo-map-int", 1) + inputFooMap.PutBool("foo-map-bool", true) + inputFooSlice := log.Attributes().PutEmptySlice("foo-slice") + inputFooSlice.AppendEmpty().SetInt(1) + inputFooSlice.AppendEmpty().SetInt(2) + + err = jlfp.ConsumeLogs(context.Background(), ld) + require.NoError(t, err) + + logs := tln.AllLogs() + require.Len(t, logs, 1) + attrs := logs[0].ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes() + require.Equal(t, attrs.Len(), 6) + fooString, exists := attrs.Get("foo-string") + require.True(t, exists) + require.Equal(t, fooString.Str(), "bar") + fooDouble, exists := attrs.Get("foo-double") + require.True(t, exists) + require.Equal(t, fooDouble.Double(), 1.0) + fooInt, exists := attrs.Get("foo-int") + require.True(t, exists) + require.Equal(t, fooInt.Int(), int64(1)) + fooBool, exists := attrs.Get("foo-bool") + require.True(t, exists) + require.True(t, fooBool.Bool()) + fooMap, exists := attrs.Get("foo-map") + require.True(t, exists) + require.Equal(t, fooMap.Type(), pcommon.ValueTypeStr) + require.Equal(t, fooMap.Str(), `{"foo-map-bool":true,"foo-map-double":1,"foo-map-int":1,"foo-map-string":"bar"}`) + fooSlice, exists := attrs.Get("foo-slice") + require.True(t, exists) + require.Equal(t, fooSlice.Type(), pcommon.ValueTypeStr) + require.Equal(t, fooSlice.Str(), `[1,2]`) +}