From a9b7c298e835bde41b4a447eeeaadc0449f4096d Mon Sep 17 00:00:00 2001 From: Vibhu Pandey Date: Thu, 2 Jan 2025 16:19:49 +0530 Subject: [PATCH] feat(metering): add meter for traces (#496) #### Features - The metering for traces is dependent on SigNoz's custom schema and not just on `ptrace.Traces`. The schema specific stuff has been moved to a separate package. It is not recommended to import the clickhousetracesexporter package in the metering package as it will bring a lot of unwanted dependencies into the codebase of whoever is importing it. - Revert the modules created for metering and pdatagen. We thought we would import unwanted dependencies in the gateway which is not the case. We are only importing what we need there. #### Fixes - Remove redundant loggers in the LogsSize function. --- .../clickhouse_exporter.go | 2 + .../clickhousetracesexporter/schema-signoz.go | 7 +- pkg/metering/go.mod | 29 ---- pkg/metering/go.sum | 83 --------- pkg/metering/json.go | 158 +++++++++++++++++- pkg/metering/meter.go | 12 ++ pkg/metering/v1/logs.go | 4 - pkg/metering/v1/traces.go | 118 +++++++++++++ pkg/metering/v1/traces_test.go | 105 ++++++++++++ pkg/pdatagen/go.mod | 19 --- pkg/pdatagen/go.sum | 72 -------- pkg/pdatagen/ptracesgen/options.go | 50 ++++++ pkg/pdatagen/ptracesgen/traces.go | 70 ++++++++ pkg/schema/common/resource.go | 15 ++ pkg/schema/traces/event.go | 76 +++++++++ pkg/schema/traces/otel_span_ref.go | 61 +++++++ 16 files changed, 671 insertions(+), 210 deletions(-) delete mode 100644 pkg/metering/go.mod delete mode 100644 pkg/metering/go.sum create mode 100644 pkg/metering/v1/traces.go create mode 100644 pkg/metering/v1/traces_test.go delete mode 100644 pkg/pdatagen/go.mod delete mode 100644 pkg/pdatagen/go.sum create mode 100644 pkg/pdatagen/ptracesgen/options.go create mode 100644 pkg/pdatagen/ptracesgen/traces.go create mode 100644 pkg/schema/common/resource.go create mode 100644 pkg/schema/traces/event.go create mode 100644 pkg/schema/traces/otel_span_ref.go diff --git a/exporter/clickhousetracesexporter/clickhouse_exporter.go b/exporter/clickhousetracesexporter/clickhouse_exporter.go index 8fb35340..34b0dd8c 100644 --- a/exporter/clickhousetracesexporter/clickhouse_exporter.go +++ b/exporter/clickhousetracesexporter/clickhouse_exporter.go @@ -88,6 +88,7 @@ func newExporter(cfg component.Config, logger *zap.Logger, settings exporter.Set wg: new(sync.WaitGroup), closeChan: make(chan struct{}), useNewSchema: configClickHouse.UseNewSchema, + logger: logger, } return &storage, nil @@ -101,6 +102,7 @@ type storage struct { wg *sync.WaitGroup closeChan chan struct{} useNewSchema bool + logger *zap.Logger } type storageConfig struct { diff --git a/exporter/clickhousetracesexporter/schema-signoz.go b/exporter/clickhousetracesexporter/schema-signoz.go index b799d3d9..86539e3e 100644 --- a/exporter/clickhousetracesexporter/schema-signoz.go +++ b/exporter/clickhousetracesexporter/schema-signoz.go @@ -20,6 +20,7 @@ import ( "go.uber.org/zap/zapcore" ) +// TODO: Read from github.com/SigNoz/signoz-otel-collector/pkg/schema/traces type Event struct { Name string `json:"name,omitempty"` TimeUnixNano uint64 `json:"timeUnixNano,omitempty"` @@ -131,6 +132,7 @@ type Span struct { SpanAttributes []SpanAttribute `json:"spanAttributes,omitempty"` } +// TODO: Read from github.com/SigNoz/signoz-otel-collector/pkg/schema/traces type ErrorEvent struct { Event Event `json:"errorEvent,omitempty"` ErrorID string `json:"errorID,omitempty"` @@ -167,7 +169,9 @@ type SpanV3 struct { ResourcesString map[string]string `json:"resources_string,omitempty"` // for events - Events []string `json:"event,omitempty"` + // TODO: Read from github.com/SigNoz/signoz-otel-collector/pkg/schema/traces + Events []string `json:"event,omitempty"` + // TODO: Read from github.com/SigNoz/signoz-otel-collector/pkg/schema/traces ErrorEvents []ErrorEvent `json:"-"` ServiceName string `json:"serviceName,omitempty"` // for error table @@ -239,6 +243,7 @@ func (s *Span) MarshalLogObject(enc zapcore.ObjectEncoder) error { return nil } +// TODO: Read from github.com/SigNoz/signoz-otel-collector/pkg/schema/traces type OtelSpanRef struct { TraceId string `json:"traceId,omitempty"` SpanId string `json:"spanId,omitempty"` diff --git a/pkg/metering/go.mod b/pkg/metering/go.mod deleted file mode 100644 index 55ed6bdd..00000000 --- a/pkg/metering/go.mod +++ /dev/null @@ -1,29 +0,0 @@ -module github.com/SigNoz/signoz-otel-collector/pkg/metering - -go 1.23.4 - -require ( - github.com/SigNoz/signoz-otel-collector/pkg/pdatagen v0.0.0-20241230104648-472e3efe3041 - github.com/stretchr/testify v1.10.0 - go.opentelemetry.io/collector/pdata v1.22.0 - go.uber.org/zap v1.27.0 -) - -require ( - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.29.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.18.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect - google.golang.org/grpc v1.68.1 // indirect - google.golang.org/protobuf v1.35.2 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) - -replace github.com/SigNoz/signoz-otel-collector/pkg/pdatagen => ../pdatagen diff --git a/pkg/metering/go.sum b/pkg/metering/go.sum deleted file mode 100644 index 8ab6def5..00000000 --- a/pkg/metering/go.sum +++ /dev/null @@ -1,83 +0,0 @@ -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opentelemetry.io/collector/pdata v1.22.0 h1:3yhjL46NLdTMoP8rkkcE9B0pzjf2973crn0KKhX5UrI= -go.opentelemetry.io/collector/pdata v1.22.0/go.mod h1:nLLf6uDg8Kn5g3WNZwGyu8+kf77SwOqQvMTb5AXEbEY= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= -google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= -google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= -google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/metering/json.go b/pkg/metering/json.go index f9156e64..07523c05 100644 --- a/pkg/metering/json.go +++ b/pkg/metering/json.go @@ -2,7 +2,14 @@ package metering import ( "encoding/json" + "fmt" + "strconv" + "strings" + "github.com/SigNoz/signoz-otel-collector/pkg/schema/traces" + "github.com/SigNoz/signoz-otel-collector/utils" + "github.com/SigNoz/signoz-otel-collector/utils/flatten" + "go.opentelemetry.io/collector/pdata/pcommon" "go.uber.org/zap" ) @@ -10,7 +17,7 @@ type jsonSizer struct { Logger *zap.Logger } -func NewJSONSizer(logger *zap.Logger) *jsonSizer { +func NewJSONSizer(logger *zap.Logger) Sizer { return &jsonSizer{ Logger: logger, } @@ -19,9 +26,156 @@ func NewJSONSizer(logger *zap.Logger) *jsonSizer { func (sizer *jsonSizer) SizeOfMapStringAny(input map[string]any) int { bytes, err := json.Marshal(input) if err != nil { - sizer.Logger.Error("cannot marshal object, setting size to 0", zap.Any("obj", input)) + sizer.Logger.Error("cannot marshal object, setting size to 0", zap.Error(err), zap.Any("obj", input)) return 0 } return len(bytes) } + +func (sizer *jsonSizer) SizeOfFlatPcommonMapInMapStringString(input pcommon.Map) int { + output := map[string]string{} + + input.Range(func(k string, v pcommon.Value) bool { + switch v.Type() { + case pcommon.ValueTypeMap: + flattened := flatten.FlattenJSON(v.Map().AsRaw(), k) + for kf, vf := range flattened { + output[kf] = fmt.Sprintf("%v", vf) + } + default: + output[k] = v.AsString() + } + return true + }) + + bytes, err := json.Marshal(output) + if err != nil { + sizer.Logger.Error("cannot marshal object, setting size to 0", zap.Error(err), zap.Any("obj", input)) + return 0 + } + + return len(bytes) +} + +func (sizer *jsonSizer) SizeOfFlatPcommonMapInNumberStringBool(input pcommon.Map) (int, int, int) { + n := map[string]float64{} + s := map[string]string{} + b := map[string]bool{} + + input.Range(func(k string, v pcommon.Value) bool { + switch v.Type() { + case pcommon.ValueTypeDouble: + if utils.IsValidFloat(v.Double()) { + n[k] = v.Double() + } + case pcommon.ValueTypeInt: + n[k] = float64(v.Int()) + case pcommon.ValueTypeBool: + b[k] = v.Bool() + case pcommon.ValueTypeMap: + flattened := flatten.FlattenJSON(v.Map().AsRaw(), k) + for kf, vf := range flattened { + switch tvf := vf.(type) { + case string: + s[kf] = tvf + case float64: + n[kf] = tvf + case bool: + b[kf] = tvf + } + } + default: + s[k] = v.AsString() + } + + return true + }) + + nbytes, err := json.Marshal(n) + if err != nil { + sizer.Logger.Error("cannot marshal object, setting size to 0", zap.Error(err), zap.Any("obj", input)) + nbytes = []byte(nil) + } + + sbytes, err := json.Marshal(s) + if err != nil { + sizer.Logger.Error("cannot marshal object, setting size to 0", zap.Error(err), zap.Any("obj", input)) + sbytes = []byte(nil) + } + + bbytes, err := json.Marshal(b) + if err != nil { + sizer.Logger.Error("cannot marshal object, setting size to 0", zap.Error(err), zap.Any("obj", input)) + bbytes = []byte(nil) + } + + return len(nbytes), len(sbytes), len(bbytes) +} + +func (sizer *jsonSizer) SizeOfInt(input int) int { + return len(strconv.Itoa(input)) +} + +func (sizer *jsonSizer) SizeOfFloat64(input float64) int { + return len(strconv.FormatFloat(input, 'f', -1, 64)) +} + +func (sizer *jsonSizer) SizeOfTraceID(input pcommon.TraceID) int { + if input.IsEmpty() { + return 0 + } + + // Since we encode to hex, the original 16 bytes are stored in 32 bytes + return 32 +} + +func (sizer *jsonSizer) SizeOfSpanID(input pcommon.SpanID) int { + if input.IsEmpty() { + return 0 + } + + // Since we encode to hex, the original 8 bytes are stored in 16 bytes + return 16 +} + +func (sizer *jsonSizer) SizeOfEvents(input []string) int { + bytes, err := json.Marshal(input) + if err != nil { + sizer.Logger.Error("cannot marshal object, setting size to 0", zap.Error(err), zap.Any("obj", input)) + return 0 + } + + return len(bytes) +} + +func (sizer *jsonSizer) SizeOfOtelSpanRefs(input []traces.OtelSpanRef) int { + if input == nil { + return 0 + } + + bytes, err := json.Marshal(input) + if err != nil { + sizer.Logger.Error("cannot marshal object, setting size to 0", zap.Error(err), zap.Any("obj", input)) + return 0 + } + + escapeCharacters := strings.Count(string(bytes), "\"") + return len(bytes) + escapeCharacters +} + +func (size *jsonSizer) TotalSizeIfKeyExists(key int, value int, extra int) int { + if value == 0 { + return 0 + } + + return key + value + extra +} + +func (size *jsonSizer) TotalSizeIfKeyExistsAndValueIsMapOrSlice(key int, value int, extra int) int { + if value <= 2 { + return 0 + } + + return key + value + extra +} diff --git a/pkg/metering/meter.go b/pkg/metering/meter.go index defcd483..231e72e5 100644 --- a/pkg/metering/meter.go +++ b/pkg/metering/meter.go @@ -1,6 +1,8 @@ package metering import ( + "github.com/SigNoz/signoz-otel-collector/pkg/schema/traces" + "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/collector/pdata/ptrace" @@ -18,6 +20,16 @@ type Meter[T ptrace.Traces | pmetric.Metrics | plog.Logs] interface { // Sizer is an interface that calculates the size of different of map[string]any type Sizer interface { SizeOfMapStringAny(map[string]any) int + SizeOfFlatPcommonMapInMapStringString(pcommon.Map) int + SizeOfInt(int) int + SizeOfFloat64(float64) int + SizeOfTraceID(pcommon.TraceID) int + SizeOfSpanID(pcommon.SpanID) int + SizeOfFlatPcommonMapInNumberStringBool(pcommon.Map) (int, int, int) + SizeOfEvents([]string) int + SizeOfOtelSpanRefs([]traces.OtelSpanRef) int + TotalSizeIfKeyExists(int, int, int) int + TotalSizeIfKeyExistsAndValueIsMapOrSlice(int, int, int) int } // Logs calculates billable metrics for logs. diff --git a/pkg/metering/v1/logs.go b/pkg/metering/v1/logs.go index a19e20c5..4ba4c207 100644 --- a/pkg/metering/v1/logs.go +++ b/pkg/metering/v1/logs.go @@ -20,9 +20,6 @@ func NewLogs(logger *zap.Logger) metering.Logs { } func (meter *logs) Size(ld plog.Logs) int { - - meter.Logger.Debug("Calculating logs size") - total := 0 for i := 0; i < ld.ResourceLogs().Len(); i++ { resourceLog := ld.ResourceLogs().At(i) @@ -40,7 +37,6 @@ func (meter *logs) Size(ld plog.Logs) int { } } - meter.Logger.Debug("Logs size", zap.Int("size", total)) return total } diff --git a/pkg/metering/v1/traces.go b/pkg/metering/v1/traces.go new file mode 100644 index 00000000..f44334fe --- /dev/null +++ b/pkg/metering/v1/traces.go @@ -0,0 +1,118 @@ +package v1 + +import ( + "encoding/json" + + "github.com/SigNoz/signoz-otel-collector/pkg/metering" + "github.com/SigNoz/signoz-otel-collector/pkg/schema/common" + schema "github.com/SigNoz/signoz-otel-collector/pkg/schema/traces" + "go.opentelemetry.io/collector/pdata/ptrace" + "go.uber.org/zap" +) + +type traces struct { + Logger *zap.Logger + Sizer metering.Sizer + KeySizes map[string]int +} + +func NewTraces(logger *zap.Logger) metering.Traces { + return &traces{ + Logger: logger, + Sizer: metering.NewJSONSizer(logger), + KeySizes: map[string]int{ + "resources_string": len("\"resources_string\""), + "startTimeUnixNano": len("\"startTimeUnixNano\""), + "traceId": len("\"traceId\""), + "spanId": len("\"spanId\""), + "traceState": len("\"traceState\""), + "parentSpanId": len("\"parentSpanId\""), + "flags": len("\"flags\""), + "name": len("\"name\""), + "kind": len("\"kind\""), + "spanKind": len("\"spanKind\""), + "attributes_string": len("\"attributes_string\""), + "attributes_bool": len("\"attributes_bool\""), + "attributes_number": len("\"attributes_number\""), + "serviceName": len("\"serviceName\""), + "event": len("\"event\""), + "references": len("\"references\""), + }, + } +} + +func (meter *traces) Size(td ptrace.Traces) int { + total := 0 + + for i := 0; i < td.ResourceSpans().Len(); i++ { + resourceSpan := td.ResourceSpans().At(i) + resourceAttributesSize := meter.Sizer.SizeOfFlatPcommonMapInMapStringString(resourceSpan.Resource().Attributes()) + + for j := 0; j < resourceSpan.ScopeSpans().Len(); j++ { + scopeSpans := resourceSpan.ScopeSpans().At(j) + + for k := 0; k < scopeSpans.Spans().Len(); k++ { + span := scopeSpans.Spans().At(k) + + // Size of references + sizeOfOtelSpanRefs := 0 + otelSpanRefs, err := schema.NewOtelSpanRefs(span.Links(), span.ParentSpanID(), span.TraceID()) + if err != nil { + meter.Logger.Error("cannot create span refs", zap.Error(err)) + } else { + sizeOfOtelSpanRefs = meter.Sizer.SizeOfOtelSpanRefs(otelSpanRefs) + } + + // Size of service name + sizeOfServiceName := 0 + serviceName := common.ServiceName(resourceSpan.Resource()) + serviceNameBytes, err := json.Marshal(serviceName) + if err != nil { + meter.Logger.Error("cannot marshal service name", zap.Error(err), zap.String("val", serviceName)) + } else { + sizeOfServiceName = len(serviceNameBytes) + } + + // Size of events + events, _ := schema.NewEventsAndErrorEvents(span.Events(), serviceName, false) + sizeOfEvents := meter.Sizer.SizeOfEvents(events) + + // Size of attributes + sizeOfNumberAttributes, sizeOfStringAttributes, sizeOfBoolAttributes := meter.Sizer.SizeOfFlatPcommonMapInNumberStringBool(span.Attributes()) + + // Size of flags + sizeOfFlags := 0 + if span.Flags() != 0 { + sizeOfFlags = meter.Sizer.SizeOfInt(int(span.Flags())) + } + + // Let's start making the json object + // 2({}) + total += 2 + + meter.Sizer.TotalSizeIfKeyExistsAndValueIsMapOrSlice(meter.KeySizes["resources_string"], resourceAttributesSize, 2) + //:, + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["startTimeUnixNano"], meter.Sizer.SizeOfInt(int(span.StartTimestamp())), 2) + //:, + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["spanId"], meter.Sizer.SizeOfSpanID(span.SpanID()), 4) + //:"", + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["traceId"], meter.Sizer.SizeOfTraceID(span.TraceID()), 4) + //:"", + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["traceState"], len(span.TraceState().AsRaw()), 4) + //:"", + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["parentSpanId"], meter.Sizer.SizeOfSpanID(span.ParentSpanID()), 4) + //:"", + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["flags"], sizeOfFlags, 2) + //:, + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["name"], len(span.Name()), 4) + //:"", + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["kind"], meter.Sizer.SizeOfInt(int(span.Kind())), 2) + //:, + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["spanKind"], len(span.Kind().String()), 4) + //:"", + meter.Sizer.TotalSizeIfKeyExistsAndValueIsMapOrSlice(meter.KeySizes["attributes_string"], sizeOfStringAttributes, 2) + //:, + meter.Sizer.TotalSizeIfKeyExistsAndValueIsMapOrSlice(meter.KeySizes["attributes_bool"], sizeOfBoolAttributes, 2) + //:, + meter.Sizer.TotalSizeIfKeyExistsAndValueIsMapOrSlice(meter.KeySizes["attributes_number"], sizeOfNumberAttributes, 2) + //:, + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["serviceName"], sizeOfServiceName, 2) + //:, + meter.Sizer.TotalSizeIfKeyExistsAndValueIsMapOrSlice(meter.KeySizes["event"], sizeOfEvents, 2) + //:, + meter.Sizer.TotalSizeIfKeyExistsAndValueIsMapOrSlice(meter.KeySizes["references"], sizeOfOtelSpanRefs, 4) - //:"", + 1 //, + } + + } + } + + return total +} +func (*traces) Count(td ptrace.Traces) int { + return td.SpanCount() +} diff --git a/pkg/metering/v1/traces_test.go b/pkg/metering/v1/traces_test.go new file mode 100644 index 00000000..475af33c --- /dev/null +++ b/pkg/metering/v1/traces_test.go @@ -0,0 +1,105 @@ +package v1 + +import ( + "fmt" + "testing" + + "github.com/SigNoz/signoz-otel-collector/pkg/pdatagen/ptracesgen" + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/pdata/ptrace" + "go.uber.org/zap" +) + +func TestTracesSizeWithNoEvents(t *testing.T) { + traces := ptracesgen.Generate( + ptracesgen.WithSpanCount(1), + ptracesgen.WithResourceAttributeCount(1), + // 4: SPAN_KIND_PRODUCER + ptracesgen.WithSpanKind(ptrace.SpanKindProducer), + ptracesgen.WithResourceAttributeStringValue("test"), + ) + + meter := NewTraces(zap.NewNop()) + size := meter.Size(traces) + + assert.Equal(t, 406, size) +} + +func TestTracesSizeWithEvents(t *testing.T) { + traces := ptracesgen.Generate( + ptracesgen.WithSpanCount(1), + ptracesgen.WithResourceAttributeCount(1), + ptracesgen.WithEventCount(2), + // 4: SPAN_KIND_PRODUCER + ptracesgen.WithSpanKind(ptrace.SpanKindProducer), + ptracesgen.WithResourceAttributeStringValue("test"), + ) + + meter := NewTraces(zap.NewNop()) + size := meter.Size(traces) + + assert.Equal(t, 540, size) +} + +func TestTracesSizeWith2SpansAnd2EventsAnd2ResourceAttributes(t *testing.T) { + traces := ptracesgen.Generate( + ptracesgen.WithSpanCount(2), + ptracesgen.WithResourceAttributeCount(2), + ptracesgen.WithEventCount(2), + // 4: SPAN_KIND_PRODUCER + ptracesgen.WithSpanKind(ptrace.SpanKindProducer), + ptracesgen.WithResourceAttributeStringValue("test"), + ) + + meter := NewTraces(zap.NewNop()) + + b, _ := (&ptrace.JSONMarshaler{}).MarshalTraces(traces) + fmt.Println(string(b)) + size := meter.Size(traces) + + assert.Equal(t, 1120, size) +} + +func TestTracesSizeWith2SpansAnd2EventsAnd2ResourceAttributesAndAttributes(t *testing.T) { + traces := ptracesgen.Generate( + ptracesgen.WithSpanCount(2), + ptracesgen.WithResourceAttributeCount(2), + ptracesgen.WithEventCount(2), + ptracesgen.WithSpanKind(ptrace.SpanKindClient), + ptracesgen.WithResourceAttributeStringValue("test"), + ptracesgen.WithAttributes(map[string]any{ + "float64": 342.5, + "int64": int64(342), + "string": "attribute", + "bool": false, + }), + ) + + meter := NewTraces(zap.NewNop()) + + b, _ := (&ptrace.JSONMarshaler{}).MarshalTraces(traces) + fmt.Println(string(b)) + size := meter.Size(traces) + + assert.Equal(t, 1368, size) +} + +func TestTracesSizeWithBoolAttributes(t *testing.T) { + traces := ptracesgen.Generate( + ptracesgen.WithSpanCount(1), + ptracesgen.WithResourceAttributeCount(1), + ptracesgen.WithSpanKind(ptrace.SpanKindClient), + ptracesgen.WithResourceAttributeStringValue("test"), + ptracesgen.WithAttributes(map[string]any{ + "bool1": false, + "bool2": true, + }), + ) + + meter := NewTraces(zap.NewNop()) + b, _ := (&ptrace.JSONMarshaler{}).MarshalTraces(traces) + fmt.Println(string(b)) + size := meter.Size(traces) + + assert.Equal(t, 451, size) +} diff --git a/pkg/pdatagen/go.mod b/pkg/pdatagen/go.mod deleted file mode 100644 index ef3dc222..00000000 --- a/pkg/pdatagen/go.mod +++ /dev/null @@ -1,19 +0,0 @@ -module github.com/SigNoz/signoz-otel-collector/pkg/pdatagen - -go 1.23.4 - -require go.opentelemetry.io/collector/pdata v1.22.0 - -require ( - github.com/gogo/protobuf v1.3.2 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.29.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.18.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect - google.golang.org/grpc v1.68.1 // indirect - google.golang.org/protobuf v1.35.2 // indirect -) diff --git a/pkg/pdatagen/go.sum b/pkg/pdatagen/go.sum deleted file mode 100644 index 8777ff54..00000000 --- a/pkg/pdatagen/go.sum +++ /dev/null @@ -1,72 +0,0 @@ -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opentelemetry.io/collector/pdata v1.22.0 h1:3yhjL46NLdTMoP8rkkcE9B0pzjf2973crn0KKhX5UrI= -go.opentelemetry.io/collector/pdata v1.22.0/go.mod h1:nLLf6uDg8Kn5g3WNZwGyu8+kf77SwOqQvMTb5AXEbEY= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= -google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= -google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= -google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/pdatagen/ptracesgen/options.go b/pkg/pdatagen/ptracesgen/options.go new file mode 100644 index 00000000..eb8a8677 --- /dev/null +++ b/pkg/pdatagen/ptracesgen/options.go @@ -0,0 +1,50 @@ +package ptracesgen + +import "go.opentelemetry.io/collector/pdata/ptrace" + +type generationOptions struct { + spanCount int + eventCount int + resourceAttributeCount int + resourceAttributeStringValue string + spanKind ptrace.SpanKind + attributes map[string]any +} + +type GenerationOption func(*generationOptions) + +func WithSpanCount(i int) GenerationOption { + return func(o *generationOptions) { + o.spanCount = i + } +} + +func WithEventCount(i int) GenerationOption { + return func(o *generationOptions) { + o.eventCount = i + } +} + +func WithSpanKind(k ptrace.SpanKind) GenerationOption { + return func(o *generationOptions) { + o.spanKind = k + } +} + +func WithResourceAttributeCount(i int) GenerationOption { + return func(o *generationOptions) { + o.resourceAttributeCount = i + } +} + +func WithResourceAttributeStringValue(s string) GenerationOption { + return func(o *generationOptions) { + o.resourceAttributeStringValue = s + } +} + +func WithAttributes(m map[string]any) GenerationOption { + return func(o *generationOptions) { + o.attributes = m + } +} diff --git a/pkg/pdatagen/ptracesgen/traces.go b/pkg/pdatagen/ptracesgen/traces.go new file mode 100644 index 00000000..b1f65c2b --- /dev/null +++ b/pkg/pdatagen/ptracesgen/traces.go @@ -0,0 +1,70 @@ +package ptracesgen + +import ( + "strconv" + "time" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/ptrace" +) + +func Generate(opts ...GenerationOption) ptrace.Traces { + generationOpts := generationOptions{ + eventCount: 0, + spanCount: 1, + resourceAttributeCount: 1, + resourceAttributeStringValue: "resource", + spanKind: ptrace.SpanKindClient, + attributes: map[string]any{}, + } + + for _, opt := range opts { + opt(&generationOpts) + } + + endTime := pcommon.NewTimestampFromTime(time.Now()) + traces := ptrace.NewTraces() + resourceSpan := traces.ResourceSpans().AppendEmpty() + for i := 0; i < generationOpts.resourceAttributeCount; i++ { + suffix := strconv.Itoa(i) + // Do not change the key name format in resource attributes below. + resourceSpan.Resource().Attributes().PutStr("resource."+suffix, generationOpts.resourceAttributeStringValue) + } + + scopeSpans := resourceSpan.ScopeSpans().AppendEmpty() + scopeSpans.Spans().EnsureCapacity(generationOpts.spanCount) + for i := 0; i < generationOpts.spanCount; i++ { + suffix := strconv.Itoa(i) + span := scopeSpans.Spans().AppendEmpty() + span.SetName("span." + suffix) + span.SetKind(generationOpts.spanKind) + span.SetStartTimestamp(endTime) + span.SetEndTimestamp(endTime) + span.SetTraceID(pcommon.TraceID([]byte("5B8EFFF798038103D269B633813FC60C"))) + span.SetSpanID(pcommon.SpanID([]byte("EEE19B7EC3C1B174"))) + span.SetParentSpanID(pcommon.SpanID([]byte("EEE19B7EC3C1B174"))) + + for j := 0; j < generationOpts.eventCount; j++ { + suffix := strconv.Itoa(j) + spanEvent := span.Events().AppendEmpty() + spanEvent.SetName("event." + suffix) + spanEvent.SetTimestamp(endTime) + } + + for k, v := range generationOpts.attributes { + switch v := v.(type) { + case string: + span.Attributes().PutStr(k, v) + case float64: + span.Attributes().PutDouble(k, v) + case bool: + span.Attributes().PutBool(k, v) + case int64: + span.Attributes().PutInt(k, v) + } + + } + } + + return traces +} diff --git a/pkg/schema/common/resource.go b/pkg/schema/common/resource.go new file mode 100644 index 00000000..c7c46385 --- /dev/null +++ b/pkg/schema/common/resource.go @@ -0,0 +1,15 @@ +package common + +import ( + "go.opentelemetry.io/collector/pdata/pcommon" + semconv "go.opentelemetry.io/collector/semconv/v1.5.0" +) + +func ServiceName(input pcommon.Resource) string { + service, found := input.Attributes().Get(semconv.AttributeServiceName) + if !found { + return "" + } + + return service.Str() +} diff --git a/pkg/schema/traces/event.go b/pkg/schema/traces/event.go new file mode 100644 index 00000000..0462c669 --- /dev/null +++ b/pkg/schema/traces/event.go @@ -0,0 +1,76 @@ +package traces + +import ( + "crypto/md5" + "encoding/json" + "fmt" + "strings" + + "github.com/google/uuid" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/ptrace" + "go.uber.org/zap/zapcore" +) + +// This has been copied from the exporter. The exporter needs to read from here. +type Event struct { + Name string `json:"name,omitempty"` + TimeUnixNano uint64 `json:"timeUnixNano,omitempty"` + AttributeMap map[string]string `json:"attributeMap,omitempty"` + IsError bool `json:"isError,omitempty"` +} + +func (e *Event) MarshalLogObject(enc zapcore.ObjectEncoder) error { + enc.AddString("name", e.Name) + enc.AddUint64("timeUnixNano", e.TimeUnixNano) + enc.AddBool("isError", e.IsError) + enc.AddString("attributeMap", fmt.Sprintf("%v", e.AttributeMap)) + return nil +} + +// This has been copied from the exporter. The exporter needs to read from here. +type ErrorEvent struct { + Event Event `json:"errorEvent,omitempty"` + ErrorID string `json:"errorID,omitempty"` + ErrorGroupID string `json:"errorGroupID,omitempty"` +} + +func NewEventsAndErrorEvents(input ptrace.SpanEventSlice, serviceName string, lowCardinalExceptionGrouping bool) ([]string, []ErrorEvent) { + events := make([]string, 0) + errorEvents := make([]ErrorEvent, 0) + + for i := 0; i < input.Len(); i++ { + event := Event{ + Name: input.At(i).Name(), + TimeUnixNano: uint64(input.At(i).Timestamp()), + AttributeMap: map[string]string{}, + IsError: false, + } + + input.At(i).Attributes().Range(func(k string, v pcommon.Value) bool { + event.AttributeMap[k] = v.AsString() + return true + }) + errorEvent := ErrorEvent{} + if event.Name == "exception" { + event.IsError = true + errorEvent.Event = event + uuidWithHyphen := uuid.New() + uuid := strings.Replace(uuidWithHyphen.String(), "-", "", -1) + errorEvent.ErrorID = uuid + var hash [16]byte + if lowCardinalExceptionGrouping { + hash = md5.Sum([]byte(serviceName + errorEvent.Event.AttributeMap["exception.type"])) + } else { + hash = md5.Sum([]byte(serviceName + errorEvent.Event.AttributeMap["exception.type"] + errorEvent.Event.AttributeMap["exception.message"])) + } + errorEvent.ErrorGroupID = fmt.Sprintf("%x", hash) + } + + stringEvent, _ := json.Marshal(event) + events = append(events, string(stringEvent)) + errorEvents = append(errorEvents, errorEvent) + } + + return events, errorEvents +} diff --git a/pkg/schema/traces/otel_span_ref.go b/pkg/schema/traces/otel_span_ref.go new file mode 100644 index 00000000..3069e280 --- /dev/null +++ b/pkg/schema/traces/otel_span_ref.go @@ -0,0 +1,61 @@ +package traces + +import ( + "github.com/SigNoz/signoz-otel-collector/utils" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/ptrace" + "go.uber.org/zap/zapcore" +) + +type OtelSpanRef struct { + TraceId string `json:"traceId,omitempty"` + SpanId string `json:"spanId,omitempty"` + RefType string `json:"refType,omitempty"` +} + +func NewOtelSpanRefs(links ptrace.SpanLinkSlice, parentSpanID pcommon.SpanID, traceID pcommon.TraceID) ([]OtelSpanRef, error) { + parentSpanIDSet := len([8]byte(parentSpanID)) != 0 + if !parentSpanIDSet && links.Len() == 0 { + return nil, nil + } + + refsCount := links.Len() + if parentSpanIDSet { + refsCount++ + } + + refs := make([]OtelSpanRef, 0, refsCount) + + // Put parent span ID at the first place because usually backends look for it + // as the first CHILD_OF item in the model.SpanRef slice. + if parentSpanIDSet { + refs = append(refs, OtelSpanRef{ + TraceId: utils.TraceIDToHexOrEmptyString(traceID), + SpanId: utils.SpanIDToHexOrEmptyString(parentSpanID), + RefType: "CHILD_OF", + }) + } + + for i := 0; i < links.Len(); i++ { + link := links.At(i) + + refs = append(refs, OtelSpanRef{ + TraceId: utils.TraceIDToHexOrEmptyString(link.TraceID()), + SpanId: utils.SpanIDToHexOrEmptyString(link.SpanID()), + + // Since Jaeger RefType is not captured in internal data, + // use SpanRefType_FOLLOWS_FROM by default. + // SpanRefType_CHILD_OF supposed to be set only from parentSpanID. + RefType: "FOLLOWS_FROM", + }) + } + + return refs, nil +} + +func (r *OtelSpanRef) MarshalLogObject(enc zapcore.ObjectEncoder) error { + enc.AddString("traceId", r.TraceId) + enc.AddString("spanId", r.SpanId) + enc.AddString("refType", r.RefType) + return nil +}