diff --git a/exporter/clickhousetracesexporter/clickhouse_exporter.go b/exporter/clickhousetracesexporter/clickhouse_exporter.go index 8fb35340..34b0dd8c 100644 --- a/exporter/clickhousetracesexporter/clickhouse_exporter.go +++ b/exporter/clickhousetracesexporter/clickhouse_exporter.go @@ -88,6 +88,7 @@ func newExporter(cfg component.Config, logger *zap.Logger, settings exporter.Set wg: new(sync.WaitGroup), closeChan: make(chan struct{}), useNewSchema: configClickHouse.UseNewSchema, + logger: logger, } return &storage, nil @@ -101,6 +102,7 @@ type storage struct { wg *sync.WaitGroup closeChan chan struct{} useNewSchema bool + logger *zap.Logger } type storageConfig struct { diff --git a/exporter/clickhousetracesexporter/schema-signoz.go b/exporter/clickhousetracesexporter/schema-signoz.go index b799d3d9..86539e3e 100644 --- a/exporter/clickhousetracesexporter/schema-signoz.go +++ b/exporter/clickhousetracesexporter/schema-signoz.go @@ -20,6 +20,7 @@ import ( "go.uber.org/zap/zapcore" ) +// TODO: Read from github.com/SigNoz/signoz-otel-collector/pkg/schema/traces type Event struct { Name string `json:"name,omitempty"` TimeUnixNano uint64 `json:"timeUnixNano,omitempty"` @@ -131,6 +132,7 @@ type Span struct { SpanAttributes []SpanAttribute `json:"spanAttributes,omitempty"` } +// TODO: Read from github.com/SigNoz/signoz-otel-collector/pkg/schema/traces type ErrorEvent struct { Event Event `json:"errorEvent,omitempty"` ErrorID string `json:"errorID,omitempty"` @@ -167,7 +169,9 @@ type SpanV3 struct { ResourcesString map[string]string `json:"resources_string,omitempty"` // for events - Events []string `json:"event,omitempty"` + // TODO: Read from github.com/SigNoz/signoz-otel-collector/pkg/schema/traces + Events []string `json:"event,omitempty"` + // TODO: Read from github.com/SigNoz/signoz-otel-collector/pkg/schema/traces ErrorEvents []ErrorEvent `json:"-"` ServiceName string `json:"serviceName,omitempty"` // for error table @@ -239,6 +243,7 @@ func (s *Span) MarshalLogObject(enc zapcore.ObjectEncoder) error { return nil } +// TODO: Read from github.com/SigNoz/signoz-otel-collector/pkg/schema/traces type OtelSpanRef struct { TraceId string `json:"traceId,omitempty"` SpanId string `json:"spanId,omitempty"` diff --git a/pkg/metering/go.mod b/pkg/metering/go.mod deleted file mode 100644 index 55ed6bdd..00000000 --- a/pkg/metering/go.mod +++ /dev/null @@ -1,29 +0,0 @@ -module github.com/SigNoz/signoz-otel-collector/pkg/metering - -go 1.23.4 - -require ( - github.com/SigNoz/signoz-otel-collector/pkg/pdatagen v0.0.0-20241230104648-472e3efe3041 - github.com/stretchr/testify v1.10.0 - go.opentelemetry.io/collector/pdata v1.22.0 - go.uber.org/zap v1.27.0 -) - -require ( - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.29.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.18.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect - google.golang.org/grpc v1.68.1 // indirect - google.golang.org/protobuf v1.35.2 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) - -replace github.com/SigNoz/signoz-otel-collector/pkg/pdatagen => ../pdatagen diff --git a/pkg/metering/go.sum b/pkg/metering/go.sum deleted file mode 100644 index 8ab6def5..00000000 --- a/pkg/metering/go.sum +++ /dev/null @@ -1,83 +0,0 @@ -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opentelemetry.io/collector/pdata v1.22.0 h1:3yhjL46NLdTMoP8rkkcE9B0pzjf2973crn0KKhX5UrI= -go.opentelemetry.io/collector/pdata v1.22.0/go.mod h1:nLLf6uDg8Kn5g3WNZwGyu8+kf77SwOqQvMTb5AXEbEY= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= -google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= -google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= -google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/metering/json.go b/pkg/metering/json.go index f9156e64..07523c05 100644 --- a/pkg/metering/json.go +++ b/pkg/metering/json.go @@ -2,7 +2,14 @@ package metering import ( "encoding/json" + "fmt" + "strconv" + "strings" + "github.com/SigNoz/signoz-otel-collector/pkg/schema/traces" + "github.com/SigNoz/signoz-otel-collector/utils" + "github.com/SigNoz/signoz-otel-collector/utils/flatten" + "go.opentelemetry.io/collector/pdata/pcommon" "go.uber.org/zap" ) @@ -10,7 +17,7 @@ type jsonSizer struct { Logger *zap.Logger } -func NewJSONSizer(logger *zap.Logger) *jsonSizer { +func NewJSONSizer(logger *zap.Logger) Sizer { return &jsonSizer{ Logger: logger, } @@ -19,9 +26,156 @@ func NewJSONSizer(logger *zap.Logger) *jsonSizer { func (sizer *jsonSizer) SizeOfMapStringAny(input map[string]any) int { bytes, err := json.Marshal(input) if err != nil { - sizer.Logger.Error("cannot marshal object, setting size to 0", zap.Any("obj", input)) + sizer.Logger.Error("cannot marshal object, setting size to 0", zap.Error(err), zap.Any("obj", input)) return 0 } return len(bytes) } + +func (sizer *jsonSizer) SizeOfFlatPcommonMapInMapStringString(input pcommon.Map) int { + output := map[string]string{} + + input.Range(func(k string, v pcommon.Value) bool { + switch v.Type() { + case pcommon.ValueTypeMap: + flattened := flatten.FlattenJSON(v.Map().AsRaw(), k) + for kf, vf := range flattened { + output[kf] = fmt.Sprintf("%v", vf) + } + default: + output[k] = v.AsString() + } + return true + }) + + bytes, err := json.Marshal(output) + if err != nil { + sizer.Logger.Error("cannot marshal object, setting size to 0", zap.Error(err), zap.Any("obj", input)) + return 0 + } + + return len(bytes) +} + +func (sizer *jsonSizer) SizeOfFlatPcommonMapInNumberStringBool(input pcommon.Map) (int, int, int) { + n := map[string]float64{} + s := map[string]string{} + b := map[string]bool{} + + input.Range(func(k string, v pcommon.Value) bool { + switch v.Type() { + case pcommon.ValueTypeDouble: + if utils.IsValidFloat(v.Double()) { + n[k] = v.Double() + } + case pcommon.ValueTypeInt: + n[k] = float64(v.Int()) + case pcommon.ValueTypeBool: + b[k] = v.Bool() + case pcommon.ValueTypeMap: + flattened := flatten.FlattenJSON(v.Map().AsRaw(), k) + for kf, vf := range flattened { + switch tvf := vf.(type) { + case string: + s[kf] = tvf + case float64: + n[kf] = tvf + case bool: + b[kf] = tvf + } + } + default: + s[k] = v.AsString() + } + + return true + }) + + nbytes, err := json.Marshal(n) + if err != nil { + sizer.Logger.Error("cannot marshal object, setting size to 0", zap.Error(err), zap.Any("obj", input)) + nbytes = []byte(nil) + } + + sbytes, err := json.Marshal(s) + if err != nil { + sizer.Logger.Error("cannot marshal object, setting size to 0", zap.Error(err), zap.Any("obj", input)) + sbytes = []byte(nil) + } + + bbytes, err := json.Marshal(b) + if err != nil { + sizer.Logger.Error("cannot marshal object, setting size to 0", zap.Error(err), zap.Any("obj", input)) + bbytes = []byte(nil) + } + + return len(nbytes), len(sbytes), len(bbytes) +} + +func (sizer *jsonSizer) SizeOfInt(input int) int { + return len(strconv.Itoa(input)) +} + +func (sizer *jsonSizer) SizeOfFloat64(input float64) int { + return len(strconv.FormatFloat(input, 'f', -1, 64)) +} + +func (sizer *jsonSizer) SizeOfTraceID(input pcommon.TraceID) int { + if input.IsEmpty() { + return 0 + } + + // Since we encode to hex, the original 16 bytes are stored in 32 bytes + return 32 +} + +func (sizer *jsonSizer) SizeOfSpanID(input pcommon.SpanID) int { + if input.IsEmpty() { + return 0 + } + + // Since we encode to hex, the original 8 bytes are stored in 16 bytes + return 16 +} + +func (sizer *jsonSizer) SizeOfEvents(input []string) int { + bytes, err := json.Marshal(input) + if err != nil { + sizer.Logger.Error("cannot marshal object, setting size to 0", zap.Error(err), zap.Any("obj", input)) + return 0 + } + + return len(bytes) +} + +func (sizer *jsonSizer) SizeOfOtelSpanRefs(input []traces.OtelSpanRef) int { + if input == nil { + return 0 + } + + bytes, err := json.Marshal(input) + if err != nil { + sizer.Logger.Error("cannot marshal object, setting size to 0", zap.Error(err), zap.Any("obj", input)) + return 0 + } + + escapeCharacters := strings.Count(string(bytes), "\"") + return len(bytes) + escapeCharacters +} + +func (size *jsonSizer) TotalSizeIfKeyExists(key int, value int, extra int) int { + if value == 0 { + return 0 + } + + return key + value + extra +} + +func (size *jsonSizer) TotalSizeIfKeyExistsAndValueIsMapOrSlice(key int, value int, extra int) int { + if value <= 2 { + return 0 + } + + return key + value + extra +} diff --git a/pkg/metering/meter.go b/pkg/metering/meter.go index defcd483..231e72e5 100644 --- a/pkg/metering/meter.go +++ b/pkg/metering/meter.go @@ -1,6 +1,8 @@ package metering import ( + "github.com/SigNoz/signoz-otel-collector/pkg/schema/traces" + "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/collector/pdata/ptrace" @@ -18,6 +20,16 @@ type Meter[T ptrace.Traces | pmetric.Metrics | plog.Logs] interface { // Sizer is an interface that calculates the size of different of map[string]any type Sizer interface { SizeOfMapStringAny(map[string]any) int + SizeOfFlatPcommonMapInMapStringString(pcommon.Map) int + SizeOfInt(int) int + SizeOfFloat64(float64) int + SizeOfTraceID(pcommon.TraceID) int + SizeOfSpanID(pcommon.SpanID) int + SizeOfFlatPcommonMapInNumberStringBool(pcommon.Map) (int, int, int) + SizeOfEvents([]string) int + SizeOfOtelSpanRefs([]traces.OtelSpanRef) int + TotalSizeIfKeyExists(int, int, int) int + TotalSizeIfKeyExistsAndValueIsMapOrSlice(int, int, int) int } // Logs calculates billable metrics for logs. diff --git a/pkg/metering/v1/logs.go b/pkg/metering/v1/logs.go index a19e20c5..4ba4c207 100644 --- a/pkg/metering/v1/logs.go +++ b/pkg/metering/v1/logs.go @@ -20,9 +20,6 @@ func NewLogs(logger *zap.Logger) metering.Logs { } func (meter *logs) Size(ld plog.Logs) int { - - meter.Logger.Debug("Calculating logs size") - total := 0 for i := 0; i < ld.ResourceLogs().Len(); i++ { resourceLog := ld.ResourceLogs().At(i) @@ -40,7 +37,6 @@ func (meter *logs) Size(ld plog.Logs) int { } } - meter.Logger.Debug("Logs size", zap.Int("size", total)) return total } diff --git a/pkg/metering/v1/traces.go b/pkg/metering/v1/traces.go new file mode 100644 index 00000000..f44334fe --- /dev/null +++ b/pkg/metering/v1/traces.go @@ -0,0 +1,118 @@ +package v1 + +import ( + "encoding/json" + + "github.com/SigNoz/signoz-otel-collector/pkg/metering" + "github.com/SigNoz/signoz-otel-collector/pkg/schema/common" + schema "github.com/SigNoz/signoz-otel-collector/pkg/schema/traces" + "go.opentelemetry.io/collector/pdata/ptrace" + "go.uber.org/zap" +) + +type traces struct { + Logger *zap.Logger + Sizer metering.Sizer + KeySizes map[string]int +} + +func NewTraces(logger *zap.Logger) metering.Traces { + return &traces{ + Logger: logger, + Sizer: metering.NewJSONSizer(logger), + KeySizes: map[string]int{ + "resources_string": len("\"resources_string\""), + "startTimeUnixNano": len("\"startTimeUnixNano\""), + "traceId": len("\"traceId\""), + "spanId": len("\"spanId\""), + "traceState": len("\"traceState\""), + "parentSpanId": len("\"parentSpanId\""), + "flags": len("\"flags\""), + "name": len("\"name\""), + "kind": len("\"kind\""), + "spanKind": len("\"spanKind\""), + "attributes_string": len("\"attributes_string\""), + "attributes_bool": len("\"attributes_bool\""), + "attributes_number": len("\"attributes_number\""), + "serviceName": len("\"serviceName\""), + "event": len("\"event\""), + "references": len("\"references\""), + }, + } +} + +func (meter *traces) Size(td ptrace.Traces) int { + total := 0 + + for i := 0; i < td.ResourceSpans().Len(); i++ { + resourceSpan := td.ResourceSpans().At(i) + resourceAttributesSize := meter.Sizer.SizeOfFlatPcommonMapInMapStringString(resourceSpan.Resource().Attributes()) + + for j := 0; j < resourceSpan.ScopeSpans().Len(); j++ { + scopeSpans := resourceSpan.ScopeSpans().At(j) + + for k := 0; k < scopeSpans.Spans().Len(); k++ { + span := scopeSpans.Spans().At(k) + + // Size of references + sizeOfOtelSpanRefs := 0 + otelSpanRefs, err := schema.NewOtelSpanRefs(span.Links(), span.ParentSpanID(), span.TraceID()) + if err != nil { + meter.Logger.Error("cannot create span refs", zap.Error(err)) + } else { + sizeOfOtelSpanRefs = meter.Sizer.SizeOfOtelSpanRefs(otelSpanRefs) + } + + // Size of service name + sizeOfServiceName := 0 + serviceName := common.ServiceName(resourceSpan.Resource()) + serviceNameBytes, err := json.Marshal(serviceName) + if err != nil { + meter.Logger.Error("cannot marshal service name", zap.Error(err), zap.String("val", serviceName)) + } else { + sizeOfServiceName = len(serviceNameBytes) + } + + // Size of events + events, _ := schema.NewEventsAndErrorEvents(span.Events(), serviceName, false) + sizeOfEvents := meter.Sizer.SizeOfEvents(events) + + // Size of attributes + sizeOfNumberAttributes, sizeOfStringAttributes, sizeOfBoolAttributes := meter.Sizer.SizeOfFlatPcommonMapInNumberStringBool(span.Attributes()) + + // Size of flags + sizeOfFlags := 0 + if span.Flags() != 0 { + sizeOfFlags = meter.Sizer.SizeOfInt(int(span.Flags())) + } + + // Let's start making the json object + // 2({}) + total += 2 + + meter.Sizer.TotalSizeIfKeyExistsAndValueIsMapOrSlice(meter.KeySizes["resources_string"], resourceAttributesSize, 2) + //:, + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["startTimeUnixNano"], meter.Sizer.SizeOfInt(int(span.StartTimestamp())), 2) + //:, + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["spanId"], meter.Sizer.SizeOfSpanID(span.SpanID()), 4) + //:"", + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["traceId"], meter.Sizer.SizeOfTraceID(span.TraceID()), 4) + //:"", + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["traceState"], len(span.TraceState().AsRaw()), 4) + //:"", + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["parentSpanId"], meter.Sizer.SizeOfSpanID(span.ParentSpanID()), 4) + //:"", + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["flags"], sizeOfFlags, 2) + //:, + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["name"], len(span.Name()), 4) + //:"", + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["kind"], meter.Sizer.SizeOfInt(int(span.Kind())), 2) + //:, + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["spanKind"], len(span.Kind().String()), 4) + //:"", + meter.Sizer.TotalSizeIfKeyExistsAndValueIsMapOrSlice(meter.KeySizes["attributes_string"], sizeOfStringAttributes, 2) + //:, + meter.Sizer.TotalSizeIfKeyExistsAndValueIsMapOrSlice(meter.KeySizes["attributes_bool"], sizeOfBoolAttributes, 2) + //:, + meter.Sizer.TotalSizeIfKeyExistsAndValueIsMapOrSlice(meter.KeySizes["attributes_number"], sizeOfNumberAttributes, 2) + //:, + meter.Sizer.TotalSizeIfKeyExists(meter.KeySizes["serviceName"], sizeOfServiceName, 2) + //:, + meter.Sizer.TotalSizeIfKeyExistsAndValueIsMapOrSlice(meter.KeySizes["event"], sizeOfEvents, 2) + //:, + meter.Sizer.TotalSizeIfKeyExistsAndValueIsMapOrSlice(meter.KeySizes["references"], sizeOfOtelSpanRefs, 4) - //:"", + 1 //, + } + + } + } + + return total +} +func (*traces) Count(td ptrace.Traces) int { + return td.SpanCount() +} diff --git a/pkg/metering/v1/traces_test.go b/pkg/metering/v1/traces_test.go new file mode 100644 index 00000000..475af33c --- /dev/null +++ b/pkg/metering/v1/traces_test.go @@ -0,0 +1,105 @@ +package v1 + +import ( + "fmt" + "testing" + + "github.com/SigNoz/signoz-otel-collector/pkg/pdatagen/ptracesgen" + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/pdata/ptrace" + "go.uber.org/zap" +) + +func TestTracesSizeWithNoEvents(t *testing.T) { + traces := ptracesgen.Generate( + ptracesgen.WithSpanCount(1), + ptracesgen.WithResourceAttributeCount(1), + // 4: SPAN_KIND_PRODUCER + ptracesgen.WithSpanKind(ptrace.SpanKindProducer), + ptracesgen.WithResourceAttributeStringValue("test"), + ) + + meter := NewTraces(zap.NewNop()) + size := meter.Size(traces) + + assert.Equal(t, 406, size) +} + +func TestTracesSizeWithEvents(t *testing.T) { + traces := ptracesgen.Generate( + ptracesgen.WithSpanCount(1), + ptracesgen.WithResourceAttributeCount(1), + ptracesgen.WithEventCount(2), + // 4: SPAN_KIND_PRODUCER + ptracesgen.WithSpanKind(ptrace.SpanKindProducer), + ptracesgen.WithResourceAttributeStringValue("test"), + ) + + meter := NewTraces(zap.NewNop()) + size := meter.Size(traces) + + assert.Equal(t, 540, size) +} + +func TestTracesSizeWith2SpansAnd2EventsAnd2ResourceAttributes(t *testing.T) { + traces := ptracesgen.Generate( + ptracesgen.WithSpanCount(2), + ptracesgen.WithResourceAttributeCount(2), + ptracesgen.WithEventCount(2), + // 4: SPAN_KIND_PRODUCER + ptracesgen.WithSpanKind(ptrace.SpanKindProducer), + ptracesgen.WithResourceAttributeStringValue("test"), + ) + + meter := NewTraces(zap.NewNop()) + + b, _ := (&ptrace.JSONMarshaler{}).MarshalTraces(traces) + fmt.Println(string(b)) + size := meter.Size(traces) + + assert.Equal(t, 1120, size) +} + +func TestTracesSizeWith2SpansAnd2EventsAnd2ResourceAttributesAndAttributes(t *testing.T) { + traces := ptracesgen.Generate( + ptracesgen.WithSpanCount(2), + ptracesgen.WithResourceAttributeCount(2), + ptracesgen.WithEventCount(2), + ptracesgen.WithSpanKind(ptrace.SpanKindClient), + ptracesgen.WithResourceAttributeStringValue("test"), + ptracesgen.WithAttributes(map[string]any{ + "float64": 342.5, + "int64": int64(342), + "string": "attribute", + "bool": false, + }), + ) + + meter := NewTraces(zap.NewNop()) + + b, _ := (&ptrace.JSONMarshaler{}).MarshalTraces(traces) + fmt.Println(string(b)) + size := meter.Size(traces) + + assert.Equal(t, 1368, size) +} + +func TestTracesSizeWithBoolAttributes(t *testing.T) { + traces := ptracesgen.Generate( + ptracesgen.WithSpanCount(1), + ptracesgen.WithResourceAttributeCount(1), + ptracesgen.WithSpanKind(ptrace.SpanKindClient), + ptracesgen.WithResourceAttributeStringValue("test"), + ptracesgen.WithAttributes(map[string]any{ + "bool1": false, + "bool2": true, + }), + ) + + meter := NewTraces(zap.NewNop()) + b, _ := (&ptrace.JSONMarshaler{}).MarshalTraces(traces) + fmt.Println(string(b)) + size := meter.Size(traces) + + assert.Equal(t, 451, size) +} diff --git a/pkg/pdatagen/go.mod b/pkg/pdatagen/go.mod deleted file mode 100644 index ef3dc222..00000000 --- a/pkg/pdatagen/go.mod +++ /dev/null @@ -1,19 +0,0 @@ -module github.com/SigNoz/signoz-otel-collector/pkg/pdatagen - -go 1.23.4 - -require go.opentelemetry.io/collector/pdata v1.22.0 - -require ( - github.com/gogo/protobuf v1.3.2 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.29.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.18.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect - google.golang.org/grpc v1.68.1 // indirect - google.golang.org/protobuf v1.35.2 // indirect -) diff --git a/pkg/pdatagen/go.sum b/pkg/pdatagen/go.sum deleted file mode 100644 index 8777ff54..00000000 --- a/pkg/pdatagen/go.sum +++ /dev/null @@ -1,72 +0,0 @@ -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opentelemetry.io/collector/pdata v1.22.0 h1:3yhjL46NLdTMoP8rkkcE9B0pzjf2973crn0KKhX5UrI= -go.opentelemetry.io/collector/pdata v1.22.0/go.mod h1:nLLf6uDg8Kn5g3WNZwGyu8+kf77SwOqQvMTb5AXEbEY= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= -google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= -google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= -google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/pdatagen/ptracesgen/options.go b/pkg/pdatagen/ptracesgen/options.go new file mode 100644 index 00000000..eb8a8677 --- /dev/null +++ b/pkg/pdatagen/ptracesgen/options.go @@ -0,0 +1,50 @@ +package ptracesgen + +import "go.opentelemetry.io/collector/pdata/ptrace" + +type generationOptions struct { + spanCount int + eventCount int + resourceAttributeCount int + resourceAttributeStringValue string + spanKind ptrace.SpanKind + attributes map[string]any +} + +type GenerationOption func(*generationOptions) + +func WithSpanCount(i int) GenerationOption { + return func(o *generationOptions) { + o.spanCount = i + } +} + +func WithEventCount(i int) GenerationOption { + return func(o *generationOptions) { + o.eventCount = i + } +} + +func WithSpanKind(k ptrace.SpanKind) GenerationOption { + return func(o *generationOptions) { + o.spanKind = k + } +} + +func WithResourceAttributeCount(i int) GenerationOption { + return func(o *generationOptions) { + o.resourceAttributeCount = i + } +} + +func WithResourceAttributeStringValue(s string) GenerationOption { + return func(o *generationOptions) { + o.resourceAttributeStringValue = s + } +} + +func WithAttributes(m map[string]any) GenerationOption { + return func(o *generationOptions) { + o.attributes = m + } +} diff --git a/pkg/pdatagen/ptracesgen/traces.go b/pkg/pdatagen/ptracesgen/traces.go new file mode 100644 index 00000000..b1f65c2b --- /dev/null +++ b/pkg/pdatagen/ptracesgen/traces.go @@ -0,0 +1,70 @@ +package ptracesgen + +import ( + "strconv" + "time" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/ptrace" +) + +func Generate(opts ...GenerationOption) ptrace.Traces { + generationOpts := generationOptions{ + eventCount: 0, + spanCount: 1, + resourceAttributeCount: 1, + resourceAttributeStringValue: "resource", + spanKind: ptrace.SpanKindClient, + attributes: map[string]any{}, + } + + for _, opt := range opts { + opt(&generationOpts) + } + + endTime := pcommon.NewTimestampFromTime(time.Now()) + traces := ptrace.NewTraces() + resourceSpan := traces.ResourceSpans().AppendEmpty() + for i := 0; i < generationOpts.resourceAttributeCount; i++ { + suffix := strconv.Itoa(i) + // Do not change the key name format in resource attributes below. + resourceSpan.Resource().Attributes().PutStr("resource."+suffix, generationOpts.resourceAttributeStringValue) + } + + scopeSpans := resourceSpan.ScopeSpans().AppendEmpty() + scopeSpans.Spans().EnsureCapacity(generationOpts.spanCount) + for i := 0; i < generationOpts.spanCount; i++ { + suffix := strconv.Itoa(i) + span := scopeSpans.Spans().AppendEmpty() + span.SetName("span." + suffix) + span.SetKind(generationOpts.spanKind) + span.SetStartTimestamp(endTime) + span.SetEndTimestamp(endTime) + span.SetTraceID(pcommon.TraceID([]byte("5B8EFFF798038103D269B633813FC60C"))) + span.SetSpanID(pcommon.SpanID([]byte("EEE19B7EC3C1B174"))) + span.SetParentSpanID(pcommon.SpanID([]byte("EEE19B7EC3C1B174"))) + + for j := 0; j < generationOpts.eventCount; j++ { + suffix := strconv.Itoa(j) + spanEvent := span.Events().AppendEmpty() + spanEvent.SetName("event." + suffix) + spanEvent.SetTimestamp(endTime) + } + + for k, v := range generationOpts.attributes { + switch v := v.(type) { + case string: + span.Attributes().PutStr(k, v) + case float64: + span.Attributes().PutDouble(k, v) + case bool: + span.Attributes().PutBool(k, v) + case int64: + span.Attributes().PutInt(k, v) + } + + } + } + + return traces +} diff --git a/pkg/schema/common/resource.go b/pkg/schema/common/resource.go new file mode 100644 index 00000000..c7c46385 --- /dev/null +++ b/pkg/schema/common/resource.go @@ -0,0 +1,15 @@ +package common + +import ( + "go.opentelemetry.io/collector/pdata/pcommon" + semconv "go.opentelemetry.io/collector/semconv/v1.5.0" +) + +func ServiceName(input pcommon.Resource) string { + service, found := input.Attributes().Get(semconv.AttributeServiceName) + if !found { + return "" + } + + return service.Str() +} diff --git a/pkg/schema/traces/event.go b/pkg/schema/traces/event.go new file mode 100644 index 00000000..0462c669 --- /dev/null +++ b/pkg/schema/traces/event.go @@ -0,0 +1,76 @@ +package traces + +import ( + "crypto/md5" + "encoding/json" + "fmt" + "strings" + + "github.com/google/uuid" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/ptrace" + "go.uber.org/zap/zapcore" +) + +// This has been copied from the exporter. The exporter needs to read from here. +type Event struct { + Name string `json:"name,omitempty"` + TimeUnixNano uint64 `json:"timeUnixNano,omitempty"` + AttributeMap map[string]string `json:"attributeMap,omitempty"` + IsError bool `json:"isError,omitempty"` +} + +func (e *Event) MarshalLogObject(enc zapcore.ObjectEncoder) error { + enc.AddString("name", e.Name) + enc.AddUint64("timeUnixNano", e.TimeUnixNano) + enc.AddBool("isError", e.IsError) + enc.AddString("attributeMap", fmt.Sprintf("%v", e.AttributeMap)) + return nil +} + +// This has been copied from the exporter. The exporter needs to read from here. +type ErrorEvent struct { + Event Event `json:"errorEvent,omitempty"` + ErrorID string `json:"errorID,omitempty"` + ErrorGroupID string `json:"errorGroupID,omitempty"` +} + +func NewEventsAndErrorEvents(input ptrace.SpanEventSlice, serviceName string, lowCardinalExceptionGrouping bool) ([]string, []ErrorEvent) { + events := make([]string, 0) + errorEvents := make([]ErrorEvent, 0) + + for i := 0; i < input.Len(); i++ { + event := Event{ + Name: input.At(i).Name(), + TimeUnixNano: uint64(input.At(i).Timestamp()), + AttributeMap: map[string]string{}, + IsError: false, + } + + input.At(i).Attributes().Range(func(k string, v pcommon.Value) bool { + event.AttributeMap[k] = v.AsString() + return true + }) + errorEvent := ErrorEvent{} + if event.Name == "exception" { + event.IsError = true + errorEvent.Event = event + uuidWithHyphen := uuid.New() + uuid := strings.Replace(uuidWithHyphen.String(), "-", "", -1) + errorEvent.ErrorID = uuid + var hash [16]byte + if lowCardinalExceptionGrouping { + hash = md5.Sum([]byte(serviceName + errorEvent.Event.AttributeMap["exception.type"])) + } else { + hash = md5.Sum([]byte(serviceName + errorEvent.Event.AttributeMap["exception.type"] + errorEvent.Event.AttributeMap["exception.message"])) + } + errorEvent.ErrorGroupID = fmt.Sprintf("%x", hash) + } + + stringEvent, _ := json.Marshal(event) + events = append(events, string(stringEvent)) + errorEvents = append(errorEvents, errorEvent) + } + + return events, errorEvents +} diff --git a/pkg/schema/traces/otel_span_ref.go b/pkg/schema/traces/otel_span_ref.go new file mode 100644 index 00000000..3069e280 --- /dev/null +++ b/pkg/schema/traces/otel_span_ref.go @@ -0,0 +1,61 @@ +package traces + +import ( + "github.com/SigNoz/signoz-otel-collector/utils" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/ptrace" + "go.uber.org/zap/zapcore" +) + +type OtelSpanRef struct { + TraceId string `json:"traceId,omitempty"` + SpanId string `json:"spanId,omitempty"` + RefType string `json:"refType,omitempty"` +} + +func NewOtelSpanRefs(links ptrace.SpanLinkSlice, parentSpanID pcommon.SpanID, traceID pcommon.TraceID) ([]OtelSpanRef, error) { + parentSpanIDSet := len([8]byte(parentSpanID)) != 0 + if !parentSpanIDSet && links.Len() == 0 { + return nil, nil + } + + refsCount := links.Len() + if parentSpanIDSet { + refsCount++ + } + + refs := make([]OtelSpanRef, 0, refsCount) + + // Put parent span ID at the first place because usually backends look for it + // as the first CHILD_OF item in the model.SpanRef slice. + if parentSpanIDSet { + refs = append(refs, OtelSpanRef{ + TraceId: utils.TraceIDToHexOrEmptyString(traceID), + SpanId: utils.SpanIDToHexOrEmptyString(parentSpanID), + RefType: "CHILD_OF", + }) + } + + for i := 0; i < links.Len(); i++ { + link := links.At(i) + + refs = append(refs, OtelSpanRef{ + TraceId: utils.TraceIDToHexOrEmptyString(link.TraceID()), + SpanId: utils.SpanIDToHexOrEmptyString(link.SpanID()), + + // Since Jaeger RefType is not captured in internal data, + // use SpanRefType_FOLLOWS_FROM by default. + // SpanRefType_CHILD_OF supposed to be set only from parentSpanID. + RefType: "FOLLOWS_FROM", + }) + } + + return refs, nil +} + +func (r *OtelSpanRef) MarshalLogObject(enc zapcore.ObjectEncoder) error { + enc.AddString("traceId", r.TraceId) + enc.AddString("spanId", r.SpanId) + enc.AddString("refType", r.RefType) + return nil +}