From 886ab842d364c42c98ef92827e39d56765bc8398 Mon Sep 17 00:00:00 2001 From: helder-junior Date: Wed, 14 Aug 2024 10:49:20 -0300 Subject: [PATCH] Support Prometheus Client metrics via DefaultRegistry (#32) * Fix client sync and async credentials provider * Init prometheus server on loadtest app * Set up load test application to report metrics * Upgrade prometheus client * Move all client metrics registration to metrics.go file * Downgrade prometheus lib version * Debugging client metrics * Change back to register metrics. Add debug metric name * Change metrics Subsystem name to avoid conflicts with older metrics * Client warn alreadyRegistered metrics * Bump prometheus lib 1.14.0 * Fix request duration calculation on client side --- Makefile | 2 +- client/async.go | 9 ++++---- client/client.go | 34 ++++----------------------- client/sync.go | 6 +++-- config/local.yaml | 9 +++++--- docker-compose.yaml | 13 +++++++++++ go.mod | 12 +++++----- go.sum | 26 ++++++++++++++------- metrics/metrics.go | 50 +++++++++++++++++++++++++++++----------- server/config/local.yaml | 4 ++-- testing/prometheus.yml | 2 +- tools/loadtest.go | 28 ++++++++++++++++++++++ 12 files changed, 125 insertions(+), 70 deletions(-) diff --git a/Makefile b/Makefile index 284b823..b7cc839 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ producer: @docker run -it -v ./:/app --network eventsgateway_eventsgateway eventsgateway-client-dev sh -c "go run main.go producer -d" load-test: - @docker run -it -v ./:/app --network eventsgateway_eventsgateway eventsgateway-client-dev sh -c "go run main.go load-test -d" + @docker compose up eventsgateway-client deps-start: @docker compose up -d eventsgateway-api --wait diff --git a/client/async.go b/client/async.go index 389a979..b27a470 100644 --- a/client/async.go +++ b/client/async.go @@ -10,6 +10,7 @@ package client import ( "context" "fmt" + "google.golang.org/grpc/credentials/insecure" "math" "sync" "time" @@ -81,7 +82,7 @@ func newGRPCClientAsync( "timeout": a.timeout, }) - if err := a.configureGRPCForwarderClient(serverAddress, client); err != nil { + if err := a.configureGRPCForwarderClient(serverAddress, client, opts...); err != nil { return nil, err } @@ -114,7 +115,7 @@ func (a *gRPCClientAsync) configureGRPCForwarderClient( }).Info("connecting to grpc server") dialOpts := append( []grpc.DialOption{ - grpc.WithInsecure(), + grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithChainUnaryInterceptor( a.metricsReporterInterceptor, ), @@ -148,7 +149,7 @@ func (a *gRPCClientAsync) metricsReporterInterceptor( retry := fmt.Sprintf("%d", req.(*pb.SendEventsRequest).Retry) defer func(startTime time.Time) { - elapsedTime := float64(time.Since(startTime).Nanoseconds() / 1000000) + elapsedTime := float64(time.Now().UnixMilli() - startTime.UnixMilli()) for _, e := range events { metrics.ClientRequestsResponseTime.WithLabelValues( method, @@ -246,7 +247,7 @@ func (a *gRPCClientAsync) sendEvents(req *pb.SendEventsRequest, retryCount int) if retryCount > a.maxRetries { l.Info("dropped events due to max retries") for _, e := range req.Events { - metrics.ClientRequestsDroppedCounter.WithLabelValues( + metrics.AsyncClientRequestsDroppedCounter.WithLabelValues( e.Topic, ).Inc() } diff --git a/client/client.go b/client/client.go index cf40e68..e90f6ec 100644 --- a/client/client.go +++ b/client/client.go @@ -12,7 +12,6 @@ import ( "fmt" "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" "github.com/opentracing/opentracing-go" - "github.com/prometheus/client_golang/prometheus" "github.com/topfreegames/eventsgateway/v4/metrics" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/otel" @@ -114,42 +113,17 @@ func (c *Client) newGRPCClient( "serverAddress": c.serverAddress, "async": async, }) + err := metrics.RegisterMetrics(configPrefix, c.config) + if err != nil { + return nil, err + } - c.registerMetrics(configPrefix) if async { return newGRPCClientAsync(configPrefix, c.config, c.logger, c.serverAddress, client, opts...) } return newGRPCClientSync(configPrefix, c.config, c.logger, c.serverAddress, client, opts...) } -func (c *Client) registerMetrics(configPrefix string) { - latencyBucketsConf := fmt.Sprintf("%sclient.prometheus.buckets.latency", configPrefix) - c.config.SetDefault(latencyBucketsConf, []float64{3, 5, 10, 50, 100, 300, 500, 1000, 5000}) - latencyBuckets := c.config.Get(latencyBucketsConf).([]float64) - - metrics.ClientRequestsResponseTime = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: "eventsgateway", - Subsystem: "client", - Name: "response_time_ms", - Help: "the response time in ms of calls to server", - Buckets: latencyBuckets, - }, - []string{"route", "topic", "retry"}, - ) - - collectors := []prometheus.Collector{ - metrics.ClientRequestsResponseTime, - metrics.ClientRequestsSuccessCounter, - metrics.ClientRequestsFailureCounter, - metrics.ClientRequestsDroppedCounter, - } - err := metrics.RegisterMetrics(collectors) - if err != nil { - c.logger.WithError(err).Error("failed to register metric") - } -} - // Send sends an event to another server via grpc using the client's configured topic func (c *Client) Send( ctx context.Context, diff --git a/client/sync.go b/client/sync.go index 7306c4f..bd4bf55 100644 --- a/client/sync.go +++ b/client/sync.go @@ -15,6 +15,7 @@ import ( "github.com/topfreegames/eventsgateway/v4/metrics" pb "github.com/topfreegames/protos/eventsgateway/grpc/generated" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "time" ) @@ -69,7 +70,7 @@ func (s *gRPCClientSync) configureGRPCForwarderClient( dialOpts := append( []grpc.DialOption{ - grpc.WithInsecure(), + grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithChainUnaryInterceptor( s.metricsReporterInterceptor, ), @@ -104,12 +105,13 @@ func (s *gRPCClientSync) metricsReporterInterceptor( event := req.(*pb.Event) defer func(startTime time.Time) { - elapsedTime := float64(time.Since(startTime).Nanoseconds() / 1000000) + elapsedTime := float64(time.Now().UnixMilli() - startTime.UnixMilli()) metrics.ClientRequestsResponseTime.WithLabelValues( method, event.Topic, "0", ).Observe(elapsedTime) + l.WithFields(map[string]interface{}{ "elapsedTime": elapsedTime, "reply": reply.(*pb.SendEventResponse), diff --git a/config/local.yaml b/config/local.yaml index eedbb63..ebd8541 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -16,13 +16,16 @@ client: numRoutines: 2 retryInterval: 2s grpc: - serverAddress: localhost:5000 #eventsgateway-api:5000 + serverAddress: eventsgateway-api:5000 #eventsgateway-api:5000 timeout: 500ms loadtestclient: - duration: 20s - threads: 20 + duration: 60s + threads: 15 randSleepCeilingMs: 500 randPropsSize: small # small, medium, large, jumbo + prometheus: + enabled: true + port: 0.0.0.0:9002 opentelemetry: enabled: false serviceName: "eventsgateway-client-loadtest-opentelemetry" diff --git a/docker-compose.yaml b/docker-compose.yaml index 193a833..2e6483d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -76,6 +76,19 @@ services: networks: - eventsgateway + eventsgateway-client: + image: eventsgateway-client-dev + ports: + - "9002:9002" + volumes: + - ./:/app + command: + - sh + - -c + - 'go run main.go load-test -d' + networks: + - eventsgateway + jupyter: image: quay.io/jupyter/all-spark-notebook ports: diff --git a/go.mod b/go.mod index a1b971c..9c65fe7 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,12 @@ require ( github.com/Shopify/sarama v1.35.0 github.com/golang/mock v1.4.4 github.com/google/uuid v1.3.0 + github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 github.com/onsi/ginkgo/v2 v2.1.4 github.com/onsi/gomega v1.20.0 github.com/opentracing/opentracing-go v1.2.0 - github.com/prometheus/client_golang v1.12.2 + github.com/prometheus/client_golang v1.14.0 github.com/satori/go.uuid v1.2.0 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v0.0.1 @@ -50,9 +51,9 @@ require ( github.com/mitchellh/mapstructure v1.1.2 // indirect github.com/pelletier/go-toml v1.2.0 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.32.1 // indirect - github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/spf13/afero v1.1.2 // indirect github.com/spf13/cast v1.3.0 // indirect @@ -60,11 +61,10 @@ require ( github.com/spf13/pflag v1.0.3 // indirect golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect golang.org/x/net v0.0.0-20220708220712-1185a9018129 // indirect - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect golang.org/x/text v0.3.7 // indirect google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1 // indirect - google.golang.org/protobuf v1.28.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index aa8bfc5..ee9506a 100644 --- a/go.sum +++ b/go.sum @@ -109,9 +109,11 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -185,6 +187,7 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= @@ -291,24 +294,27 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= -github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -480,6 +486,7 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220708220712-1185a9018129 h1:vucSRfWwTsoXro7P+3Cjlr6flUMtzCwzlvkxEQtHHB0= @@ -490,8 +497,9 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -502,8 +510,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -713,8 +722,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/metrics/metrics.go b/metrics/metrics.go index e38039a..79d4bb7 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -25,16 +25,31 @@ package metrics import ( "errors" "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + "github.com/spf13/viper" ) +const metricsNamespace = "eventsgateway" +const metricsSubsystem = "client_v4" + var ( + // ClientRequestsResponseTime summary, observes the API response time as perceived by the client - ClientRequestsResponseTime *prometheus.HistogramVec + ClientRequestsResponseTime = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "response_time_ms", + Help: "the response time in ms of calls to server", + Buckets: []float64{3, 5, 10, 50, 100, 300, 500, 1000, 5000}, + }, + []string{"route", "topic", "retry"}, + ) // ClientRequestsSuccessCounter is the count of successfull calls to the server ClientRequestsSuccessCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "eventsgateway", - Subsystem: "client", + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, Name: "requests_success_counter", Help: "the count of successfull client requests to the server", }, @@ -43,21 +58,21 @@ var ( // ClientRequestsFailureCounter is the count of failed calls to the server ClientRequestsFailureCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "eventsgateway", - Subsystem: "client", + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, Name: "requests_failure_counter", Help: "the count of failed client requests to the server", }, []string{"route", "topic", "retry", "reason"}, ) - // ClientRequestsDroppedCounter is the count of requests that were dropped due - // to req.Retry > maxRetries - ClientRequestsDroppedCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "eventsgateway", - Subsystem: "client", - Name: "requests_dropped_counter", - Help: "the count of dropped client requests to the server", + // AsyncClientRequestsDroppedCounter is the count of requests that were dropped due + // to req.Retry > maxRetries. Only available for Async mode + AsyncClientRequestsDroppedCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "async_requests_dropped_counter", + Help: "the count of dropped client async requests to the server", }, []string{"topic"}, ) @@ -66,10 +81,19 @@ var ( // RegisterMetrics is a wrapper to handle prometheus.AlreadyRegisteredError; // it only returns an error if the metric wasn't already registered and there was an // actual error registering it. -func RegisterMetrics(collectors []prometheus.Collector) error { +func RegisterMetrics(configPrefix string, config *viper.Viper) error { + + collectors := []prometheus.Collector{ + ClientRequestsResponseTime, + ClientRequestsSuccessCounter, + ClientRequestsFailureCounter, + AsyncClientRequestsDroppedCounter, + } + for _, collector := range collectors { err := prometheus.Register(collector) if err != nil { + logrus.New().Warnf("Error while registering EG metrics: %s", err) var alreadyRegisteredError prometheus.AlreadyRegisteredError if !errors.As(err, &alreadyRegisteredError) { return err diff --git a/server/config/local.yaml b/server/config/local.yaml index 1d3f16e..2c9291b 100644 --- a/server/config/local.yaml +++ b/server/config/local.yaml @@ -28,11 +28,11 @@ pprof: enabled: true address: localhost:6060 otlp: - enabled: true + enabled: false serviceName: "eventsgateway-server" traceSamplingRatio: 0.2 jaegerHost: jaeger jaegerPort: 4317 prometheus: - enabled: false + enabled: true port: 0.0.0.0:9091 \ No newline at end of file diff --git a/testing/prometheus.yml b/testing/prometheus.yml index b7429ec..6b0dde2 100644 --- a/testing/prometheus.yml +++ b/testing/prometheus.yml @@ -2,7 +2,7 @@ scrape_configs: - job_name: app scrape_interval: 5s static_configs: - - targets: ['eventsgateway-api:9091'] + - targets: ['eventsgateway-api:9091', 'eventsgateway-client:9002'] # prometheus metrics # - job_name: 'prometheus' diff --git a/tools/loadtest.go b/tools/loadtest.go index 17d70d8..499c86f 100644 --- a/tools/loadtest.go +++ b/tools/loadtest.go @@ -25,7 +25,10 @@ package tools import ( "context" "fmt" + "github.com/gorilla/mux" "github.com/opentracing/opentracing-go" + "github.com/prometheus/client_golang/prometheus/promhttp" + log "github.com/sirupsen/logrus" jaegerclient "github.com/uber/jaeger-client-go" jaegercfg "github.com/uber/jaeger-client-go/config" "go.opentelemetry.io/otel" @@ -36,6 +39,7 @@ import ( semconv "go.opentelemetry.io/otel/semconv/v1.7.0" "io" "math/rand" + "net/http" "os" "os/signal" "sync" @@ -156,8 +160,32 @@ func (lt *LoadTest) configureOpenTelemetry() error { return nil } +func (lt *LoadTest) startMetricsServer() { + go func() { + envEnabled := lt.config.GetString("loadtestclient.prometheus.enabled") + if envEnabled != "true" { + log.Warn("Prometheus web server disabled") + return + } + + r := mux.NewRouter() + r.Handle("/metrics", promhttp.Handler()) + + s := &http.Server{ + Addr: lt.config.GetString("loadtestclient.prometheus.port"), + ReadTimeout: 8 * time.Second, + WriteTimeout: 8 * time.Second, + MaxHeaderBytes: 1 << 20, + Handler: r, + } + log.Info("Starting Metrics server...") + log.Fatal(s.ListenAndServe()) + }() +} + // Run starts all runners func (lt *LoadTest) Run() { + lt.startMetricsServer() lt.wg.Add(lt.threads) for _, runner := range lt.runners { go runner.run()