diff --git a/client/async.go b/client/async.go index fbd3537..d48eae3 100644 --- a/client/async.go +++ b/client/async.go @@ -247,7 +247,7 @@ func (a *gRPCClientAsync) sendEvents(req *pb.SendEventsRequest, retryCount int) if retryCount > a.maxRetries { l.Info("dropped events due to max retries") for _, e := range req.Events { - metrics.ClientRequestsDroppedCounter.WithLabelValues( + metrics.AsyncClientRequestsDroppedCounter.WithLabelValues( e.Topic, ).Inc() } diff --git a/client/client.go b/client/client.go index cf40e68..1d07cc5 100644 --- a/client/client.go +++ b/client/client.go @@ -12,7 +12,6 @@ import ( "fmt" "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" "github.com/opentracing/opentracing-go" - "github.com/prometheus/client_golang/prometheus" "github.com/topfreegames/eventsgateway/v4/metrics" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/otel" @@ -115,41 +114,17 @@ func (c *Client) newGRPCClient( "async": async, }) - c.registerMetrics(configPrefix) + err := metrics.RegisterMetrics(configPrefix, c.config) + if err != nil { + return nil, err + } + if async { return newGRPCClientAsync(configPrefix, c.config, c.logger, c.serverAddress, client, opts...) } return newGRPCClientSync(configPrefix, c.config, c.logger, c.serverAddress, client, opts...) } -func (c *Client) registerMetrics(configPrefix string) { - latencyBucketsConf := fmt.Sprintf("%sclient.prometheus.buckets.latency", configPrefix) - c.config.SetDefault(latencyBucketsConf, []float64{3, 5, 10, 50, 100, 300, 500, 1000, 5000}) - latencyBuckets := c.config.Get(latencyBucketsConf).([]float64) - - metrics.ClientRequestsResponseTime = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: "eventsgateway", - Subsystem: "client", - Name: "response_time_ms", - Help: "the response time in ms of calls to server", - Buckets: latencyBuckets, - }, - []string{"route", "topic", "retry"}, - ) - - collectors := []prometheus.Collector{ - metrics.ClientRequestsResponseTime, - metrics.ClientRequestsSuccessCounter, - metrics.ClientRequestsFailureCounter, - metrics.ClientRequestsDroppedCounter, - } - err := metrics.RegisterMetrics(collectors) - if err != nil { - c.logger.WithError(err).Error("failed to register metric") - } -} - // Send sends an event to another server via grpc using the client's configured topic func (c *Client) Send( ctx context.Context, diff --git a/client/sync.go b/client/sync.go index da163a9..e0af5cd 100644 --- a/client/sync.go +++ b/client/sync.go @@ -111,12 +111,12 @@ func (s *gRPCClientSync) metricsReporterInterceptor( event.Topic, "0", ).Observe(elapsedTime) + l.WithFields(map[string]interface{}{ "elapsedTime": elapsedTime, "reply": reply.(*pb.SendEventResponse), }).Debug("request processed") }(time.Now()) - if err := invoker(ctx, method, req, reply, cc, opts...); err != nil { l.WithError(err).Error("error processing request") metrics.ClientRequestsFailureCounter.WithLabelValues( diff --git a/config/local.yaml b/config/local.yaml index b2cb646..6cef023 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -19,8 +19,8 @@ client: serverAddress: eventsgateway-api:5000 #eventsgateway-api:5000 timeout: 500ms loadtestclient: - duration: 20s - threads: 20 + duration: 10s + threads: 10 randSleepCeilingMs: 500 randPropsSize: small # small, medium, large, jumbo prometheus: diff --git a/docker-compose.yaml b/docker-compose.yaml index c40fe05..2e6483d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -85,7 +85,7 @@ services: command: - sh - -c - - 'go run main.go load-test' + - 'go run main.go load-test -d' networks: - eventsgateway diff --git a/metrics/metrics.go b/metrics/metrics.go index e38039a..5769ce4 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -25,16 +25,30 @@ package metrics import ( "errors" "github.com/prometheus/client_golang/prometheus" + "github.com/spf13/viper" ) +const metricsNamespace = "eventsgateway" +const metricsSubsystem = "client" + var ( + // ClientRequestsResponseTime summary, observes the API response time as perceived by the client - ClientRequestsResponseTime *prometheus.HistogramVec + ClientRequestsResponseTime = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "response_time_ms", + Help: "the response time in ms of calls to server", + Buckets: []float64{3, 5, 10, 50, 100, 300, 500, 1000, 5000}, + }, + []string{"route", "topic", "retry"}, + ) // ClientRequestsSuccessCounter is the count of successfull calls to the server ClientRequestsSuccessCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "eventsgateway", - Subsystem: "client", + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, Name: "requests_success_counter", Help: "the count of successfull client requests to the server", }, @@ -43,21 +57,21 @@ var ( // ClientRequestsFailureCounter is the count of failed calls to the server ClientRequestsFailureCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "eventsgateway", - Subsystem: "client", + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, Name: "requests_failure_counter", Help: "the count of failed client requests to the server", }, []string{"route", "topic", "retry", "reason"}, ) - // ClientRequestsDroppedCounter is the count of requests that were dropped due - // to req.Retry > maxRetries - ClientRequestsDroppedCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "eventsgateway", - Subsystem: "client", - Name: "requests_dropped_counter", - Help: "the count of dropped client requests to the server", + // AsyncClientRequestsDroppedCounter is the count of requests that were dropped due + // to req.Retry > maxRetries. Only available for Async mode + AsyncClientRequestsDroppedCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "async_requests_dropped_counter", + Help: "the count of dropped client async requests to the server", }, []string{"topic"}, ) @@ -66,7 +80,14 @@ var ( // RegisterMetrics is a wrapper to handle prometheus.AlreadyRegisteredError; // it only returns an error if the metric wasn't already registered and there was an // actual error registering it. -func RegisterMetrics(collectors []prometheus.Collector) error { +func RegisterMetrics(configPrefix string, config *viper.Viper) error { + collectors := []prometheus.Collector{ + ClientRequestsResponseTime, + ClientRequestsSuccessCounter, + ClientRequestsFailureCounter, + AsyncClientRequestsDroppedCounter, + } + for _, collector := range collectors { err := prometheus.Register(collector) if err != nil {