From 3eb9eef00fb0f3979353b48bf1ffd8857f731fa1 Mon Sep 17 00:00:00 2001 From: Thomas Poignant Date: Fri, 3 Nov 2023 14:28:14 +0100 Subject: [PATCH] Add new prometheus metrics to follow flag changes in the server (#1204) * WIP Signed-off-by: Thomas Poignant * add new metrics to check the lifecycle of the configuration file of your flag Signed-off-by: Thomas Poignant * Add test for notifier Signed-off-by: Thomas Poignant * lint fix Signed-off-by: Thomas Poignant --------- Signed-off-by: Thomas Poignant --- cmd/relayproxy/main.go | 9 +- cmd/relayproxy/metric/metrics.go | 116 +++++++++++++++++- cmd/relayproxy/metric/metrics_test.go | 47 +++++++ cmd/relayproxy/metric/notifier_prometheus.go | 34 +++++ .../metric/notifier_prometheus_test.go | 69 +++++++++++ cmd/relayproxy/service/gofeatureflag.go | 4 +- ...er_relayproxy.go => notifier_websocket.go} | 8 +- ...oxy_test.go => notifier_websocket_test.go} | 4 +- 8 files changed, 276 insertions(+), 15 deletions(-) create mode 100644 cmd/relayproxy/metric/notifier_prometheus.go create mode 100644 cmd/relayproxy/metric/notifier_prometheus_test.go rename cmd/relayproxy/service/{notifier_relayproxy.go => notifier_websocket.go} (51%) rename cmd/relayproxy/service/{notifier_relayproxy_test.go => notifier_websocket_test.go} (95%) diff --git a/cmd/relayproxy/main.go b/cmd/relayproxy/main.go index 670fc79bdff..0b589cb4fac 100644 --- a/cmd/relayproxy/main.go +++ b/cmd/relayproxy/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" "github.com/thomaspoignant/go-feature-flag/cmd/relayproxy/metric" + "github.com/thomaspoignant/go-feature-flag/notifier" "os" "github.com/spf13/pflag" @@ -74,8 +75,12 @@ func main() { } wsService := service.NewWebsocketService() defer wsService.Close() // close all the open connections - proxyNotifier := service.NewNotifierRelayProxy(wsService) - goff, err := service.NewGoFeatureFlagClient(proxyConf, zapLog, proxyNotifier) + prometheusNotifier := metric.NewPrometheusNotifier(metricsV2) + proxyNotifier := service.NewNotifierWebsocket(wsService) + goff, err := service.NewGoFeatureFlagClient(proxyConf, zapLog, []notifier.Notifier{ + prometheusNotifier, + proxyNotifier, + }) if err != nil { panic(err) } diff --git a/cmd/relayproxy/metric/metrics.go b/cmd/relayproxy/metric/metrics.go index 46b46f401d2..ad29738c727 100644 --- a/cmd/relayproxy/metric/metrics.go +++ b/cmd/relayproxy/metric/metrics.go @@ -8,31 +8,92 @@ import ( const GOFFSubSystem = "gofeatureflag" // NewMetrics is the constructor for the custom metrics - +// nolint:funlen func NewMetrics() (Metrics, error) { customRegistry := prom.NewRegistry() - metricToRegister := []prom.Collector{} + // counts the number of flag evaluations flagEvaluationCounter := prom.NewCounterVec(prom.CounterOpts{ Name: "flag_evaluations_total", Help: "Counter events for number of flag evaluation.", Subsystem: GOFFSubSystem, }, []string{"flag_name"}) - metricToRegister = append(metricToRegister, flagEvaluationCounter) + // counts the number of call to the all flag endpoint allFlagCounter := prom.NewCounter(prom.CounterOpts{ Name: "all_flag_evaluations_total", Help: "Counter events for number of all flags requests.", Subsystem: GOFFSubSystem, }) - metricToRegister = append(metricToRegister, allFlagCounter) + // counts the number of tracking events collected through the API collectEvalDataCounter := prom.NewCounter(prom.CounterOpts{ Name: "collect_eval_data_total", Help: "Counter events for data collector.", Subsystem: GOFFSubSystem, }) - metricToRegister = append(metricToRegister, collectEvalDataCounter) + + // counts the number of flag changes (create, update, delete) from your configuration + flagChange := prom.NewCounter(prom.CounterOpts{ + Name: "flag_changes_total", + Help: "Counter that counts the number of flag changes.", + Subsystem: GOFFSubSystem, + }) + + // counts the number of new flag created from your configuration + flagCreateCounter := prom.NewCounter(prom.CounterOpts{ + Name: "flag_create_total", + Help: "Counter that counts the number of flag created.", + Subsystem: GOFFSubSystem, + }) + + // counts the number of flag deleted from your configuration + flagDeleteCounter := prom.NewCounter(prom.CounterOpts{ + Name: "flag_delete_total", + Help: "Counter that counts the number of flag deleted.", + Subsystem: GOFFSubSystem, + }) + + // counts the number of flag updated from your configuration + flagUpdateCounter := prom.NewCounter(prom.CounterOpts{ + Name: "flag_update_total", + Help: "Counter that counts the number of flag updated.", + Subsystem: GOFFSubSystem, + }) + + // counts the number of update per flag + flagUpdateCounterVec := prom.NewCounterVec(prom.CounterOpts{ + Name: "flag_update", + Help: "Counter events for number of update per flag.", + Subsystem: GOFFSubSystem, + }, []string{"flag_name"}) + + // counts the number of delete per flag + flagDeleteCounterVec := prom.NewCounterVec(prom.CounterOpts{ + Name: "flag_delete", + Help: "Counter events for number of delete per flag.", + Subsystem: GOFFSubSystem, + }, []string{"flag_name"}) + + // flagCreateCounterVec counts the number of create per flag + flagCreateCounterVec := prom.NewCounterVec(prom.CounterOpts{ + Name: "flag_create", + Help: "Counter events for number of create per flag.", + Subsystem: GOFFSubSystem, + }, []string{"flag_name"}) + + metricToRegister := []prom.Collector{ + flagEvaluationCounter, + allFlagCounter, + collectEvalDataCounter, + flagChange, + flagCreateCounter, + flagDeleteCounter, + flagUpdateCounter, + flagUpdateCounterVec, + flagDeleteCounterVec, + flagCreateCounterVec, + } // register all the metric in the custom registry for _, metric := range metricToRegister { @@ -45,6 +106,13 @@ func NewMetrics() (Metrics, error) { flagEvaluationCounter: *flagEvaluationCounter, allFlagCounter: allFlagCounter, collectEvalDataCounter: collectEvalDataCounter, + flagChange: flagChange, + flagCreateCounter: flagCreateCounter, + flagDeleteCounter: flagDeleteCounter, + flagUpdateCounter: flagUpdateCounter, + flagUpdateCounterVec: *flagUpdateCounterVec, + flagDeleteCounterVec: *flagDeleteCounterVec, + flagCreateCounterVec: *flagCreateCounterVec, Registry: customRegistry, }, nil } @@ -55,6 +123,13 @@ type Metrics struct { flagEvaluationCounter prom.CounterVec allFlagCounter prom.Counter collectEvalDataCounter prom.Counter + flagChange prom.Counter + flagCreateCounter prom.Counter + flagDeleteCounter prom.Counter + flagUpdateCounter prom.Counter + flagUpdateCounterVec prom.CounterVec + flagDeleteCounterVec prom.CounterVec + flagCreateCounterVec prom.CounterVec } func (m *Metrics) IncFlagEvaluation(flagName string) { @@ -77,3 +152,34 @@ func (m *Metrics) IncCollectEvalData(numberEvents float64) { m.collectEvalDataCounter.Add(numberEvents) } } + +// IncFlagUpdated is incrementing the counters when a flag is updated. +func (m *Metrics) IncFlagUpdated(flagName string) { + if m.flagUpdateCounterVec.MetricVec != nil { + m.flagUpdateCounterVec.With(prom.Labels{"flag_name": flagName}).Inc() + m.flagUpdateCounter.Inc() + } +} + +// IncFlagDeleted is incrementing the counters when a flag is deleted. +func (m *Metrics) IncFlagDeleted(flagName string) { + if m.flagDeleteCounterVec.MetricVec != nil { + m.flagDeleteCounterVec.With(prom.Labels{"flag_name": flagName}).Inc() + m.flagDeleteCounter.Inc() + } +} + +// IncFlagCreated is incrementing the counters when a flag is created. +func (m *Metrics) IncFlagCreated(flagName string) { + if m.flagCreateCounterVec.MetricVec != nil { + m.flagCreateCounterVec.With(prom.Labels{"flag_name": flagName}).Inc() + m.flagCreateCounter.Inc() + } +} + +// IncFlagChange is incrementing the counters when a flag is created, updated or deleted. +func (m *Metrics) IncFlagChange() { + if m.flagChange != nil { + m.flagChange.Inc() + } +} diff --git a/cmd/relayproxy/metric/metrics_test.go b/cmd/relayproxy/metric/metrics_test.go index ba99e56695d..c0b9b5114b7 100644 --- a/cmd/relayproxy/metric/metrics_test.go +++ b/cmd/relayproxy/metric/metrics_test.go @@ -37,3 +37,50 @@ func TestMetrics_IncFlagEvaluation(t *testing.T) { assert.Equal(t, 2.0, testutil.ToFloat64(metricSrv.flagEvaluationCounter.WithLabelValues("test-flag"))) assert.Equal(t, 1.0, testutil.ToFloat64(metricSrv.flagEvaluationCounter.WithLabelValues("test-flag2"))) } + +func TestMetrics_IncFlagCreated(t *testing.T) { + metricSrv, err := NewMetrics() + assert.NoError(t, err) + + metricSrv.IncFlagCreated("test-flag") + metricSrv.IncFlagCreated("test-flag2") + + assert.Equal(t, 2.0, testutil.ToFloat64(metricSrv.flagCreateCounter)) + assert.Equal(t, 1.0, testutil.ToFloat64(metricSrv.flagCreateCounterVec.WithLabelValues("test-flag2"))) + assert.Equal(t, 1.0, testutil.ToFloat64(metricSrv.flagCreateCounterVec.WithLabelValues("test-flag"))) +} + +func TestMetrics_IncFlagUpdated(t *testing.T) { + metricSrv, err := NewMetrics() + assert.NoError(t, err) + + metricSrv.IncFlagUpdated("test-flag") + metricSrv.IncFlagUpdated("test-flag2") + + assert.Equal(t, 2.0, testutil.ToFloat64(metricSrv.flagUpdateCounter)) + assert.Equal(t, 1.0, testutil.ToFloat64(metricSrv.flagUpdateCounterVec.WithLabelValues("test-flag2"))) + assert.Equal(t, 1.0, testutil.ToFloat64(metricSrv.flagUpdateCounterVec.WithLabelValues("test-flag"))) +} + +func TestMetrics_IncFlagDeleted(t *testing.T) { + metricSrv, err := NewMetrics() + assert.NoError(t, err) + + metricSrv.IncFlagDeleted("test-flag") + metricSrv.IncFlagDeleted("test-flag2") + + assert.Equal(t, 2.0, testutil.ToFloat64(metricSrv.flagDeleteCounter)) + assert.Equal(t, 1.0, testutil.ToFloat64(metricSrv.flagDeleteCounterVec.WithLabelValues("test-flag2"))) + assert.Equal(t, 1.0, testutil.ToFloat64(metricSrv.flagDeleteCounterVec.WithLabelValues("test-flag"))) +} + +func TestMetrics_IncFlagChange(t *testing.T) { + metricSrv, err := NewMetrics() + assert.NoError(t, err) + + metricSrv.IncFlagChange() + metricSrv.IncFlagChange() + metricSrv.IncFlagChange() + + assert.Equal(t, 3.0, testutil.ToFloat64(metricSrv.flagChange)) +} diff --git a/cmd/relayproxy/metric/notifier_prometheus.go b/cmd/relayproxy/metric/notifier_prometheus.go new file mode 100644 index 00000000000..b16eaf1c51f --- /dev/null +++ b/cmd/relayproxy/metric/notifier_prometheus.go @@ -0,0 +1,34 @@ +package metric + +import ( + "github.com/thomaspoignant/go-feature-flag/notifier" +) + +type notifierPrometheus struct { + metricsService Metrics +} + +func NewPrometheusNotifier(metricsService Metrics) notifier.Notifier { + return ¬ifierPrometheus{ + metricsService: metricsService, + } +} + +func (n *notifierPrometheus) Notify(diff notifier.DiffCache) error { + if !diff.HasDiff() { + return nil + } + n.metricsService.IncFlagChange() + for flagName := range diff.Deleted { + n.metricsService.IncFlagDeleted(flagName) + } + + for flagName := range diff.Added { + n.metricsService.IncFlagCreated(flagName) + } + + for flagName := range diff.Updated { + n.metricsService.IncFlagUpdated(flagName) + } + return nil +} diff --git a/cmd/relayproxy/metric/notifier_prometheus_test.go b/cmd/relayproxy/metric/notifier_prometheus_test.go new file mode 100644 index 00000000000..ca00faa7b69 --- /dev/null +++ b/cmd/relayproxy/metric/notifier_prometheus_test.go @@ -0,0 +1,69 @@ +package metric + +import ( + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/thomaspoignant/go-feature-flag/internal/flag" + "github.com/thomaspoignant/go-feature-flag/notifier" + "testing" +) + +func TestPrometheusNotifier_with_diff(t *testing.T) { + m, err := NewMetrics() + assert.NoError(t, err) + + diff := notifier.DiffCache{ + Deleted: map[string]flag.Flag{ + "test-flag": &flag.InternalFlag{}, + }, + Updated: map[string]notifier.DiffUpdated{ + "test-flag2": { + Before: &flag.InternalFlag{}, + After: &flag.InternalFlag{}, + }, + }, + Added: map[string]flag.Flag{ + "test-flagAdd1": &flag.InternalFlag{}, + "test-flagAdd2": &flag.InternalFlag{}, + "test-flagAdd3": &flag.InternalFlag{}, + }, + } + + n := NewPrometheusNotifier(m) + err = n.Notify(diff) + assert.NoError(t, err) + + assert.Equal(t, 1.0, testutil.ToFloat64(m.flagDeleteCounter)) + assert.Equal(t, 1.0, testutil.ToFloat64(m.flagUpdateCounter)) + assert.Equal(t, 3.0, testutil.ToFloat64(m.flagCreateCounter)) + assert.Equal(t, 1.0, testutil.ToFloat64(m.flagCreateCounterVec.WithLabelValues("test-flagAdd1"))) + assert.Equal(t, 1.0, testutil.ToFloat64(m.flagCreateCounterVec.WithLabelValues("test-flagAdd2"))) + assert.Equal(t, 1.0, testutil.ToFloat64(m.flagCreateCounterVec.WithLabelValues("test-flagAdd3"))) + assert.Equal(t, 1.0, testutil.ToFloat64(m.flagUpdateCounterVec.WithLabelValues("test-flag2"))) + assert.Equal(t, 1.0, testutil.ToFloat64(m.flagDeleteCounterVec.WithLabelValues("test-flag"))) + assert.Equal(t, 1.0, testutil.ToFloat64(m.flagChange)) +} + +func TestPrometheusNotifier_no_diff(t *testing.T) { + m, err := NewMetrics() + assert.NoError(t, err) + + diff := notifier.DiffCache{ + Deleted: map[string]flag.Flag{}, + Updated: map[string]notifier.DiffUpdated{}, + Added: map[string]flag.Flag{}, + } + + n := NewPrometheusNotifier(m) + err = n.Notify(diff) + assert.NoError(t, err) + assert.Equal(t, 0.0, testutil.ToFloat64(m.flagDeleteCounter)) + assert.Equal(t, 0.0, testutil.ToFloat64(m.flagUpdateCounter)) + assert.Equal(t, 0.0, testutil.ToFloat64(m.flagCreateCounter)) + assert.Equal(t, 0.0, testutil.ToFloat64(m.flagCreateCounterVec.WithLabelValues("test-flagAdd1"))) + assert.Equal(t, 0.0, testutil.ToFloat64(m.flagCreateCounterVec.WithLabelValues("test-flagAdd2"))) + assert.Equal(t, 0.0, testutil.ToFloat64(m.flagCreateCounterVec.WithLabelValues("test-flagAdd3"))) + assert.Equal(t, 0.0, testutil.ToFloat64(m.flagUpdateCounterVec.WithLabelValues("test-flag2"))) + assert.Equal(t, 0.0, testutil.ToFloat64(m.flagDeleteCounterVec.WithLabelValues("test-flag"))) + assert.Equal(t, 0.0, testutil.ToFloat64(m.flagChange)) +} diff --git a/cmd/relayproxy/service/gofeatureflag.go b/cmd/relayproxy/service/gofeatureflag.go index 68ea5d57591..1086654a4f1 100644 --- a/cmd/relayproxy/service/gofeatureflag.go +++ b/cmd/relayproxy/service/gofeatureflag.go @@ -33,7 +33,7 @@ import ( func NewGoFeatureFlagClient( proxyConf *config.Config, logger *zap.Logger, - proxyNotifier notifier.Notifier, + notifiers []notifier.Notifier, ) (*ffclient.GoFeatureFlag, error) { var mainRetriever retriever.Retriever var err error @@ -72,7 +72,7 @@ func NewGoFeatureFlagClient( return nil, err } } - notif = append(notif, proxyNotifier) + notif = append(notif, notifiers...) f := ffclient.Config{ PollingInterval: time.Duration(proxyConf.PollingInterval) * time.Millisecond, diff --git a/cmd/relayproxy/service/notifier_relayproxy.go b/cmd/relayproxy/service/notifier_websocket.go similarity index 51% rename from cmd/relayproxy/service/notifier_relayproxy.go rename to cmd/relayproxy/service/notifier_websocket.go index 383197bf924..3d20c9dae9f 100644 --- a/cmd/relayproxy/service/notifier_relayproxy.go +++ b/cmd/relayproxy/service/notifier_websocket.go @@ -4,17 +4,17 @@ import ( "github.com/thomaspoignant/go-feature-flag/notifier" ) -type notifierRelayProxy struct { +type notifierWebsocket struct { websocketService WebsocketService } -func NewNotifierRelayProxy(websocketService WebsocketService) notifier.Notifier { - return ¬ifierRelayProxy{ +func NewNotifierWebsocket(websocketService WebsocketService) notifier.Notifier { + return ¬ifierWebsocket{ websocketService: websocketService, } } -func (n *notifierRelayProxy) Notify(diff notifier.DiffCache) error { +func (n *notifierWebsocket) Notify(diff notifier.DiffCache) error { n.websocketService.BroadcastFlagChanges(diff) return nil } diff --git a/cmd/relayproxy/service/notifier_relayproxy_test.go b/cmd/relayproxy/service/notifier_websocket_test.go similarity index 95% rename from cmd/relayproxy/service/notifier_relayproxy_test.go rename to cmd/relayproxy/service/notifier_websocket_test.go index fa4d72cb4b4..3f03ef5e15a 100644 --- a/cmd/relayproxy/service/notifier_relayproxy_test.go +++ b/cmd/relayproxy/service/notifier_websocket_test.go @@ -36,8 +36,8 @@ func TestNotify(t *testing.T) { // Create a mock WebsocketService mockService := &mockWebsocketService{} - // Create the notifierRelayProxy instance with the mock service - n := service.NewNotifierRelayProxy(mockService) + // Create the notifierWebsocket instance with the mock service + n := service.NewNotifierWebsocket(mockService) // Prepare the input data diff := notifier.DiffCache{