Add new prometheus metrics to follow flag changes in the server (#1204)

* WIP Signed-off-by: Thomas Poignant <[email protected]> * add new metrics to check the lifecycle of the configuration file of your flag Signed-off-by: Thomas Poignant <[email protected]> * Add test for notifier Signed-off-by: Thomas Poignant <[email protected]> * lint fix Signed-off-by: Thomas Poignant <[email protected]> --------- Signed-off-by: Thomas Poignant <[email protected]>
thomaspoignant · Nov 3, 2023 · 3eb9eef · 3eb9eef
1 parent da42931
commit 3eb9eef
Show file tree

Hide file tree

Showing 8 changed files with 276 additions and 15 deletions.
diff --git a/cmd/relayproxy/main.go b/cmd/relayproxy/main.go
@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"github.com/thomaspoignant/go-feature-flag/cmd/relayproxy/metric"
+	"github.com/thomaspoignant/go-feature-flag/notifier"
 	"os"
 
 	"github.com/spf13/pflag"
@@ -74,8 +75,12 @@ func main() {
 	}
 	wsService := service.NewWebsocketService()
 	defer wsService.Close() // close all the open connections
-	proxyNotifier := service.NewNotifierRelayProxy(wsService)
-	goff, err := service.NewGoFeatureFlagClient(proxyConf, zapLog, proxyNotifier)
+	prometheusNotifier := metric.NewPrometheusNotifier(metricsV2)
+	proxyNotifier := service.NewNotifierWebsocket(wsService)
+	goff, err := service.NewGoFeatureFlagClient(proxyConf, zapLog, []notifier.Notifier{
+		prometheusNotifier,
+		proxyNotifier,
+	})
 	if err != nil {
 		panic(err)
 	}

diff --git a/cmd/relayproxy/metric/metrics.go b/cmd/relayproxy/metric/metrics.go
@@ -8,31 +8,92 @@ import (
 const GOFFSubSystem = "gofeatureflag"
 
 // NewMetrics is the constructor for the custom metrics
-
+// nolint:funlen
 func NewMetrics() (Metrics, error) {
 	customRegistry := prom.NewRegistry()
-	metricToRegister := []prom.Collector{}
 
+	// counts the number of flag evaluations
 	flagEvaluationCounter := prom.NewCounterVec(prom.CounterOpts{
 		Name:      "flag_evaluations_total",
 		Help:      "Counter events for number of flag evaluation.",
 		Subsystem: GOFFSubSystem,
 	}, []string{"flag_name"})
-	metricToRegister = append(metricToRegister, flagEvaluationCounter)
 
+	// counts the number of call to the all flag endpoint
 	allFlagCounter := prom.NewCounter(prom.CounterOpts{
 		Name:      "all_flag_evaluations_total",
 		Help:      "Counter events for number of all flags requests.",
 		Subsystem: GOFFSubSystem,
 	})
-	metricToRegister = append(metricToRegister, allFlagCounter)
 
+	// counts the number of tracking events collected through the API
 	collectEvalDataCounter := prom.NewCounter(prom.CounterOpts{
 		Name:      "collect_eval_data_total",
 		Help:      "Counter events for data collector.",
 		Subsystem: GOFFSubSystem,
 	})
-	metricToRegister = append(metricToRegister, collectEvalDataCounter)
+
+	// counts the number of flag changes (create, update, delete) from your configuration
+	flagChange := prom.NewCounter(prom.CounterOpts{
+		Name:      "flag_changes_total",
+		Help:      "Counter that counts the number of flag changes.",
+		Subsystem: GOFFSubSystem,
+	})
+
+	// counts the number of new flag created from your configuration
+	flagCreateCounter := prom.NewCounter(prom.CounterOpts{
+		Name:      "flag_create_total",
+		Help:      "Counter that counts the number of flag created.",
+		Subsystem: GOFFSubSystem,
+	})
+
+	// counts the number of flag deleted from your configuration
+	flagDeleteCounter := prom.NewCounter(prom.CounterOpts{
+		Name:      "flag_delete_total",
+		Help:      "Counter that counts the number of flag deleted.",
+		Subsystem: GOFFSubSystem,
+	})
+
+	// counts the number of flag updated from your configuration
+	flagUpdateCounter := prom.NewCounter(prom.CounterOpts{
+		Name:      "flag_update_total",
+		Help:      "Counter that counts the number of flag updated.",
+		Subsystem: GOFFSubSystem,
+	})
+
+	// counts the number of update per flag
+	flagUpdateCounterVec := prom.NewCounterVec(prom.CounterOpts{
+		Name:      "flag_update",
+		Help:      "Counter events for number of update per flag.",
+		Subsystem: GOFFSubSystem,
+	}, []string{"flag_name"})
+
+	// counts the number of delete per flag
+	flagDeleteCounterVec := prom.NewCounterVec(prom.CounterOpts{
+		Name:      "flag_delete",
+		Help:      "Counter events for number of delete per flag.",
+		Subsystem: GOFFSubSystem,
+	}, []string{"flag_name"})
+
+	// flagCreateCounterVec counts the number of create per flag
+	flagCreateCounterVec := prom.NewCounterVec(prom.CounterOpts{
+		Name:      "flag_create",
+		Help:      "Counter events for number of create per flag.",
+		Subsystem: GOFFSubSystem,
+	}, []string{"flag_name"})
+
+	metricToRegister := []prom.Collector{
+		flagEvaluationCounter,
+		allFlagCounter,
+		collectEvalDataCounter,
+		flagChange,
+		flagCreateCounter,
+		flagDeleteCounter,
+		flagUpdateCounter,
+		flagUpdateCounterVec,
+		flagDeleteCounterVec,
+		flagCreateCounterVec,
+	}
 
 	// register all the metric in the custom registry
 	for _, metric := range metricToRegister {
@@ -45,6 +106,13 @@ func NewMetrics() (Metrics, error) {
 		flagEvaluationCounter:  *flagEvaluationCounter,
 		allFlagCounter:         allFlagCounter,
 		collectEvalDataCounter: collectEvalDataCounter,
+		flagChange:             flagChange,
+		flagCreateCounter:      flagCreateCounter,
+		flagDeleteCounter:      flagDeleteCounter,
+		flagUpdateCounter:      flagUpdateCounter,
+		flagUpdateCounterVec:   *flagUpdateCounterVec,
+		flagDeleteCounterVec:   *flagDeleteCounterVec,
+		flagCreateCounterVec:   *flagCreateCounterVec,
 		Registry:               customRegistry,
 	}, nil
 }
@@ -55,6 +123,13 @@ type Metrics struct {
 	flagEvaluationCounter  prom.CounterVec
 	allFlagCounter         prom.Counter
 	collectEvalDataCounter prom.Counter
+	flagChange             prom.Counter
+	flagCreateCounter      prom.Counter
+	flagDeleteCounter      prom.Counter
+	flagUpdateCounter      prom.Counter
+	flagUpdateCounterVec   prom.CounterVec
+	flagDeleteCounterVec   prom.CounterVec
+	flagCreateCounterVec   prom.CounterVec
 }
 
 func (m *Metrics) IncFlagEvaluation(flagName string) {
@@ -77,3 +152,34 @@ func (m *Metrics) IncCollectEvalData(numberEvents float64) {
 		m.collectEvalDataCounter.Add(numberEvents)
 	}
 }
+
+// IncFlagUpdated is incrementing the counters when a flag is updated.
+func (m *Metrics) IncFlagUpdated(flagName string) {
+	if m.flagUpdateCounterVec.MetricVec != nil {
+		m.flagUpdateCounterVec.With(prom.Labels{"flag_name": flagName}).Inc()
+		m.flagUpdateCounter.Inc()
+	}
+}
+
+// IncFlagDeleted is incrementing the counters when a flag is deleted.
+func (m *Metrics) IncFlagDeleted(flagName string) {
+	if m.flagDeleteCounterVec.MetricVec != nil {
+		m.flagDeleteCounterVec.With(prom.Labels{"flag_name": flagName}).Inc()
+		m.flagDeleteCounter.Inc()
+	}
+}
+
+// IncFlagCreated is incrementing the counters when a flag is created.
+func (m *Metrics) IncFlagCreated(flagName string) {
+	if m.flagCreateCounterVec.MetricVec != nil {
+		m.flagCreateCounterVec.With(prom.Labels{"flag_name": flagName}).Inc()
+		m.flagCreateCounter.Inc()
+	}
+}
+
+// IncFlagChange is incrementing the counters when a flag is created, updated or deleted.
+func (m *Metrics) IncFlagChange() {
+	if m.flagChange != nil {
+		m.flagChange.Inc()
+	}
+}
diff --git a/cmd/relayproxy/metric/metrics_test.go b/cmd/relayproxy/metric/metrics_test.go
@@ -37,3 +37,50 @@ func TestMetrics_IncFlagEvaluation(t *testing.T) {
 	assert.Equal(t, 2.0, testutil.ToFloat64(metricSrv.flagEvaluationCounter.WithLabelValues("test-flag")))
 	assert.Equal(t, 1.0, testutil.ToFloat64(metricSrv.flagEvaluationCounter.WithLabelValues("test-flag2")))
 }
+
+func TestMetrics_IncFlagCreated(t *testing.T) {
+	metricSrv, err := NewMetrics()
+	assert.NoError(t, err)
+
+	metricSrv.IncFlagCreated("test-flag")
+	metricSrv.IncFlagCreated("test-flag2")
+
+	assert.Equal(t, 2.0, testutil.ToFloat64(metricSrv.flagCreateCounter))
+	assert.Equal(t, 1.0, testutil.ToFloat64(metricSrv.flagCreateCounterVec.WithLabelValues("test-flag2")))
+	assert.Equal(t, 1.0, testutil.ToFloat64(metricSrv.flagCreateCounterVec.WithLabelValues("test-flag")))
+}
+
+func TestMetrics_IncFlagUpdated(t *testing.T) {
+	metricSrv, err := NewMetrics()
+	assert.NoError(t, err)
+
+	metricSrv.IncFlagUpdated("test-flag")
+	metricSrv.IncFlagUpdated("test-flag2")
+
+	assert.Equal(t, 2.0, testutil.ToFloat64(metricSrv.flagUpdateCounter))
+	assert.Equal(t, 1.0, testutil.ToFloat64(metricSrv.flagUpdateCounterVec.WithLabelValues("test-flag2")))
+	assert.Equal(t, 1.0, testutil.ToFloat64(metricSrv.flagUpdateCounterVec.WithLabelValues("test-flag")))
+}
+
+func TestMetrics_IncFlagDeleted(t *testing.T) {
+	metricSrv, err := NewMetrics()
+	assert.NoError(t, err)
+
+	metricSrv.IncFlagDeleted("test-flag")
+	metricSrv.IncFlagDeleted("test-flag2")
+
+	assert.Equal(t, 2.0, testutil.ToFloat64(metricSrv.flagDeleteCounter))
+	assert.Equal(t, 1.0, testutil.ToFloat64(metricSrv.flagDeleteCounterVec.WithLabelValues("test-flag2")))
+	assert.Equal(t, 1.0, testutil.ToFloat64(metricSrv.flagDeleteCounterVec.WithLabelValues("test-flag")))
+}
+
+func TestMetrics_IncFlagChange(t *testing.T) {
+	metricSrv, err := NewMetrics()
+	assert.NoError(t, err)
+
+	metricSrv.IncFlagChange()
+	metricSrv.IncFlagChange()
+	metricSrv.IncFlagChange()
+
+	assert.Equal(t, 3.0, testutil.ToFloat64(metricSrv.flagChange))
+}
diff --git a/cmd/relayproxy/metric/notifier_prometheus.go b/cmd/relayproxy/metric/notifier_prometheus.go
@@ -0,0 +1,34 @@
+package metric
+
+import (
+	"github.com/thomaspoignant/go-feature-flag/notifier"
+)
+
+type notifierPrometheus struct {
+	metricsService Metrics
+}
+
+func NewPrometheusNotifier(metricsService Metrics) notifier.Notifier {
+	return &notifierPrometheus{
+		metricsService: metricsService,
+	}
+}
+
+func (n *notifierPrometheus) Notify(diff notifier.DiffCache) error {
+	if !diff.HasDiff() {
+		return nil
+	}
+	n.metricsService.IncFlagChange()
+	for flagName := range diff.Deleted {
+		n.metricsService.IncFlagDeleted(flagName)
+	}
+
+	for flagName := range diff.Added {
+		n.metricsService.IncFlagCreated(flagName)
+	}
+
+	for flagName := range diff.Updated {
+		n.metricsService.IncFlagUpdated(flagName)
+	}
+	return nil
+}
diff --git a/cmd/relayproxy/metric/notifier_prometheus_test.go b/cmd/relayproxy/metric/notifier_prometheus_test.go
@@ -0,0 +1,69 @@
+package metric
+
+import (
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/thomaspoignant/go-feature-flag/internal/flag"
+	"github.com/thomaspoignant/go-feature-flag/notifier"
+	"testing"
+)
+
+func TestPrometheusNotifier_with_diff(t *testing.T) {
+	m, err := NewMetrics()
+	assert.NoError(t, err)
+
+	diff := notifier.DiffCache{
+		Deleted: map[string]flag.Flag{
+			"test-flag": &flag.InternalFlag{},
+		},
+		Updated: map[string]notifier.DiffUpdated{
+			"test-flag2": {
+				Before: &flag.InternalFlag{},
+				After:  &flag.InternalFlag{},
+			},
+		},
+		Added: map[string]flag.Flag{
+			"test-flagAdd1": &flag.InternalFlag{},
+			"test-flagAdd2": &flag.InternalFlag{},
+			"test-flagAdd3": &flag.InternalFlag{},
+		},
+	}
+
+	n := NewPrometheusNotifier(m)
+	err = n.Notify(diff)
+	assert.NoError(t, err)
+
+	assert.Equal(t, 1.0, testutil.ToFloat64(m.flagDeleteCounter))
+	assert.Equal(t, 1.0, testutil.ToFloat64(m.flagUpdateCounter))
+	assert.Equal(t, 3.0, testutil.ToFloat64(m.flagCreateCounter))
+	assert.Equal(t, 1.0, testutil.ToFloat64(m.flagCreateCounterVec.WithLabelValues("test-flagAdd1")))
+	assert.Equal(t, 1.0, testutil.ToFloat64(m.flagCreateCounterVec.WithLabelValues("test-flagAdd2")))
+	assert.Equal(t, 1.0, testutil.ToFloat64(m.flagCreateCounterVec.WithLabelValues("test-flagAdd3")))
+	assert.Equal(t, 1.0, testutil.ToFloat64(m.flagUpdateCounterVec.WithLabelValues("test-flag2")))
+	assert.Equal(t, 1.0, testutil.ToFloat64(m.flagDeleteCounterVec.WithLabelValues("test-flag")))
+	assert.Equal(t, 1.0, testutil.ToFloat64(m.flagChange))
+}
+
+func TestPrometheusNotifier_no_diff(t *testing.T) {
+	m, err := NewMetrics()
+	assert.NoError(t, err)
+
+	diff := notifier.DiffCache{
+		Deleted: map[string]flag.Flag{},
+		Updated: map[string]notifier.DiffUpdated{},
+		Added:   map[string]flag.Flag{},
+	}
+
+	n := NewPrometheusNotifier(m)
+	err = n.Notify(diff)
+	assert.NoError(t, err)
+	assert.Equal(t, 0.0, testutil.ToFloat64(m.flagDeleteCounter))
+	assert.Equal(t, 0.0, testutil.ToFloat64(m.flagUpdateCounter))
+	assert.Equal(t, 0.0, testutil.ToFloat64(m.flagCreateCounter))
+	assert.Equal(t, 0.0, testutil.ToFloat64(m.flagCreateCounterVec.WithLabelValues("test-flagAdd1")))
+	assert.Equal(t, 0.0, testutil.ToFloat64(m.flagCreateCounterVec.WithLabelValues("test-flagAdd2")))
+	assert.Equal(t, 0.0, testutil.ToFloat64(m.flagCreateCounterVec.WithLabelValues("test-flagAdd3")))
+	assert.Equal(t, 0.0, testutil.ToFloat64(m.flagUpdateCounterVec.WithLabelValues("test-flag2")))
+	assert.Equal(t, 0.0, testutil.ToFloat64(m.flagDeleteCounterVec.WithLabelValues("test-flag")))
+	assert.Equal(t, 0.0, testutil.ToFloat64(m.flagChange))
+}
diff --git a/cmd/relayproxy/service/gofeatureflag.go b/cmd/relayproxy/service/gofeatureflag.go
@@ -33,7 +33,7 @@ import (
 func NewGoFeatureFlagClient(
 	proxyConf *config.Config,
 	logger *zap.Logger,
-	proxyNotifier notifier.Notifier,
+	notifiers []notifier.Notifier,
 ) (*ffclient.GoFeatureFlag, error) {
 	var mainRetriever retriever.Retriever
 	var err error
@@ -72,7 +72,7 @@ func NewGoFeatureFlagClient(
 			return nil, err
 		}
 	}
-	notif = append(notif, proxyNotifier)
+	notif = append(notif, notifiers...)
 
 	f := ffclient.Config{
 		PollingInterval:         time.Duration(proxyConf.PollingInterval) * time.Millisecond,

diff --git a/...relayproxy/service/notifier_relayproxy.go → cmd/relayproxy/service/notifier_websocket.go b/...relayproxy/service/notifier_relayproxy.go → cmd/relayproxy/service/notifier_websocket.go
@@ -4,17 +4,17 @@ import (
 	"github.com/thomaspoignant/go-feature-flag/notifier"
 )
 
-type notifierRelayProxy struct {
+type notifierWebsocket struct {
 	websocketService WebsocketService
 }
 
-func NewNotifierRelayProxy(websocketService WebsocketService) notifier.Notifier {
-	return &notifierRelayProxy{
+func NewNotifierWebsocket(websocketService WebsocketService) notifier.Notifier {
+	return &notifierWebsocket{
 		websocketService: websocketService,
 	}
 }
 
-func (n *notifierRelayProxy) Notify(diff notifier.DiffCache) error {
+func (n *notifierWebsocket) Notify(diff notifier.DiffCache) error {
 	n.websocketService.BroadcastFlagChanges(diff)
 	return nil
 }
diff --git a/...proxy/service/notifier_relayproxy_test.go → ...yproxy/service/notifier_websocket_test.go b/...proxy/service/notifier_relayproxy_test.go → ...yproxy/service/notifier_websocket_test.go
@@ -36,8 +36,8 @@ func TestNotify(t *testing.T) {
 	// Create a mock WebsocketService
 	mockService := &mockWebsocketService{}
 
-	// Create the notifierRelayProxy instance with the mock service
-	n := service.NewNotifierRelayProxy(mockService)
+	// Create the notifierWebsocket instance with the mock service
+	n := service.NewNotifierWebsocket(mockService)
 
 	// Prepare the input data
 	diff := notifier.DiffCache{