Skip to content

Commit

Permalink
New flag to disable detailed metrics for path
Browse files Browse the repository at this point in the history
Flag detailed-metrics added to provide a way to disable exposing all accessed paths to the metrics and  prevent potential metrics cardinality explosion

Signed-off-by: Alexey Kuzbekov <[email protected]>
  • Loading branch information
Kuzbekov committed Aug 24, 2024
1 parent 0871ad7 commit ab58c81
Show file tree
Hide file tree
Showing 5 changed files with 30 additions and 15 deletions.
2 changes: 1 addition & 1 deletion internal/controller/receiver_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ func TestReceiverReconciler_EventHandler(t *testing.T) {

// Use the client from the manager as the server handler needs to list objects from the cache
// which the "live" k8s client does not have access to.
receiverServer := server.NewReceiverServer("127.0.0.1:56788", logf.Log, testEnv.GetClient())
receiverServer := server.NewReceiverServer("127.0.0.1:56788", logf.Log, testEnv.GetClient(), true)
receiverMdlw := middleware.New(middleware.Config{
Recorder: prommetrics.NewRecorder(prommetrics.Config{
Prefix: "gotk_receiver",
Expand Down
13 changes: 10 additions & 3 deletions internal/server/event_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,17 +52,19 @@ type EventServer struct {
logger logr.Logger
kubeClient client.Client
noCrossNamespaceRefs bool
detailedMetrics bool
kuberecorder.EventRecorder
}

// NewEventServer returns an HTTP server that handles events
func NewEventServer(port string, logger logr.Logger, kubeClient client.Client, eventRecorder kuberecorder.EventRecorder, noCrossNamespaceRefs bool) *EventServer {
func NewEventServer(port string, logger logr.Logger, kubeClient client.Client, eventRecorder kuberecorder.EventRecorder, noCrossNamespaceRefs bool, detailedMetrics bool) *EventServer {
return &EventServer{
port: port,
logger: logger.WithName("event-server"),
kubeClient: kubeClient,
EventRecorder: eventRecorder,
noCrossNamespaceRefs: noCrossNamespaceRefs,
detailedMetrics: detailedMetrics,
}
}

Expand All @@ -82,8 +84,13 @@ func (s *EventServer) ListenAndServe(stopCh <-chan struct{}, mdlw middleware.Mid
handler = middleware(handler)
}
mux := http.NewServeMux()
mux.Handle("/", handler)
h := std.Handler("", mdlw, mux)
path := "/"
mux.Handle(path, handler)
middlewarePath := ""
if !s.detailedMetrics {
middlewarePath = path
}
h := std.Handler(middlewarePath, mdlw, mux)
srv := &http.Server{
Addr: s.port,
Handler: h,
Expand Down
2 changes: 1 addition & 1 deletion internal/server/event_server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ func TestEventServer(t *testing.T) {
t.Fatalf("failed to create memory storage")
}
eventServer := NewEventServer("127.0.0.1:"+eventServerPort,
log.Log, kclient, record.NewFakeRecorder(32), true)
log.Log, kclient, record.NewFakeRecorder(32), true, true)
stopCh := make(chan struct{})
go eventServer.ListenAndServe(stopCh, eventMdlw, store)
defer close(stopCh)
Expand Down
22 changes: 14 additions & 8 deletions internal/server/receiver_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,25 +32,31 @@ import (

// ReceiverServer handles webhook POST requests
type ReceiverServer struct {
port string
logger logr.Logger
kubeClient client.Client
port string
logger logr.Logger
kubeClient client.Client
detailedMetrics bool
}

// NewReceiverServer returns an HTTP server that handles webhooks
func NewReceiverServer(port string, logger logr.Logger, kubeClient client.Client) *ReceiverServer {
func NewReceiverServer(port string, logger logr.Logger, kubeClient client.Client, detailedMetrics bool) *ReceiverServer {
return &ReceiverServer{
port: port,
logger: logger.WithName("receiver-server"),
kubeClient: kubeClient,
port: port,
logger: logger.WithName("receiver-server"),
kubeClient: kubeClient,
detailedMetrics: detailedMetrics,
}
}

// ListenAndServe starts the HTTP server on the specified port
func (s *ReceiverServer) ListenAndServe(stopCh <-chan struct{}, mdlw middleware.Middleware) {
mux := http.NewServeMux()
mux.Handle(apiv1.ReceiverWebhookPath, http.HandlerFunc(s.handlePayload()))
h := std.Handler("", mdlw, mux)
middlewarePath := ""
if !s.detailedMetrics {
middlewarePath = apiv1.ReceiverWebhookPath
}
h := std.Handler(middlewarePath, mdlw, mux)
srv := &http.Server{
Addr: s.port,
Handler: h,
Expand Down
6 changes: 4 additions & 2 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ func main() {
aclOptions acl.Options
rateLimiterOptions helper.RateLimiterOptions
featureGates feathelper.FeatureGates
detailedMetrics bool
)

flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
Expand All @@ -97,6 +98,7 @@ func main() {
flag.BoolVar(&watchAllNamespaces, "watch-all-namespaces", true,
"Watch for custom resources in all namespaces, if set to false it will only watch the runtime namespace.")
flag.DurationVar(&rateLimitInterval, "rate-limit-interval", 5*time.Minute, "Interval in which rate limit has effect.")
flag.BoolVar(&detailedMetrics, "detailed-metrics", true, "Count metrics for every requested path")

clientOptions.BindFlags(flag.CommandLine)
logOptions.BindFlags(flag.CommandLine)
Expand Down Expand Up @@ -220,11 +222,11 @@ func main() {
Registry: crtlmetrics.Registry,
}),
})
eventServer := server.NewEventServer(eventsAddr, ctrl.Log, mgr.GetClient(), mgr.GetEventRecorderFor(controllerName), aclOptions.NoCrossNamespaceRefs)
eventServer := server.NewEventServer(eventsAddr, ctrl.Log, mgr.GetClient(), mgr.GetEventRecorderFor(controllerName), aclOptions.NoCrossNamespaceRefs, detailedMetrics)
go eventServer.ListenAndServe(ctx.Done(), eventMdlw, store)

setupLog.Info("starting webhook receiver server", "addr", receiverAddr)
receiverServer := server.NewReceiverServer(receiverAddr, ctrl.Log, mgr.GetClient())
receiverServer := server.NewReceiverServer(receiverAddr, ctrl.Log, mgr.GetClient(), detailedMetrics)
receiverMdlw := middleware.New(middleware.Config{
Recorder: prommetrics.NewRecorder(prommetrics.Config{
Prefix: "gotk_receiver",
Expand Down

0 comments on commit ab58c81

Please sign in to comment.