add downstream timeout propagation logic with metrics

pkg/auth/authenticator.go

@@ -17,6 +17,10 @@ import (
 	v1 "k8s.io/api/core/v1"
 )
 
+// downstreamDeadlineOffset sets an offset to downstream deadline inorder
+// to save a little time to update metrics and answer downstream request
+const downstreamDeadlineOffset = 50 * time.Microsecond
+
 // Authenticator can generate cache from Kubernetes API server
 // and it implements envoy.CheckRequest interface
 type Authenticator struct {
@@ -364,7 +368,7 @@ func (a *Authenticator) Check(ctx context.Context, request *Request) (*Response,
 		ok, reason, extraHeaders = a.TestAccess(request, wsvcCacheEntry)
 		if ok && hasUpstreamAuth(wsvcCacheEntry) {
 			request.Context[HasUpstreamAuth] = "true"
-			ok, reason = a.checkServiceUpstreamAuth(wsvcCacheEntry, request, &extraHeaders)
+			ok, reason = a.checkServiceUpstreamAuth(wsvcCacheEntry, request, &extraHeaders, ctx)
 		}
 	}
 
@@ -442,8 +446,9 @@ func CheckDomain(domain string, domainAllowedList []string) (bool, error) {
 
 // checkServiceUpstreamAuth function is designed to validate the request through
 // the upstream authentication for a given webservice
-func (a *Authenticator) checkServiceUpstreamAuth(service ServicesCacheEntry, request *Request, extraHeaders *ExtraHeaders) (bool, CerberusReason) {
-	serviceUpstreamAuthCalls.Inc()
+func (a *Authenticator) checkServiceUpstreamAuth(service ServicesCacheEntry, request *Request, extraHeaders *ExtraHeaders, ctx context.Context) (bool, CerberusReason) {
+	downstreamDeadline, hasDownstreamDeadline := ctx.Deadline()
+	serviceUpstreamAuthCalls.With(AddWithDownstreamDeadline(nil, hasDownstreamDeadline)).Inc()
 
 	if service.Spec.UpstreamHttpAuth.ReadTokenFrom == "" {
 		return false, CerberusReasonSourceAuthTokenEmpty
@@ -469,14 +474,20 @@ func (a *Authenticator) checkServiceUpstreamAuth(service ServicesCacheEntry, req
 	}
 
 	a.httpClient.Timeout = time.Duration(service.Spec.UpstreamHttpAuth.Timeout) * time.Millisecond
+	if hasDownstreamDeadline {
+		if time.Until(downstreamDeadline)-downstreamDeadlineOffset < a.httpClient.Timeout {
+			a.httpClient.Timeout = time.Until(downstreamDeadline) - downstreamDeadlineOffset
+		}
+	}
+
 	reqStart := time.Now()
 	resp, err := a.httpClient.Do(req)
 	reqDuration := time.Since(reqStart)
 	if err != nil {
 		return false, CerberusReasonUpstreamAuthFailed
 	}
 
-	labels := AddStatusLabel(nil, resp.StatusCode)
+	labels := AddWithDownstreamDeadline(AddStatusLabel(nil, resp.StatusCode), hasDownstreamDeadline)
 	upstreamAuthRequestDuration.With(labels).Observe(reqDuration.Seconds())
 
 	if resp.StatusCode != http.StatusOK {

pkg/auth/metrics.go

@@ -1,16 +1,18 @@
 package auth
 
 import (
+	"strconv"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
-	"strconv"
 )
 
 const (
-	CerberusReasonLabel      = "cerberus_reason"
-	CheckRequestVersionLabel = "check_request_version"
-	HasUpstreamAuth		     = "upstream_auth_enabled"
-	ObjectKindLabel          = "kind"
+	CerberusReasonLabel         = "cerberus_reason"
+	CheckRequestVersionLabel    = "check_request_version"
+	HasUpstreamAuth             = "upstream_auth_enabled"
+	ObjectKindLabel             = "kind"
+	WithDownstreamDeadlineLabel = "with_downstream_deadline"
 
 	MetricsKindSecret                  = "secret"
 	MetricsKindWebservice              = "webservice"
@@ -23,7 +25,7 @@ const (
 )
 
 var (
-	DurationBuckets      = []float64{0.000005, 0.00001, 0.000015, 0.00003, 0.00004, 0.00005, 0.000075, 0.0001, 0.000125, 0.00015, 0.000175, 0.0002,  0.00025, .0005, .001, .002, .003, .004, .005, .006, .007, .008, .009, .01, .02, .05, .1, 1, 2.5, 5}
+	DurationBuckets      = []float64{0.000005, 0.00001, 0.000015, 0.00003, 0.00004, 0.00005, 0.000075, 0.0001, 0.000125, 0.00015, 0.000175, 0.0002, 0.00025, .0005, .001, .002, .003, .004, .005, .006, .007, .008, .009, .01, .02, .05, .1, 1, 2.5, 5}
 	SmallDurationBuckets = []float64{0.0000001, 0.000001, 0.0000025, 0.000005, 0.00001, 0.000025, 0.00005, 0.0001, 0.001, 0.01, 0.05, 0.1}
 
 	reqCount = prometheus.NewCounterVec(
@@ -104,19 +106,21 @@ var (
 		[]string{ObjectKindLabel},
 	)
 
-	serviceUpstreamAuthCalls = prometheus.NewCounter(
+	serviceUpstreamAuthCalls = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "upstream_auth_calls_total",
 			Help: "The total number of checkServiceUpstreamAuth function calls",
-		})
+		},
+		[]string{WithDownstreamDeadlineLabel},
+	)
 
 	upstreamAuthRequestDuration = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Name:    "upstream_auth_request_duration_seconds",
 			Help:    "Duration of the UpstreamAuth Requests in seconds",
 			Buckets: DurationBuckets,
 		},
-		[]string{StatusCode},
+		[]string{StatusCode, WithDownstreamDeadlineLabel},
 	)
 )
 
@@ -168,3 +172,15 @@ func AddUpstreamAuthLabel(labels prometheus.Labels, hasUpstreamAuth string) prom
 	labels[HasUpstreamAuth] = hasUpstreamAuth
 	return labels
 }
+
+func AddWithDownstreamDeadline(labels prometheus.Labels, hasDeadline bool) prometheus.Labels {
+	if labels == nil {
+		labels = prometheus.Labels{}
+	}
+	if hasDeadline {
+		labels[WithDownstreamDeadlineLabel] = "true"
+	} else {
+		labels[WithDownstreamDeadlineLabel] = "false"
+	}
+	return labels
+}