From 401c0f9b29536baaf6eb6cb02be068b22613f2fe Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Fri, 28 Jun 2024 21:24:28 -0700 Subject: [PATCH] chore: Add prometheus metrics for AWS client calls (#6421) --- go.mod | 1 + go.sum | 2 ++ pkg/apis/v1/doc.go | 3 ++- pkg/apis/v1/labels.go | 3 ++- pkg/apis/v1beta1/doc.go | 3 ++- pkg/apis/v1beta1/labels.go | 3 ++- pkg/operator/operator.go | 9 +++++++-- pkg/webhooks/webhooks.go | 3 ++- 8 files changed, 20 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 9bbc0b511278..0881c4eae2ac 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/awslabs/operatorpkg v0.0.0-20240605172541-88cf99023fa4 github.com/go-logr/zapr v1.3.0 github.com/imdario/mergo v0.3.16 + github.com/jonathan-innis/aws-sdk-go-prometheus v0.1.0 github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/onsi/ginkgo/v2 v2.19.0 github.com/onsi/gomega v1.33.1 diff --git a/go.sum b/go.sum index 176060a5901b..dd1992975f3d 100644 --- a/go.sum +++ b/go.sum @@ -222,6 +222,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jonathan-innis/aws-sdk-go-prometheus v0.1.0 h1:6eJFFxJ+2hbSEshwyLCLGUAf4/tXq8vpajf5QxcL8ew= +github.com/jonathan-innis/aws-sdk-go-prometheus v0.1.0/go.mod h1:DDom1Ae898wsni+arqgipv+JgtDtVDmbJB5YLOQz25s= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= diff --git a/pkg/apis/v1/doc.go b/pkg/apis/v1/doc.go index 650a652d4084..9d40f9d57285 100644 --- a/pkg/apis/v1/doc.go +++ b/pkg/apis/v1/doc.go @@ -19,10 +19,11 @@ limitations under the License. package v1 // doc.go is discovered by codegen import ( - "github.com/aws/karpenter-provider-aws/pkg/apis" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/scheme" + + "github.com/aws/karpenter-provider-aws/pkg/apis" ) func init() { diff --git a/pkg/apis/v1/labels.go b/pkg/apis/v1/labels.go index f055645bd480..ea3403ed48e5 100644 --- a/pkg/apis/v1/labels.go +++ b/pkg/apis/v1/labels.go @@ -18,11 +18,12 @@ import ( "fmt" "regexp" - "github.com/aws/karpenter-provider-aws/pkg/apis" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" coreapis "sigs.k8s.io/karpenter/pkg/apis" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + + "github.com/aws/karpenter-provider-aws/pkg/apis" ) func init() { diff --git a/pkg/apis/v1beta1/doc.go b/pkg/apis/v1beta1/doc.go index 1bd056a104ae..361d5f55f217 100644 --- a/pkg/apis/v1beta1/doc.go +++ b/pkg/apis/v1beta1/doc.go @@ -19,10 +19,11 @@ limitations under the License. package v1beta1 // doc.go is discovered by codegen import ( - "github.com/aws/karpenter-provider-aws/pkg/apis" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/scheme" + + "github.com/aws/karpenter-provider-aws/pkg/apis" ) func init() { diff --git a/pkg/apis/v1beta1/labels.go b/pkg/apis/v1beta1/labels.go index 91b93a3019e3..4b51945b2306 100644 --- a/pkg/apis/v1beta1/labels.go +++ b/pkg/apis/v1beta1/labels.go @@ -18,11 +18,12 @@ import ( "fmt" "regexp" - "github.com/aws/karpenter-provider-aws/pkg/apis" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" coreapis "sigs.k8s.io/karpenter/pkg/apis" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + + "github.com/aws/karpenter-provider-aws/pkg/apis" ) func init() { diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 6719c31d3593..b6a4ddd45508 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -37,6 +37,7 @@ import ( "github.com/aws/aws-sdk-go/service/eks/eksiface" "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/ssm" + prometheusv1 "github.com/jonathan-innis/aws-sdk-go-prometheus/v1" "github.com/patrickmn/go-cache" "github.com/samber/lo" corev1 "k8s.io/api/core/v1" @@ -45,6 +46,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/transport" "sigs.k8s.io/controller-runtime/pkg/log" + crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/operator" @@ -95,12 +97,15 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont func(provider *stscreds.AssumeRoleProvider) { SetDurationAndExpiry(ctx, provider) }) } - sess := WithUserAgent(session.Must(session.NewSession( + // prometheusv1.WithPrometheusMetrics is used until the upstream aws-sdk-go or aws-sdk-go-v2 supports + // Prometheus metrics for client-side metrics out-of-the-box + // See: https://github.com/aws/aws-sdk-go-v2/issues/1744 + sess := prometheusv1.WithPrometheusMetrics(WithUserAgent(session.Must(session.NewSession( request.WithRetryer( config, awsclient.DefaultRetryer{NumMaxRetries: awsclient.DefaultRetryerMaxNumRetries}, ), - ))) + ))), crmetrics.Registry) if *sess.Config.Region == "" { log.FromContext(ctx).V(1).Info("retrieving region from IMDS") diff --git a/pkg/webhooks/webhooks.go b/pkg/webhooks/webhooks.go index 9ede88fdd480..feb627d89de1 100644 --- a/pkg/webhooks/webhooks.go +++ b/pkg/webhooks/webhooks.go @@ -25,8 +25,9 @@ import ( "knative.dev/pkg/webhook/resourcesemantics/defaulting" "knative.dev/pkg/webhook/resourcesemantics/validation" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/awslabs/operatorpkg/object" + + "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" ) func NewWebhooks() []knativeinjection.ControllerConstructor {