chore: Add prometheus metrics for AWS client calls (aws#6421)

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/awslabs/operatorpkg v0.0.0-20240605172541-88cf99023fa4
 	github.com/go-logr/zapr v1.3.0
 	github.com/imdario/mergo v0.3.16
+	github.com/jonathan-innis/aws-sdk-go-prometheus v0.1.0
 	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/onsi/ginkgo/v2 v2.19.0
 	github.com/onsi/gomega v1.33.1

go.sum

@@ -222,6 +222,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/jonathan-innis/aws-sdk-go-prometheus v0.1.0 h1:6eJFFxJ+2hbSEshwyLCLGUAf4/tXq8vpajf5QxcL8ew=
+github.com/jonathan-innis/aws-sdk-go-prometheus v0.1.0/go.mod h1:DDom1Ae898wsni+arqgipv+JgtDtVDmbJB5YLOQz25s=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=

pkg/apis/v1/doc.go

@@ -19,10 +19,11 @@ limitations under the License.
 package v1 // doc.go is discovered by codegen
 
 import (
-	"github.com/aws/karpenter-provider-aws/pkg/apis"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes/scheme"
+
+	"github.com/aws/karpenter-provider-aws/pkg/apis"
 )
 
 func init() {

pkg/apis/v1/labels.go

@@ -18,11 +18,12 @@ import (
 	"fmt"
 	"regexp"
 
-	"github.com/aws/karpenter-provider-aws/pkg/apis"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	coreapis "sigs.k8s.io/karpenter/pkg/apis"
 	"sigs.k8s.io/karpenter/pkg/apis/v1beta1"
+
+	"github.com/aws/karpenter-provider-aws/pkg/apis"
 )
 
 func init() {

pkg/apis/v1beta1/doc.go

@@ -19,10 +19,11 @@ limitations under the License.
 package v1beta1 // doc.go is discovered by codegen
 
 import (
-	"github.com/aws/karpenter-provider-aws/pkg/apis"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes/scheme"
+
+	"github.com/aws/karpenter-provider-aws/pkg/apis"
 )
 
 func init() {

pkg/apis/v1beta1/labels.go

@@ -18,11 +18,12 @@ import (
 	"fmt"
 	"regexp"
 
-	"github.com/aws/karpenter-provider-aws/pkg/apis"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	coreapis "sigs.k8s.io/karpenter/pkg/apis"
 	"sigs.k8s.io/karpenter/pkg/apis/v1beta1"
+
+	"github.com/aws/karpenter-provider-aws/pkg/apis"
 )
 
 func init() {

pkg/operator/operator.go

@@ -37,6 +37,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/eks/eksiface"
 	"github.com/aws/aws-sdk-go/service/iam"
 	"github.com/aws/aws-sdk-go/service/ssm"
+	prometheusv1 "github.com/jonathan-innis/aws-sdk-go-prometheus/v1"
 	"github.com/patrickmn/go-cache"
 	"github.com/samber/lo"
 	corev1 "k8s.io/api/core/v1"
@@ -45,6 +46,7 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/transport"
 	"sigs.k8s.io/controller-runtime/pkg/log"
+	crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1"
 	"sigs.k8s.io/karpenter/pkg/operator"
@@ -95,12 +97,15 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont
 			func(provider *stscreds.AssumeRoleProvider) { SetDurationAndExpiry(ctx, provider) })
 	}
 
-	sess := WithUserAgent(session.Must(session.NewSession(
+	// prometheusv1.WithPrometheusMetrics is used until the upstream aws-sdk-go or aws-sdk-go-v2 supports
+	// Prometheus metrics for client-side metrics out-of-the-box
+	// See: https://github.com/aws/aws-sdk-go-v2/issues/1744
+	sess := prometheusv1.WithPrometheusMetrics(WithUserAgent(session.Must(session.NewSession(
 		request.WithRetryer(
 			config,
 			awsclient.DefaultRetryer{NumMaxRetries: awsclient.DefaultRetryerMaxNumRetries},
 		),
-	)))
+	))), crmetrics.Registry)
 
 	if *sess.Config.Region == "" {
 		log.FromContext(ctx).V(1).Info("retrieving region from IMDS")

pkg/webhooks/webhooks.go

@@ -25,8 +25,9 @@ import (
 	"knative.dev/pkg/webhook/resourcesemantics/defaulting"
 	"knative.dev/pkg/webhook/resourcesemantics/validation"
 
-	"github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
 	"github.com/awslabs/operatorpkg/object"
+
+	"github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
 )
 
 func NewWebhooks() []knativeinjection.ControllerConstructor {