Add operator status condition metrics

aws · Jul 9, 2024 · 86a050d · 86a050d
1 parent 6a01acc
commit 86a050d
Show file tree

Hide file tree

Showing 31 changed files with 79 additions and 37 deletions.
diff --git a/cmd/controller/main.go b/cmd/controller/main.go
@@ -44,6 +44,7 @@ func main() {
 
 	op.
 		WithControllers(ctx, corecontrollers.NewControllers(
+			op.Manager,
 			op.Clock,
 			op.GetClient(),
 			op.EventRecorder,
@@ -52,6 +53,7 @@ func main() {
 		WithWebhooks(ctx, corewebhooks.NewWebhooks()...).
 		WithControllers(ctx, controllers.NewControllers(
 			ctx,
+			op.Manager,
 			op.Session,
 			op.Clock,
 			op.GetClient(),

diff --git a/hack/docgen.sh b/hack/docgen.sh
@@ -13,7 +13,7 @@ AWS_SDK_GO_PROMETHEUS_DIR=$(go list -m -f '{{ .Dir }}' github.com/jonathan-innis
 OPERATORPKG_DIR=$(go list -m -f '{{ .Dir }}' github.com/awslabs/operatorpkg)
 
 compatibilitymatrix
-go run hack/docs/metrics_gen_docs.go pkg/ "${KARPENTER_CORE_DIR}/pkg" "${CONTROLLER_RUNTIME_DIR}/pkg" "${AWS_SDK_GO_PROMETHEUS_DIR}" "${OPERATORPKG_DIR}/metrics" website/content/en/preview/reference/metrics.md
+go run hack/docs/metrics_gen_docs.go pkg/ "${KARPENTER_CORE_DIR}/pkg" "${CONTROLLER_RUNTIME_DIR}/pkg" "${AWS_SDK_GO_PROMETHEUS_DIR}" "${OPERATORPKG_DIR}" website/content/en/preview/reference/metrics.md
 go run hack/docs/instancetypes_gen_docs.go website/content/en/preview/reference/instance-types.md
 go run hack/docs/configuration_gen_docs.go website/content/en/preview/reference/settings.md
 cd charts/karpenter && helm-docs
diff --git a/hack/docs/metrics_gen_docs.go b/hack/docs/metrics_gen_docs.go
@@ -176,15 +176,16 @@ func bySubsystem(metrics []metricInfo) func(i int, j int) bool {
 	// Higher ordering comes first. If a value isn't designated here then the subsystem will be given a default of 0.
 	// Metrics without a subsystem come first since there is no designation for the bucket they fall under
 	subSystemSortOrder := map[string]int{
-		"":                100,
-		"nodepool":        10,
-		"nodeclaims":      9,
-		"nodes":           8,
-		"pods":            7,
-		"workqueue":       -1,
-		"client_go":       -1,
-		"aws_sdk_go":      -1,
-		"leader_election": -2,
+		"":                 100,
+		"nodepool":         10,
+		"nodeclaims":       9,
+		"nodes":            8,
+		"pods":             7,
+		"status_condition": -1,
+		"workqueue":        -1,
+		"client_go":        -1,
+		"aws_sdk_go":       -1,
+		"leader_election":  -2,
 	}
 
 	return func(i, j int) bool {
@@ -314,6 +315,8 @@ func getIdentMapping(identName string) (string, error) {
 		"metrics.Namespace": metrics.Namespace,
 		"Namespace":         metrics.Namespace,
 
+		"MetricNamespace":            "operator",
+		"MetricSubsystem":            "status_condition",
 		"WorkQueueSubsystem":         "workqueue",
 		"DepthKey":                   "depth",
 		"AddsKey":                    "adds_total",

diff --git a/pkg/apis/v1/ec2nodeclass_conversion.go b/pkg/apis/v1/ec2nodeclass_conversion.go
@@ -17,9 +17,10 @@ package v1
 import (
 	"context"
 
-	"github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
 	"github.com/samber/lo"
 	"knative.dev/pkg/apis"
+
+	"github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
 )
 
 func (in *EC2NodeClass) ConvertTo(ctx context.Context, to apis.Convertible) error {

diff --git a/pkg/apis/v1/suite_test.go b/pkg/apis/v1/suite_test.go
@@ -16,9 +16,10 @@ package v1_test
 
 import (
 	"context"
-	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"testing"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	. "knative.dev/pkg/logging/testing"

diff --git a/pkg/apis/v1beta1/suite_test.go b/pkg/apis/v1beta1/suite_test.go
@@ -16,9 +16,10 @@ package v1beta1_test
 
 import (
 	"context"
-	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"testing"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 

diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go
@@ -18,11 +18,12 @@ import (
 	"context"
 	"fmt"
 	"net"
-	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"strings"
 	"testing"
 	"time"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	"github.com/awslabs/operatorpkg/object"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"

diff --git a/pkg/controllers/controllers.go b/pkg/controllers/controllers.go
@@ -18,8 +18,11 @@ import (
 	"context"
 
 	"github.com/awslabs/operatorpkg/controller"
+	"github.com/awslabs/operatorpkg/status"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/karpenter/pkg/cloudprovider"
 
+	"github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
 	nodeclasshash "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/hash"
 	nodeclassstatus "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/status"
 	nodeclasstermination "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/termination"
@@ -50,7 +53,7 @@ import (
 	"github.com/aws/karpenter-provider-aws/pkg/providers/subnet"
 )
 
-func NewControllers(ctx context.Context, sess *session.Session, clk clock.Clock, kubeClient client.Client, recorder events.Recorder,
+func NewControllers(ctx context.Context, mgr manager.Manager, sess *session.Session, clk clock.Clock, kubeClient client.Client, recorder events.Recorder,
 	unavailableOfferings *cache.UnavailableOfferings, cloudProvider cloudprovider.CloudProvider, subnetProvider subnet.Provider,
 	securityGroupProvider securitygroup.Provider, instanceProfileProvider instanceprofile.Provider, instanceProvider instance.Provider,
 	pricingProvider pricing.Provider, amiProvider amifamily.Provider, launchTemplateProvider launchtemplate.Provider, instanceTypeProvider instancetype.Provider) []controller.Controller {
@@ -63,6 +66,7 @@ func NewControllers(ctx context.Context, sess *session.Session, clk clock.Clock,
 		nodeclaimtagging.NewController(kubeClient, instanceProvider),
 		controllerspricing.NewController(pricingProvider),
 		controllersinstancetype.NewController(instanceTypeProvider),
+		status.NewController[*v1beta1.EC2NodeClass](kubeClient, mgr.GetEventRecorderFor("karpenter")),
 	}
 	if options.FromContext(ctx).InterruptionQueue != "" {
 		sqsapi := servicesqs.New(sess)

diff --git a/pkg/controllers/interruption/suite_test.go b/pkg/controllers/interruption/suite_test.go
@@ -18,10 +18,11 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"testing"
 	"time"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	servicesqs "github.com/aws/aws-sdk-go/service/sqs"

diff --git a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go
@@ -17,11 +17,12 @@ package garbagecollection_test
 import (
 	"context"
 	"fmt"
-	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"sync"
 	"testing"
 	"time"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/awslabs/operatorpkg/object"

diff --git a/pkg/controllers/nodeclaim/tagging/suite_test.go b/pkg/controllers/nodeclaim/tagging/suite_test.go
@@ -17,9 +17,10 @@ package tagging_test
 import (
 	"context"
 	"fmt"
-	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"testing"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/samber/lo"

diff --git a/pkg/controllers/nodeclass/hash/suite_test.go b/pkg/controllers/nodeclass/hash/suite_test.go
@@ -16,9 +16,10 @@ package hash_test
 
 import (
 	"context"
-	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"testing"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/awslabs/operatorpkg/object"
 	"github.com/imdario/mergo"

diff --git a/pkg/controllers/nodeclass/status/ami_test.go b/pkg/controllers/nodeclass/status/ami_test.go
@@ -309,7 +309,7 @@ var _ = Describe("NodeClass AMI Status Controller", func() {
 	It("should get error when resolving AMIs and have status condition set to false", func() {
 		awsEnv.EC2API.NextError.Set(fmt.Errorf("unable to resolve AMI"))
 		ExpectApplied(ctx, env.Client, nodeClass)
-		ExpectObjectReconcileFailed(ctx, env.Client, statusController, nodeClass)
+		_ = ExpectObjectReconcileFailed(ctx, env.Client, statusController, nodeClass)
 		nodeClass = ExpectExists(ctx, env.Client, nodeClass)
 		Expect(nodeClass.StatusConditions().IsTrue(v1beta1.ConditionTypeAMIsReady)).To(BeFalse())
 	})

diff --git a/pkg/controllers/nodeclass/status/instanceprofile_test.go b/pkg/controllers/nodeclass/status/instanceprofile_test.go
@@ -17,9 +17,10 @@ package status_test
 import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/iam"
-	"github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
 	"github.com/samber/lo"
 
+	"github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
+
 	"github.com/aws/karpenter-provider-aws/pkg/fake"
 	"github.com/aws/karpenter-provider-aws/pkg/operator/options"
 

diff --git a/pkg/controllers/nodeclass/status/subnet_test.go b/pkg/controllers/nodeclass/status/subnet_test.go
@@ -17,6 +17,7 @@ package status_test
 import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+
 	"github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
 	"github.com/aws/karpenter-provider-aws/pkg/test"
 

diff --git a/pkg/controllers/nodeclass/status/suite_test.go b/pkg/controllers/nodeclass/status/suite_test.go
@@ -16,9 +16,10 @@ package status_test
 
 import (
 	"context"
-	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"testing"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	coreoptions "sigs.k8s.io/karpenter/pkg/operator/options"
 	coretest "sigs.k8s.io/karpenter/pkg/test"
 

diff --git a/pkg/controllers/nodeclass/termination/suite_test.go b/pkg/controllers/nodeclass/termination/suite_test.go
@@ -17,10 +17,11 @@ package termination_test
 import (
 	"context"
 	"fmt"
-	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"testing"
 	"time"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/iam"

diff --git a/pkg/controllers/providers/instancetype/suite_test.go b/pkg/controllers/providers/instancetype/suite_test.go
@@ -16,9 +16,10 @@ package instancetype_test
 
 import (
 	"context"
-	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"testing"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	v1 "k8s.io/api/core/v1"
 	corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1"
 	coreoptions "sigs.k8s.io/karpenter/pkg/operator/options"

diff --git a/pkg/controllers/providers/pricing/suite_test.go b/pkg/controllers/providers/pricing/suite_test.go
@@ -17,10 +17,11 @@ package pricing_test
 import (
 	"context"
 	"fmt"
-	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"testing"
 	"time"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	awspricing "github.com/aws/aws-sdk-go/service/pricing"

diff --git a/pkg/fake/cloudprovider.go b/pkg/fake/cloudprovider.go
@@ -16,14 +16,16 @@ package fake
 
 import (
 	"context"
+
 	"github.com/awslabs/operatorpkg/status"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	providerv1beta1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
 	"sigs.k8s.io/karpenter/pkg/apis/v1beta1"
 	corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider"
 	"sigs.k8s.io/karpenter/pkg/test"
+
+	providerv1beta1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
 )
 
 const (

diff --git a/pkg/operator/suite_test.go b/pkg/operator/suite_test.go
@@ -17,9 +17,10 @@ package operator_test
 import (
 	"context"
 	"errors"
-	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"testing"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	"github.com/aws/aws-sdk-go/service/eks"
 	"github.com/samber/lo"
 

diff --git a/pkg/providers/amifamily/suite_test.go b/pkg/providers/amifamily/suite_test.go
@@ -17,12 +17,13 @@ package amifamily_test
 import (
 	"context"
 	"fmt"
-	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"sort"
 	"sync"
 	"testing"
 	"time"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 

diff --git a/pkg/providers/instance/suite_test.go b/pkg/providers/instance/suite_test.go
@@ -17,10 +17,11 @@ package instance_test
 import (
 	"context"
 	"fmt"
-	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"testing"
 	"time"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/awslabs/operatorpkg/object"

diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go
@@ -20,13 +20,14 @@ import (
 	"math"
 	"net"
 	"reflect"
-	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"sort"
 	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/awslabs/operatorpkg/status"

diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go
@@ -21,12 +21,13 @@ import (
 	"fmt"
 	"net"
 	"os"
-	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/ec2"

diff --git a/pkg/providers/securitygroup/suite_test.go b/pkg/providers/securitygroup/suite_test.go
@@ -16,11 +16,12 @@ package securitygroup_test
 
 import (
 	"context"
-	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"sort"
 	"sync"
 	"testing"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/samber/lo"

diff --git a/pkg/providers/subnet/suite_test.go b/pkg/providers/subnet/suite_test.go
@@ -16,11 +16,12 @@ package subnet_test
 
 import (
 	"context"
-	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"sort"
 	"sync"
 	"testing"
 
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/samber/lo"

diff --git a/pkg/webhooks/webhooks.go b/pkg/webhooks/webhooks.go
@@ -26,9 +26,10 @@ import (
 	"knative.dev/pkg/webhook/resourcesemantics/defaulting"
 	"knative.dev/pkg/webhook/resourcesemantics/validation"
 
+	"github.com/awslabs/operatorpkg/object"
+
 	v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
 	"github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
-	"github.com/awslabs/operatorpkg/object"
 )
 
 var (