chore: Update Karpenter to use v1 APIs #1389

cmd/controller/main.go

@@ -44,6 +44,7 @@ func main() {
 
 	op.
 		WithControllers(ctx, corecontrollers.NewControllers(
+			op.Manager,
 			op.Clock,
 			op.GetClient(),
 			op.EventRecorder,

go.mod

@@ -31,7 +31,7 @@ require (
 	k8s.io/utils v0.0.0-20240102154912-e7106e64919e
 	knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd
 	sigs.k8s.io/controller-runtime v0.18.4
-	sigs.k8s.io/karpenter v0.37.1-0.20240708235259-bcd33e924905
+	sigs.k8s.io/karpenter v0.37.1-0.20240710233245-c691dd498fff
 	sigs.k8s.io/yaml v1.4.0
 )
 

go.sum

@@ -763,8 +763,8 @@ sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHv
 sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
-sigs.k8s.io/karpenter v0.37.1-0.20240708235259-bcd33e924905 h1:4ulecMwrSJl9EunKwfDGBhY0i3vJOYmI1GWs/JzsrjQ=
-sigs.k8s.io/karpenter v0.37.1-0.20240708235259-bcd33e924905/go.mod h1:eqRbKU0hmncoJXhh+MI8sCLYTjKDvoVPzo+myhDjvUI=
+sigs.k8s.io/karpenter v0.37.1-0.20240710233245-c691dd498fff h1:m4nfqPVZ/te1VbeGbAI5HD3R8Aj7y73ODcYPG6Ic/ak=
+sigs.k8s.io/karpenter v0.37.1-0.20240710233245-c691dd498fff/go.mod h1:jwEZ2Efxsc0yyNkrDEFN2RduAwlm/s7reIVNblZ8vyM=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=

hack/docs/instancetypes_gen/main.go

@@ -33,13 +33,13 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
-	corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1"
+	corev1 "sigs.k8s.io/karpenter/pkg/apis/v1"
 
 	coreoperator "sigs.k8s.io/karpenter/pkg/operator"
 	coreoptions "sigs.k8s.io/karpenter/pkg/operator/options"
 	coretest "sigs.k8s.io/karpenter/pkg/test"
 
-	"github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
+	providerv1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
 	"github.com/aws/karpenter-provider-aws/pkg/operator"
 	"github.com/aws/karpenter-provider-aws/pkg/operator/options"
 	"github.com/aws/karpenter-provider-aws/pkg/test"
@@ -103,10 +103,10 @@ func main() {
 		log.Fatalf("updating instance types offerings, %s", err)
 	}
 	// Fake a NodeClass so we can use it to get InstanceTypes
-	nodeClass := &v1beta1.EC2NodeClass{
-		Spec: v1beta1.EC2NodeClassSpec{
-			AMIFamily: &v1beta1.AMIFamilyAL2023,
-			SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{
+	nodeClass := &providerv1.EC2NodeClass{
+		Spec: providerv1.EC2NodeClassSpec{
+			AMIFamily: &providerv1.AMIFamilyAL2023,
+			SubnetSelectorTerms: []providerv1.SubnetSelectorTerm{
 				{
 					Tags: map[string]string{
 						"*": "*",
@@ -119,13 +119,13 @@ func main() {
 	if err != nil {
 		log.Fatalf("listing subnets, %s", err)
 	}
-	nodeClass.Status.Subnets = lo.Map(subnets, func(ec2subnet *ec2.Subnet, _ int) v1beta1.Subnet {
-		return v1beta1.Subnet{
+	nodeClass.Status.Subnets = lo.Map(subnets, func(ec2subnet *ec2.Subnet, _ int) providerv1.Subnet {
+		return providerv1.Subnet{
 			ID:   *ec2subnet.SubnetId,
 			Zone: *ec2subnet.AvailabilityZone,
 		}
 	})
-	instanceTypes, err := op.InstanceTypesProvider.List(ctx, &corev1beta1.KubeletConfiguration{}, nodeClass)
+	instanceTypes, err := op.InstanceTypesProvider.List(ctx, &providerv1.KubeletConfiguration{}, nodeClass)
 	if err != nil {
 		log.Fatalf("listing instance types, %s", err)
 	}
@@ -172,8 +172,8 @@ below are the resources available with some assumptions and after the instance o
 
 	// we don't want to show a few labels that will vary amongst regions
 	delete(labelNameMap, v1.LabelTopologyZone)
-	delete(labelNameMap, v1beta1.LabelTopologyZoneID)
-	delete(labelNameMap, corev1beta1.CapacityTypeLabelKey)
+	delete(labelNameMap, providerv1.LabelTopologyZoneID)
+	delete(labelNameMap, corev1.CapacityTypeLabelKey)
 
 	labelNames := lo.Keys(labelNameMap)
 

pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml

@@ -717,7 +717,7 @@ spec:
               type: object
           type: object
       served: true
-      storage: false
+      storage: true
       subresources:
         status: {}
     - name: v1beta1
@@ -1293,6 +1293,6 @@ spec:
               type: object
           type: object
       served: true
-      storage: true
+      storage: false
       subresources:
         status: {}

pkg/apis/crds/karpenter.sh_nodeclaims.yaml

@@ -348,7 +348,7 @@ spec:
             - spec
           type: object
       served: true
-      storage: false
+      storage: true
       subresources:
         status: {}
     - additionalPrinterColumns:
@@ -790,6 +790,6 @@ spec:
             - spec
           type: object
       served: true
-      storage: true
+      storage: false
       subresources:
         status: {}

pkg/apis/crds/karpenter.sh_nodepools.yaml

@@ -488,7 +488,7 @@ spec:
             - spec
           type: object
       served: true
-      storage: false
+      storage: true
       subresources:
         status: {}
     - additionalPrinterColumns:
@@ -1068,6 +1068,6 @@ spec:
             - spec
           type: object
       served: true
-      storage: true
+      storage: false
       subresources:
         status: {}

pkg/apis/v1/ec2nodeclass.go

@@ -21,7 +21,7 @@ import (
 	"github.com/samber/lo"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1"
+	v1 "sigs.k8s.io/karpenter/pkg/apis/v1"
 )
 
 // EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider.
@@ -92,7 +92,7 @@ type EC2NodeClassSpec struct {
 	// +kubebuilder:validation:XValidation:message="evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod",rule="has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true"
 	// +kubebuilder:validation:XValidation:message="evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft",rule="has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true"
 	// +optional
-	Kubelet *KubeletConfiguration `json:"kubelet,omitempty"`
+	Kubelet *KubeletConfiguration `json:"kubelet,omitempty" hash:"ignore"`
 	// BlockDeviceMappings to be applied to provisioned nodes.
 	// +kubebuilder:validation:XValidation:message="must have only one blockDeviceMappings with rootVolume",rule="self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1"
 	// +kubebuilder:validation:MaxItems:=50
@@ -186,7 +186,7 @@ type AMISelectorTerm struct {
 // KubeletConfiguration defines args to be used when configuring kubelet on provisioned nodes.
 // They are a subset of the upstream types, recognizing not all options may be supported.
 // Wherever possible, the types and names should reflect the upstream kubelet types.
-// https://pkg.go.dev/k8s.io/kubelet/config/v1beta1#KubeletConfiguration
+// https://pkg.go.dev/k8s.io/kubelet/config/v1#KubeletConfiguration
 // https://github.com/kubernetes/kubernetes/blob/9f82d81e55cafdedab619ea25cabf5d42736dacf/cmd/kubelet/app/options/options.go#L53
 type KubeletConfiguration struct {
 	// clusterDNS is a list of IP addresses for the cluster DNS server.
@@ -395,6 +395,7 @@ const (
 
 // EC2NodeClass is the Schema for the EC2NodeClass API
 // +kubebuilder:object:root=true
+// +kubebuilder:storageversion
 // +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status",description=""
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description=""
 // +kubebuilder:printcolumn:name="Role",type="string",JSONPath=".spec.role",priority=1,description=""
@@ -415,7 +416,7 @@ type EC2NodeClass struct {
 // 1. A field changes its default value for an existing field that is already hashed
 // 2. A field is added to the hash calculation with an already-set value
 // 3. A field is removed from the hash calculations
-const EC2NodeClassHashVersion = "v2"
+const EC2NodeClassHashVersion = "v3"
 
 func (in *EC2NodeClass) Hash() string {
 	return fmt.Sprint(lo.Must(hashstructure.Hash(in.Spec, hashstructure.FormatV2, &hashstructure.HashOptions{
@@ -436,7 +437,7 @@ func (in *EC2NodeClass) InstanceProfileRole() string {
 func (in *EC2NodeClass) InstanceProfileTags(clusterName string) map[string]string {
 	return lo.Assign(in.Spec.Tags, map[string]string{
 		fmt.Sprintf("kubernetes.io/cluster/%s", clusterName): "owned",
-		corev1beta1.ManagedByAnnotationKey:                   clusterName,
+		v1.ManagedByAnnotationKey:                            clusterName,
 		LabelNodeClass:                                       in.Name,
 	})
 }

pkg/apis/v1/ec2nodeclass_hash_test.go

@@ -15,6 +15,8 @@ limitations under the License.
 package v1_test
 
 import (
+	"time"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/imdario/mergo"
 	"github.com/samber/lo"
@@ -152,6 +154,18 @@ var _ = Describe("Hash", func() {
 		Entry("BlockDeviceMapping SnapshotID", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{SnapshotID: lo.ToPtr("test")}}}}}),
 		Entry("BlockDeviceMapping Throughput", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{Throughput: lo.ToPtr(int64(10))}}}}}),
 		Entry("BlockDeviceMapping VolumeType", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{VolumeType: lo.ToPtr("io1")}}}}}),
+		Entry("Kubelet ClusterDNS", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Kubelet: &v1.KubeletConfiguration{ClusterDNS: []string{"test-dns"}}}}),
+		Entry("Kubelet MaxPods", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Kubelet: &v1.KubeletConfiguration{MaxPods: lo.ToPtr(int32(10))}}}),
+		Entry("Kubelet PodsPerCore", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Kubelet: &v1.KubeletConfiguration{PodsPerCore: lo.ToPtr(int32(31))}}}),
+		Entry("Kubelet SystemReserved", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Kubelet: &v1.KubeletConfiguration{SystemReserved: map[string]string{"test-key-1": "test-value-1"}}}}),
+		Entry("Kubelet KubeReserved", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Kubelet: &v1.KubeletConfiguration{KubeReserved: map[string]string{"test-key-2": "test-value-2"}}}}),
+		Entry("Kubelet EvictionHard", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Kubelet: &v1.KubeletConfiguration{EvictionHard: map[string]string{"test-key-3": "test-value-3"}}}}),
+		Entry("Kubelet EvictionSoft", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Kubelet: &v1.KubeletConfiguration{EvictionSoft: map[string]string{"test-key-4": "test-value-4"}}}}),
+		Entry("Kubelet EvictionSoftGracePeriod", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Kubelet: &v1.KubeletConfiguration{EvictionSoftGracePeriod: map[string]metav1.Duration{"test-key": metav1.Duration{Duration: time.Minute}}}}}),
+		Entry("Kubelet EvictionMaxPodGracePeriod", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Kubelet: &v1.KubeletConfiguration{EvictionMaxPodGracePeriod: lo.ToPtr(int32(92))}}}),
+		Entry("Kubelet ImageGCHighThresholdPercent", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Kubelet: &v1.KubeletConfiguration{ImageGCHighThresholdPercent: lo.ToPtr(int32(23))}}}),
+		Entry("Kubelet ImageGCLowThresholdPercent", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Kubelet: &v1.KubeletConfiguration{ImageGCLowThresholdPercent: lo.ToPtr(int32(334))}}}),
+		Entry("Kubelet CPUCFSQuota", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Kubelet: &v1.KubeletConfiguration{CPUCFSQuota: lo.ToPtr(true)}}}),
 	)
 	// We create a separate test for updating blockDeviceMapping volumeSize, since resource.Quantity is a struct, and mergo.WithSliceDeepCopy
 	// doesn't work well with unexported fields, like the ones that are present in resource.Quantity