Merge branch 'main' into switch-to-v1

.github/workflows/approval-comment.yaml

@@ -19,7 +19,7 @@ jobs:
           mkdir -p /tmp/artifacts
           { echo "$REVIEW_BODY"; echo "$PULL_REQUEST_NUMBER"; echo "$COMMIT_ID"; } >> /tmp/artifacts/metadata.txt
           cat /tmp/artifacts/metadata.txt
-      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+      - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
         with:
           name: artifacts
           path: /tmp/artifacts

README.md

@@ -21,6 +21,8 @@ Come discuss Karpenter in the [#karpenter](https://kubernetes.slack.com/archives
 Check out the [Docs](https://karpenter.sh/docs/) to learn more.
 
 ## Talks
+- 03/19/2024 [Harnessing Karpenter: Transforming Kubernetes Clusters with Argo Workflows](https://www.youtube.com/watch?v=rq57liGu0H4)
+- 12/04/2023 [AWS re:Invent 2023 - Harness the power of Karpenter to scale, optimize & upgrade Kubernetes](https://www.youtube.com/watch?v=lkg_9ETHeks)
 - 09/08/2022 [Workload Consolidation with Karpenter](https://youtu.be/BnksdJ3oOEs)
 - 05/19/2022 [Scaling K8s Nodes Without Breaking the Bank or Your Sanity](https://www.youtube.com/watch?v=UBb8wbfSc34)
 - 03/25/2022 [Karpenter @ AWS Community Day 2022](https://youtu.be/sxDtmzbNHwE?t=3931)

charts/karpenter/templates/clusterrole-core.yaml

@@ -61,7 +61,7 @@ rules:
     verbs: ["create", "patch"]
   - apiGroups: [""]
     resources: ["nodes"]
-    verbs: ["patch", "delete"]
+    verbs: ["patch", "delete", "update"]
   - apiGroups: [""]
     resources: ["pods/eviction"]
     verbs: ["create"]

cmd/controller/main.go

@@ -68,5 +68,5 @@ func main() {
 			op.InstanceTypesProvider,
 		)...).
 		WithWebhooks(ctx, webhooks.NewWebhooks()...).
-		Start(ctx)
+		Start(ctx, cloudProvider)
 }

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/Pallinder/go-randomdata v1.2.0
 	github.com/PuerkitoBio/goquery v1.9.2
 	github.com/avast/retry-go v3.0.0+incompatible
-	github.com/aws/aws-sdk-go v1.54.11
+	github.com/aws/aws-sdk-go v1.54.15
 	github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20240410220356-6b868db24881
 	github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647
 	github.com/awslabs/operatorpkg v0.0.0-20240701195752-116cbcffbcb4
@@ -33,7 +33,7 @@ require (
 	k8s.io/utils v0.0.0-20240102154912-e7106e64919e
 	knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd
 	sigs.k8s.io/controller-runtime v0.18.4
-	sigs.k8s.io/karpenter v0.37.1-0.20240629051434-89a81c3ae853
+	sigs.k8s.io/karpenter v0.37.1-0.20240705194601-4a72da3afe8e
 	sigs.k8s.io/yaml v1.4.0
 )
 

go.sum

@@ -54,8 +54,8 @@ github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
 github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
-github.com/aws/aws-sdk-go v1.54.11 h1:Zxuv/R+IVS0B66yz4uezhxH9FN9/G2nbxejYqAMFjxk=
-github.com/aws/aws-sdk-go v1.54.11/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
+github.com/aws/aws-sdk-go v1.54.15 h1:ErgCEVbzuSfuZl9nR+g8FFnzjgeJ/AqAGOEWn6tgAHo=
+github.com/aws/aws-sdk-go v1.54.15/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
 github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20240410220356-6b868db24881 h1:m9rhsGhdepdQV96tZgfy68oU75AWAjOH8u65OefTjwA=
 github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20240410220356-6b868db24881/go.mod h1:+Mk5k0b6HpKobxNq+B56DOhZ+I/NiPhd5MIBhQMSTSs=
 github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647 h1:8yRBVsjGmI7qQsPWtIrbWP+XfwHO9Wq7gdLVzjqiZFs=
@@ -765,6 +765,8 @@ sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHv
 sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
+sigs.k8s.io/karpenter v0.37.1-0.20240705194601-4a72da3afe8e h1:FriyVb7NI+MsIlHW93QgHDWfhVf8Gt0E2NNYy+WA3Jg=
+sigs.k8s.io/karpenter v0.37.1-0.20240705194601-4a72da3afe8e/go.mod h1:Capmf9EwVSBoANOkP5qy4xHBaZ9/y0wUDyh4UzgRJlk=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=

pkg/apis/apis.go

@@ -18,12 +18,10 @@ package apis
 import (
 	_ "embed"
 
+	"github.com/awslabs/operatorpkg/object"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 
-	"github.com/samber/lo"
-
 	"sigs.k8s.io/karpenter/pkg/apis"
-	"sigs.k8s.io/karpenter/pkg/utils/functional"
 )
 
 //go:generate controller-gen crd object:headerFile="../../hack/boilerplate.go.txt" paths="./..." output:crd:artifacts:config=crds
@@ -32,6 +30,6 @@ var (
 	//go:embed crds/karpenter.k8s.aws_ec2nodeclasses.yaml
 	EC2NodeClassCRD []byte
 	CRDs            = append(apis.CRDs,
-		lo.Must(functional.Unmarshal[apiextensionsv1.CustomResourceDefinition](EC2NodeClassCRD)),
+		object.Unmarshal[apiextensionsv1.CustomResourceDefinition](EC2NodeClassCRD),
 	)
 )

pkg/apis/v1/ec2nodeclass_status.go

@@ -15,10 +15,17 @@ limitations under the License.
 package v1
 
 import (
-	op "github.com/awslabs/operatorpkg/status"
+	"github.com/awslabs/operatorpkg/status"
 	v1 "k8s.io/api/core/v1"
 )
 
+const (
+	ConditionTypeSubnetsReady         = "SubnetsReady"
+	ConditionTypeSecurityGroupsReady  = "SecurityGroupsReady"
+	ConditionTypeAMIsReady            = "AMIsReady"
+	ConditionTypeInstanceProfileReady = "InstanceProfileReady"
+)
+
 // Subnet contains resolved Subnet selector values utilized for node launch
 type Subnet struct {
 	// ID of the subnet
@@ -74,22 +81,22 @@ type EC2NodeClassStatus struct {
 	InstanceProfile string `json:"instanceProfile,omitempty"`
 	// Conditions contains signals for health and readiness
 	// +optional
-	Conditions []op.Condition `json:"conditions,omitempty"`
+	Conditions []status.Condition `json:"conditions,omitempty"`
 }
 
-const (
-	// 	ConditionTypeNodeClassReady = "Ready" condition indicates that subnets, security groups, AMIs and instance profile for nodeClass were resolved
-	ConditionTypeNodeClassReady = "Ready"
-)
-
-func (in *EC2NodeClass) StatusConditions() op.ConditionSet {
-	return op.NewReadyConditions(ConditionTypeNodeClassReady).For(in)
+func (in *EC2NodeClass) StatusConditions() status.ConditionSet {
+	return status.NewReadyConditions(
+		ConditionTypeAMIsReady,
+		ConditionTypeSubnetsReady,
+		ConditionTypeSecurityGroupsReady,
+		ConditionTypeInstanceProfileReady,
+	).For(in)
 }
 
-func (in *EC2NodeClass) GetConditions() []op.Condition {
+func (in *EC2NodeClass) GetConditions() []status.Condition {
 	return in.Status.Conditions
 }
 
-func (in *EC2NodeClass) SetConditions(conditions []op.Condition) {
+func (in *EC2NodeClass) SetConditions(conditions []status.Condition) {
 	in.Status.Conditions = conditions
 }

pkg/apis/v1/suite_test.go

@@ -16,6 +16,7 @@ package v1_test
 
 import (
 	"context"
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -41,7 +42,7 @@ func TestAPIs(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...))
+	env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...))
 	awsEnv = test.NewEnvironment(ctx, env)
 })
 

pkg/apis/v1beta1/ec2nodeclass_status.go

@@ -19,6 +19,13 @@ import (
 	v1 "k8s.io/api/core/v1"
 )
 
+const (
+	ConditionTypeSubnetsReady         = "SubnetsReady"
+	ConditionTypeSecurityGroupsReady  = "SecurityGroupsReady"
+	ConditionTypeAMIsReady            = "AMIsReady"
+	ConditionTypeInstanceProfileReady = "InstanceProfileReady"
+)
+
 // Subnet contains resolved Subnet selector values utilized for node launch
 type Subnet struct {
 	// ID of the subnet
@@ -78,7 +85,12 @@ type EC2NodeClassStatus struct {
 }
 
 func (in *EC2NodeClass) StatusConditions() status.ConditionSet {
-	return status.NewReadyConditions().For(in)
+	return status.NewReadyConditions(
+		ConditionTypeAMIsReady,
+		ConditionTypeSubnetsReady,
+		ConditionTypeSecurityGroupsReady,
+		ConditionTypeInstanceProfileReady,
+	).For(in)
 }
 
 func (in *EC2NodeClass) GetConditions() []status.Condition {

pkg/apis/v1beta1/suite_test.go

@@ -16,6 +16,7 @@ package v1beta1_test
 
 import (
 	"context"
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -41,7 +42,7 @@ func TestAPIs(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...))
+	env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...))
 	awsEnv = test.NewEnvironment(ctx, env)
 })
 

pkg/cloudprovider/cloudprovider.go

@@ -21,7 +21,6 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/awslabs/operatorpkg/object"
 	"github.com/awslabs/operatorpkg/status"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -32,7 +31,6 @@ import (
 	corev1 "sigs.k8s.io/karpenter/pkg/apis/v1"
 	"sigs.k8s.io/karpenter/pkg/events"
 	"sigs.k8s.io/karpenter/pkg/scheduling"
-	"sigs.k8s.io/karpenter/pkg/utils/functional"
 	"sigs.k8s.io/karpenter/pkg/utils/resources"
 
 	"github.com/aws/karpenter-provider-aws/pkg/apis"
@@ -225,7 +223,7 @@ func (c *CloudProvider) Name() string {
 }
 
 func (c *CloudProvider) GetSupportedNodeClasses() []schema.GroupVersionKind {
-	return []schema.GroupVersionKind{object.GVK(&providerv1.EC2NodeClass{})}
+  return []status.Object{&providerv1.EC2NodeClass{}}
 }
 
 func (c *CloudProvider) resolveNodeClassFromNodeClaim(ctx context.Context, nodeClaim *corev1.NodeClaim) (*providerv1.EC2NodeClass, error) {
@@ -331,8 +329,8 @@ func (c *CloudProvider) instanceToNodeClaim(i *instance.Instance, instanceType *
 			}
 			return true
 		}
-		nodeClaim.Status.Capacity = functional.FilterMap(instanceType.Capacity, resourceFilter)
-		nodeClaim.Status.Allocatable = functional.FilterMap(instanceType.Allocatable(), resourceFilter)
+		nodeClaim.Status.Capacity = lo.PickBy(instanceType.Capacity, resourceFilter)
+		nodeClaim.Status.Allocatable = lo.PickBy(instanceType.Allocatable(), resourceFilter)
 	}
 	labels[v1.LabelTopologyZone] = i.Zone
 	// Attempt to resolve the zoneID from the instance's EC2NodeClass' status condition.

pkg/cloudprovider/suite_test.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"strings"
 	"testing"
 	"time"
@@ -77,7 +78,7 @@ func TestAWS(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...))
+	env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...))
 	ctx = coreoptions.ToContext(ctx, coretest.Options())
 	ctx = options.ToContext(ctx, test.Options())
 	ctx, stop = context.WithCancel(ctx)

pkg/controllers/interruption/suite_test.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"testing"
 	"time"
 
@@ -75,7 +76,7 @@ func TestAPIs(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...))
+	env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...))
 	fakeClock = &clock.FakeClock{}
 	unavailableOfferingsCache = awscache.NewUnavailableOfferings()
 	sqsapi = &fake.SQSAPI{}

pkg/controllers/nodeclaim/garbagecollection/suite_test.go

@@ -17,6 +17,7 @@ package garbagecollection_test
 import (
 	"context"
 	"fmt"
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"sync"
 	"testing"
 	"time"
@@ -60,7 +61,7 @@ func TestAPIs(t *testing.T) {
 
 var _ = BeforeSuite(func() {
 	ctx = options.ToContext(ctx, test.Options())
-	env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...))
+	env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...))
 	awsEnv = test.NewEnvironment(ctx, env)
 	cloudProvider = cloudprovider.New(awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}),
 		env.Client, awsEnv.AMIProvider, awsEnv.SecurityGroupProvider)

pkg/controllers/nodeclaim/tagging/suite_test.go

@@ -17,6 +17,7 @@ package tagging_test
 import (
 	"context"
 	"fmt"
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -54,7 +55,7 @@ func TestAPIs(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...))
+	env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...))
 	ctx = coreoptions.ToContext(ctx, coretest.Options())
 	ctx = options.ToContext(ctx, test.Options())
 	awsEnv = test.NewEnvironment(ctx, env)

pkg/controllers/nodeclass/hash/suite_test.go

@@ -16,6 +16,7 @@ package hash_test
 
 import (
 	"context"
+	"sigs.k8s.io/karpenter/pkg/test/v1alpha1"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -50,7 +51,7 @@ func TestAPIs(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithFieldIndexers(test.EC2NodeClassFieldIndexer(ctx)))
+	env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...), coretest.WithFieldIndexers(test.EC2NodeClassFieldIndexer(ctx)))
 	ctx = coreoptions.ToContext(ctx, coretest.Options())
 	ctx = options.ToContext(ctx, test.Options())
 	awsEnv = test.NewEnvironment(ctx, env)

pkg/controllers/nodeclass/status/ami.go

@@ -41,6 +41,7 @@ func (a *AMI) Reconcile(ctx context.Context, nodeClass *providerv1.EC2NodeClass)
 	}
 	if len(amis) == 0 {
 		nodeClass.Status.AMIs = nil
+		nodeClass.StatusConditions().SetFalse(v1beta1.ConditionTypeAMIsReady, "AMINotFound", "AMISelector did not match any AMIs")
 		return reconcile.Result{}, nil
 	}
 	nodeClass.Status.AMIs = lo.Map(amis, func(ami amifamily.AMI, _ int) providerv1.AMI {
@@ -60,5 +61,6 @@ func (a *AMI) Reconcile(ctx context.Context, nodeClass *providerv1.EC2NodeClass)
 			Requirements: reqs,
 		}
 	})
+	nodeClass.StatusConditions().SetTrue(v1beta1.ConditionTypeAMIsReady)
 	return reconcile.Result{RequeueAfter: 5 * time.Minute}, nil
 }

pkg/controllers/nodeclass/status/ami_test.go

@@ -205,6 +205,7 @@ var _ = Describe("NodeClass AMI Status Controller", func() {
 				},
 			},
 		}))
+		Expect(nodeClass.StatusConditions().IsTrue(v1beta1.ConditionTypeAMIsReady)).To(BeTrue())
 	})
 	It("should resolve amiSelector AMis and requirements into status when all SSM aliases don't resolve", func() {
 		version := lo.Must(awsEnv.VersionProvider.Get(ctx))
@@ -283,6 +284,7 @@ var _ = Describe("NodeClass AMI Status Controller", func() {
 				},
 			},
 		}))
+		Expect(nodeClass.StatusConditions().IsTrue(v1beta1.ConditionTypeAMIsReady)).To(BeTrue())
 	})
 	It("Should resolve a valid AMI selector", func() {
 		ExpectApplied(ctx, env.Client, nodeClass)
@@ -302,5 +304,13 @@ var _ = Describe("NodeClass AMI Status Controller", func() {
 				},
 			},
 		))
+		Expect(nodeClass.StatusConditions().IsTrue(v1beta1.ConditionTypeAMIsReady)).To(BeTrue())
+	})
+	It("should get error when resolving AMIs and have status condition set to false", func() {
+		awsEnv.EC2API.NextError.Set(fmt.Errorf("unable to resolve AMI"))
+		ExpectApplied(ctx, env.Client, nodeClass)
+		ExpectObjectReconcileFailed(ctx, env.Client, statusController, nodeClass)
+		nodeClass = ExpectExists(ctx, env.Client, nodeClass)
+		Expect(nodeClass.StatusConditions().IsTrue(v1beta1.ConditionTypeAMIsReady)).To(BeFalse())
 	})
 })

pkg/controllers/nodeclass/status/instanceprofile.go

@@ -39,6 +39,6 @@ func (ip *InstanceProfile) Reconcile(ctx context.Context, nodeClass *providerv1.
 	} else {
 		nodeClass.Status.InstanceProfile = lo.FromPtr(nodeClass.Spec.InstanceProfile)
 	}
-
+	nodeClass.StatusConditions().SetTrue(v1beta1.ConditionTypeInstanceProfileReady)
 	return reconcile.Result{}, nil
 }