diff --git a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml index 1de256a30958..66402178321a 100644 --- a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -1112,8 +1112,8 @@ spec: rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) - message: tag contains a restricted tag matching karpenter.sh/nodepool rule: self.all(k, k != 'karpenter.sh/nodepool') - - message: tag contains a restricted tag matching karpenter.sh/managed-by - rule: self.all(k, k !='karpenter.sh/managed-by') + - message: tag contains a restricted tag matching eks:eks-cluster-name + rule: self.all(k, k !='eks:eks-cluster-name') - message: tag contains a restricted tag matching karpenter.sh/nodeclaim rule: self.all(k, k !='karpenter.sh/nodeclaim') - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass diff --git a/pkg/apis/v1beta1/ec2nodeclass.go b/pkg/apis/v1beta1/ec2nodeclass.go index b3395dd3a60d..846aa9bba534 100644 --- a/pkg/apis/v1beta1/ec2nodeclass.go +++ b/pkg/apis/v1beta1/ec2nodeclass.go @@ -21,7 +21,6 @@ import ( "github.com/samber/lo" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" ) // EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. @@ -80,7 +79,7 @@ type EC2NodeClassSpec struct { // +kubebuilder:validation:XValidation:message="empty tag keys aren't supported",rule="self.all(k, k != '')" // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching kubernetes.io/cluster/",rule="self.all(k, !k.startsWith('kubernetes.io/cluster') )" // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.sh/nodepool",rule="self.all(k, k != 'karpenter.sh/nodepool')" - // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.sh/managed-by",rule="self.all(k, k !='karpenter.sh/managed-by')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching eks:eks-cluster-name",rule="self.all(k, k !='eks:eks-cluster-name')" // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.sh/nodeclaim",rule="self.all(k, k !='karpenter.sh/nodeclaim')" // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass",rule="self.all(k, k !='karpenter.k8s.aws/ec2nodeclass')" // +optional @@ -357,8 +356,8 @@ func (in *EC2NodeClass) InstanceProfileRole() string { func (in *EC2NodeClass) InstanceProfileTags(clusterName string) map[string]string { return lo.Assign(in.Spec.Tags, map[string]string{ fmt.Sprintf("kubernetes.io/cluster/%s", clusterName): "owned", - corev1beta1.ManagedByAnnotationKey: clusterName, - LabelNodeClass: in.Name, + EKSClusterNameTag: clusterName, + LabelNodeClass: in.Name, }) } diff --git a/pkg/apis/v1beta1/ec2nodeclass_validation_cel_test.go b/pkg/apis/v1beta1/ec2nodeclass_validation_cel_test.go index b0104c9c506a..84a3cfab3f59 100644 --- a/pkg/apis/v1beta1/ec2nodeclass_validation_cel_test.go +++ b/pkg/apis/v1beta1/ec2nodeclass_validation_cel_test.go @@ -80,7 +80,7 @@ var _ = Describe("CEL/Validation", func() { } Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) nc.Spec.Tags = map[string]string{ - corev1beta1.ManagedByAnnotationKey: "test", + v1beta1.EKSClusterNameTag: "test", } Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) nc.Spec.Tags = map[string]string{ diff --git a/pkg/apis/v1beta1/ec2nodeclass_validation_webhook_test.go b/pkg/apis/v1beta1/ec2nodeclass_validation_webhook_test.go index 18035649e235..255eafa52b93 100644 --- a/pkg/apis/v1beta1/ec2nodeclass_validation_webhook_test.go +++ b/pkg/apis/v1beta1/ec2nodeclass_validation_webhook_test.go @@ -78,7 +78,7 @@ var _ = Describe("Webhook/Validation", func() { } Expect(nc.Validate(ctx)).To(Succeed()) nc.Spec.Tags = map[string]string{ - "karpenterzsh/managed-by": "test", + "ekszsh:eks-cluster-name": "test", } Expect(nc.Validate(ctx)).To(Succeed()) }) @@ -92,7 +92,7 @@ var _ = Describe("Webhook/Validation", func() { } Expect(nc.Validate(ctx)).To(Not(Succeed())) nc.Spec.Tags = map[string]string{ - "karpenter.sh/managed-by": "test", + "eks:eks-cluster-name": "test", } Expect(nc.Validate(ctx)).To(Not(Succeed())) nc.Spec.Tags = map[string]string{ diff --git a/pkg/apis/v1beta1/labels.go b/pkg/apis/v1beta1/labels.go index 4b51945b2306..e163ced7cbce 100644 --- a/pkg/apis/v1beta1/labels.go +++ b/pkg/apis/v1beta1/labels.go @@ -71,10 +71,13 @@ var ( // https://docs.aws.amazon.com/eks/latest/APIReference/API_CreateCluster.html regexp.MustCompile(`^kubernetes\.io/cluster/[0-9A-Za-z][A-Za-z0-9\-_]*$`), regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(v1beta1.NodePoolLabelKey))), - regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(v1beta1.ManagedByAnnotationKey))), + regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(EKSClusterNameTag))), regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(LabelNodeClass))), regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(TagNodeClaim))), } + + EKSClusterNameTag = "eks:eks-cluster-name" + AMIFamilyBottlerocket = "Bottlerocket" AMIFamilyAL2 = "AL2" AMIFamilyAL2023 = "AL2023" diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index db76eed0eb34..e073e17f479e 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -342,8 +342,8 @@ func (c *CloudProvider) instanceToNodeClaim(i *instance.Instance, instanceType * if v, ok := i.Tags[corev1beta1.NodePoolLabelKey]; ok { labels[corev1beta1.NodePoolLabelKey] = v } - if v, ok := i.Tags[corev1beta1.ManagedByAnnotationKey]; ok { - annotations[corev1beta1.ManagedByAnnotationKey] = v + if v, ok := i.Tags[corev1beta1.NodePoolLabelKey]; ok { + annotations[corev1beta1.NodePoolLabelKey] = v } nodeClaim.Labels = labels nodeClaim.Annotations = annotations diff --git a/pkg/controllers/nodeclaim/garbagecollection/controller.go b/pkg/controllers/nodeclaim/garbagecollection/controller.go index 04b27aa88af5..0de0b838e761 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/controller.go +++ b/pkg/controllers/nodeclaim/garbagecollection/controller.go @@ -31,10 +31,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" + corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/operator/injection" - - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" ) type Controller struct { @@ -61,10 +60,10 @@ func (c *Controller) Reconcile(ctx context.Context) (reconcile.Result, error) { if err != nil { return reconcile.Result{}, fmt.Errorf("listing cloudprovider machines, %w", err) } - managedRetrieved := lo.Filter(retrieved, func(nc *v1beta1.NodeClaim, _ int) bool { - return nc.Annotations[v1beta1.ManagedByAnnotationKey] != "" && nc.DeletionTimestamp.IsZero() + managedRetrieved := lo.Filter(retrieved, func(nc *corev1beta1.NodeClaim, _ int) bool { + return nc.Annotations[corev1beta1.NodePoolLabelKey] != "" && nc.DeletionTimestamp.IsZero() }) - nodeClaimList := &v1beta1.NodeClaimList{} + nodeClaimList := &corev1beta1.NodeClaimList{} if err = c.kubeClient.List(ctx, nodeClaimList); err != nil { return reconcile.Result{}, err } @@ -72,7 +71,7 @@ func (c *Controller) Reconcile(ctx context.Context) (reconcile.Result, error) { if err = c.kubeClient.List(ctx, nodeList); err != nil { return reconcile.Result{}, err } - resolvedProviderIDs := sets.New[string](lo.FilterMap(nodeClaimList.Items, func(n v1beta1.NodeClaim, _ int) (string, bool) { + resolvedProviderIDs := sets.New[string](lo.FilterMap(nodeClaimList.Items, func(n corev1beta1.NodeClaim, _ int) (string, bool) { return n.Status.ProviderID, n.Status.ProviderID != "" })...) errs := make([]error, len(retrieved)) @@ -89,7 +88,7 @@ func (c *Controller) Reconcile(ctx context.Context) (reconcile.Result, error) { return reconcile.Result{RequeueAfter: lo.Ternary(c.successfulCount <= 20, time.Second*10, time.Minute*2)}, nil } -func (c *Controller) garbageCollect(ctx context.Context, nodeClaim *v1beta1.NodeClaim, nodeList *v1.NodeList) error { +func (c *Controller) garbageCollect(ctx context.Context, nodeClaim *corev1beta1.NodeClaim, nodeList *v1.NodeList) error { ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("provider-id", nodeClaim.Status.ProviderID)) if err := c.cloudProvider.Delete(ctx, nodeClaim); err != nil { return cloudprovider.IgnoreNodeClaimNotFoundError(err) diff --git a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go index 37e433e94f31..cf3a309ae75a 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go +++ b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go @@ -115,7 +115,7 @@ var _ = Describe("GarbageCollection", func() { Value: aws.String(nodeClass.Name), }, { - Key: aws.String(corev1beta1.ManagedByAnnotationKey), + Key: aws.String(v1beta1.EKSClusterNameTag), Value: aws.String(options.FromContext(ctx).ClusterName), }, }, @@ -183,7 +183,7 @@ var _ = Describe("GarbageCollection", func() { Value: aws.String("default"), }, { - Key: aws.String(corev1beta1.ManagedByAnnotationKey), + Key: aws.String(v1beta1.EKSClusterNameTag), Value: aws.String(options.FromContext(ctx).ClusterName), }, }, @@ -288,9 +288,9 @@ var _ = Describe("GarbageCollection", func() { Expect(err).NotTo(HaveOccurred()) }) It("should not delete an instance if it was not launched by a NodeClaim", func() { - // Remove the "karpenter.sh/managed-by" tag (this isn't launched by a machine) + // Remove the nodepool tag (this isn't launched by a machine) instance.Tags = lo.Reject(instance.Tags, func(t *ec2.Tag, _ int) bool { - return aws.StringValue(t.Key) == corev1beta1.ManagedByAnnotationKey + return aws.StringValue(t.Key) == corev1beta1.NodePoolLabelKey }) // Launch time was 1m ago @@ -350,7 +350,7 @@ var _ = Describe("GarbageCollection", func() { Value: aws.String("default"), }, { - Key: aws.String(corev1beta1.ManagedByAnnotationKey), + Key: aws.String(v1beta1.EKSClusterNameTag), Value: aws.String(options.FromContext(ctx).ClusterName), }, }, diff --git a/pkg/controllers/nodeclaim/tagging/suite_test.go b/pkg/controllers/nodeclaim/tagging/suite_test.go index e379439ff7dc..aeea53228ba9 100644 --- a/pkg/controllers/nodeclaim/tagging/suite_test.go +++ b/pkg/controllers/nodeclaim/tagging/suite_test.go @@ -90,7 +90,7 @@ var _ = Describe("TaggingController", func() { Value: aws.String("default"), }, { - Key: aws.String(corev1beta1.ManagedByAnnotationKey), + Key: aws.String(v1beta1.EKSClusterNameTag), Value: aws.String(options.FromContext(ctx).ClusterName), }, }, diff --git a/pkg/providers/instance/instance.go b/pkg/providers/instance/instance.go index 95c17cefb206..f80f532dddac 100644 --- a/pkg/providers/instance/instance.go +++ b/pkg/providers/instance/instance.go @@ -261,9 +261,9 @@ func (p *DefaultProvider) launchInstance(ctx context.Context, nodeClass *v1beta1 func getTags(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim) map[string]string { staticTags := map[string]string{ fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName): "owned", - corev1beta1.NodePoolLabelKey: nodeClaim.Labels[corev1beta1.NodePoolLabelKey], - corev1beta1.ManagedByAnnotationKey: options.FromContext(ctx).ClusterName, - v1beta1.LabelNodeClass: nodeClass.Name, + corev1beta1.NodePoolLabelKey: nodeClaim.Labels[corev1beta1.NodePoolLabelKey], + v1beta1.EKSClusterNameTag: options.FromContext(ctx).ClusterName, + v1beta1.LabelNodeClass: nodeClass.Name, } return lo.Assign(nodeClass.Spec.Tags, staticTags) } diff --git a/pkg/providers/instance/suite_test.go b/pkg/providers/instance/suite_test.go index 43b01993624c..df3740f99cb9 100644 --- a/pkg/providers/instance/suite_test.go +++ b/pkg/providers/instance/suite_test.go @@ -160,7 +160,7 @@ var _ = Describe("InstanceProvider", func() { Value: aws.String("default"), }, { - Key: aws.String(corev1beta1.ManagedByAnnotationKey), + Key: aws.String(v1beta1.EKSClusterNameTag), Value: aws.String(options.FromContext(ctx).ClusterName), }, }, diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index 8a14daaf04a6..39148de3ea13 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -262,7 +262,11 @@ func (p *DefaultProvider) createLaunchTemplate(ctx context.Context, options *ami TagSpecifications: []*ec2.TagSpecification{ { ResourceType: aws.String(ec2.ResourceTypeLaunchTemplate), - Tags: utils.MergeTags(options.Tags, map[string]string{v1beta1.TagManagedLaunchTemplate: options.ClusterName, v1beta1.LabelNodeClass: options.NodeClassName}), + Tags: utils.MergeTags(options.Tags, map[string]string{ + v1beta1.EKSClusterNameTag: options.ClusterName, + v1beta1.TagManagedLaunchTemplate: options.ClusterName, + v1beta1.LabelNodeClass: options.NodeClassName, + }), }, }, }) diff --git a/test/hack/resource/pkg/resourcetypes/resourcetypes.go b/test/hack/resource/pkg/resourcetypes/resourcetypes.go index ce8a7bfd9441..9757c14fcd26 100644 --- a/test/hack/resource/pkg/resourcetypes/resourcetypes.go +++ b/test/hack/resource/pkg/resourcetypes/resourcetypes.go @@ -20,7 +20,7 @@ import ( ) const ( - karpenterClusterNameTag = "karpenter.sh/managed-by" + karpenterClusterNameTag = "eks:eks-cluster-name" karpenterNodePoolTag = "karpenter.sh/nodepool" karpenterLaunchTemplateTag = "karpenter.k8s.aws/cluster" karpenterSecurityGroupTag = "karpenter.sh/discovery" diff --git a/test/suites/integration/tags_test.go b/test/suites/integration/tags_test.go index c2e28fe2fd51..b740cec8a818 100644 --- a/test/suites/integration/tags_test.go +++ b/test/suites/integration/tags_test.go @@ -87,7 +87,7 @@ var _ = Describe("Tags", func() { }) Context("Tagging Controller", func() { - It("should tag with karpenter.sh/nodeclaim and Name tag", func() { + It("should tag with karpenter.sh/nodeclaim, eks:eks-cluster-name, and Name tag", func() { pod := coretest.Pod() env.ExpectCreated(nodePool, nodeClass, pod) @@ -102,6 +102,7 @@ var _ = Describe("Tags", func() { }, time.Minute) nodeInstance := instance.NewInstance(lo.ToPtr(env.GetInstance(node.Name))) + Expect(nodeInstance.Tags).To(HaveKeyWithValue(v1beta1.EKSClusterNameTag, env.ClusterName)) Expect(nodeInstance.Tags).To(HaveKeyWithValue("Name", node.Name)) Expect(nodeInstance.Tags).To(HaveKey("karpenter.sh/nodeclaim")) }) diff --git a/test/suites/integration/validation_test.go b/test/suites/integration/validation_test.go index d116e07af33a..5011559bfcf0 100644 --- a/test/suites/integration/validation_test.go +++ b/test/suites/integration/validation_test.go @@ -194,7 +194,7 @@ var _ = Describe("Validation", func() { nodeClass.Spec.Tags = map[string]string{"karpenter.sh/nodepool": "custom-value"} Expect(env.Client.Create(env.Context, nodeClass)).ToNot(Succeed()) - nodeClass.Spec.Tags = map[string]string{"karpenter.sh/managed-by": env.ClusterName} + nodeClass.Spec.Tags = map[string]string{v1beta1.EKSClusterNameTag: env.ClusterName} Expect(env.Client.Create(env.Context, nodeClass)).ToNot(Succeed()) nodeClass.Spec.Tags = map[string]string{fmt.Sprintf("kubernetes.io/cluster/%s", env.ClusterName): "owned"} diff --git a/test/suites/nodeclaim/garbage_collection_test.go b/test/suites/nodeclaim/garbage_collection_test.go index 78f7cca88f64..90ab35407c8d 100644 --- a/test/suites/nodeclaim/garbage_collection_test.go +++ b/test/suites/nodeclaim/garbage_collection_test.go @@ -125,12 +125,12 @@ var _ = Describe("GarbageCollection", func() { // Wait for the node to register with the cluster node := env.EventuallyExpectCreatedNodeCount("==", 1)[0] - // Update the tags to add the karpenter.sh/managed-by tag + // Update the tags to add the eks:eks-cluster-name tag _, err = env.EC2API.CreateTagsWithContext(env.Context, &ec2.CreateTagsInput{ Resources: []*string{out.Instances[0].InstanceId}, Tags: []*ec2.Tag{ { - Key: aws.String(corev1beta1.ManagedByAnnotationKey), + Key: aws.String(v1beta1.EKSClusterNameTag), Value: aws.String(env.ClusterName), }, },