diff --git a/charts/karpenter/templates/clusterrole.yaml b/charts/karpenter/templates/clusterrole.yaml index 6b506e32b60f..92146277405e 100644 --- a/charts/karpenter/templates/clusterrole.yaml +++ b/charts/karpenter/templates/clusterrole.yaml @@ -42,5 +42,5 @@ rules: resourceNames: ["defaulting.webhook.karpenter.k8s.aws"] # Write - apiGroups: ["karpenter.k8s.aws"] - resources: ["awsnodetemplates/status"] + resources: ["awsnodetemplates", "awsnodetemplates/status"] verbs: ["patch", "update"] diff --git a/pkg/controllers/nodetemplate/controller.go b/pkg/controllers/nodetemplate/controller.go index 251057364959..db812ee6fb49 100644 --- a/pkg/controllers/nodetemplate/controller.go +++ b/pkg/controllers/nodetemplate/controller.go @@ -22,6 +22,7 @@ import ( "go.uber.org/multierr" "golang.org/x/time/rate" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/client-go/util/workqueue" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -61,14 +62,21 @@ func NewController(kubeClient client.Client, subnetProvider *subnet.Provider, se func (c *Controller) Reconcile(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) (reconcile.Result, error) { stored := nodeTemplate.DeepCopy() + nodeTemplate.Annotations = lo.Assign(nodeTemplate.ObjectMeta.Annotations, map[string]string{v1alpha1.AnnotationNodeTemplateHash: nodeTemplate.Hash()}) err := multierr.Combine( c.resolveSubnets(ctx, nodeTemplate), c.resolveSecurityGroups(ctx, nodeTemplate), c.resolveAMIs(ctx, nodeTemplate), ) - if patchErr := c.kubeClient.Status().Patch(ctx, nodeTemplate, client.MergeFrom(stored)); patchErr != nil { - err = multierr.Append(err, client.IgnoreNotFound(patchErr)) + if !equality.Semantic.DeepEqual(stored, nodeTemplate) { + statusCopy := nodeTemplate.DeepCopy() + if patchErr := c.kubeClient.Patch(ctx, nodeTemplate, client.MergeFrom(stored)); patchErr != nil { + err = multierr.Append(err, client.IgnoreNotFound(patchErr)) + } + if patchErr := c.kubeClient.Status().Patch(ctx, statusCopy, client.MergeFrom(stored)); patchErr != nil { + err = multierr.Append(err, client.IgnoreNotFound(patchErr)) + } } return reconcile.Result{RequeueAfter: 5 * time.Minute}, err diff --git a/pkg/controllers/nodetemplate/suite_test.go b/pkg/controllers/nodetemplate/suite_test.go index c1c43222103c..9b14d803e47b 100644 --- a/pkg/controllers/nodetemplate/suite_test.go +++ b/pkg/controllers/nodetemplate/suite_test.go @@ -720,6 +720,55 @@ var _ = Describe("AWSNodeTemplateController", func() { )) }) }) + Context("AWSNodeTemplate Static Drift Hash", func() { + DescribeTable("should update the static drift hash when nodeTemplate static field is updated", func(awsnodetemplatespec v1alpha1.AWSNodeTemplateSpec) { + updatedAWSNodeTemplate := test.AWSNodeTemplate(*nodeTemplate.Spec.DeepCopy(), awsnodetemplatespec) + updatedAWSNodeTemplate.ObjectMeta = nodeTemplate.ObjectMeta + updatedAWSNodeTemplate.Annotations = map[string]string{v1alpha1.AnnotationNodeTemplateHash: updatedAWSNodeTemplate.Hash()} + + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + + expectedHash := nodeTemplate.Hash() + Expect(nodeTemplate.ObjectMeta.Annotations[v1alpha1.AnnotationNodeTemplateHash]).To(Equal(expectedHash)) + + ExpectApplied(ctx, env.Client, updatedAWSNodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + + expectedHashTwo := nodeTemplate.Hash() + Expect(expectedHash).ToNot(Equal(expectedHashTwo)) + Expect(nodeTemplate.ObjectMeta.Annotations[v1alpha1.AnnotationNodeTemplateHash]).To(Equal(expectedHashTwo)) + }, + Entry("InstanceProfile Drift", v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{InstanceProfile: aws.String("profile-2")}}), + Entry("UserData Drift", v1alpha1.AWSNodeTemplateSpec{UserData: aws.String("userdata-test-2")}), + Entry("Tags Drift", v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), + Entry("MetadataOptions Drift", v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{LaunchTemplate: v1alpha1.LaunchTemplate{MetadataOptions: &v1alpha1.MetadataOptions{HTTPEndpoint: aws.String("test-metadata-2")}}}}), + Entry("BlockDeviceMappings Drift", v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{LaunchTemplate: v1alpha1.LaunchTemplate{BlockDeviceMappings: []*v1alpha1.BlockDeviceMapping{{DeviceName: aws.String("map-device-test-3")}}}}}), + Entry("Context Drift", v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{Context: aws.String("context-2")}}), + Entry("DetailedMonitoring Drift", v1alpha1.AWSNodeTemplateSpec{DetailedMonitoring: aws.Bool(true)}), + Entry("AMIFamily Drift", v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{AMIFamily: aws.String(v1alpha1.AMIFamilyBottlerocket)}}), + ) + It("should not update the static drift hash when nodeTemplate dynamic field is updated", func() { + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + + expectedHash := nodeTemplate.Hash() + Expect(nodeTemplate.ObjectMeta.Annotations[v1alpha1.AnnotationNodeTemplateHash]).To(Equal(expectedHash)) + + nodeTemplate.Spec.SubnetSelector = map[string]string{"aws-ids": "subnet-test1"} + nodeTemplate.Spec.SecurityGroupSelector = map[string]string{"aws-ids": "sg-test1"} + nodeTemplate.Spec.AMISelector = map[string]string{"ami-test-key": "ami-test-value"} + + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + + Expect(nodeTemplate.ObjectMeta.Annotations[v1alpha1.AnnotationNodeTemplateHash]).To(Equal(expectedHash)) + }) + }) }) func sortRequirements(amis []v1alpha1.AMI) { diff --git a/test/suites/integration/hash_test.go b/test/suites/integration/hash_test.go new file mode 100644 index 000000000000..59a290562dd3 --- /dev/null +++ b/test/suites/integration/hash_test.go @@ -0,0 +1,65 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + "github.com/aws/karpenter-core/pkg/test" + "github.com/aws/karpenter/pkg/apis/settings" + "github.com/aws/karpenter/pkg/apis/v1alpha1" + + awstest "github.com/aws/karpenter/pkg/test" +) + +var _ = Describe("CRD Hash", func() { + It("should have Provisioner hash", func() { + provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ + AWS: v1alpha1.AWS{ + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + }, + }) + provisioner := test.Provisioner(test.ProvisionerOptions{ + ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, + }) + + env.ExpectCreated(provider, provisioner) + + var prov v1alpha5.Provisioner + err := env.Client.Get(env, client.ObjectKeyFromObject(provisioner), &prov) + Expect(err).ToNot(HaveOccurred()) + + Expect(prov.Annotations[v1alpha5.ProvisionerHashAnnotationKey]).To(Equal(provisioner.Hash())) + }) + It("should have AWSNodeTemplate hash", func() { + nodeTemplate := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ + AWS: v1alpha1.AWS{ + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + }, + }) + env.ExpectCreated(nodeTemplate) + + var ant v1alpha1.AWSNodeTemplate + err := env.Client.Get(env, client.ObjectKeyFromObject(nodeTemplate), &ant) + Expect(err).ToNot(HaveOccurred()) + + Expect(ant.Annotations[v1alpha1.AnnotationNodeTemplateHash]).To(Equal(nodeTemplate.Hash())) + }) +})