Update Controller to Include Hash

charts/karpenter/templates/clusterrole.yaml

@@ -42,5 +42,5 @@ rules:
     resourceNames: ["defaulting.webhook.karpenter.k8s.aws"]
   # Write
   - apiGroups: ["karpenter.k8s.aws"]
-    resources: ["awsnodetemplates/status"]
+    resources: ["awsnodetemplates", "awsnodetemplates/status"]
     verbs: ["patch", "update"]

pkg/controllers/nodetemplate/controller.go

@@ -16,12 +16,11 @@ package nodetemplate
 
 import (
 	"context"
-	"fmt"
-	"sort"
 	"time"
 
 	"go.uber.org/multierr"
 	"golang.org/x/time/rate"
+	"k8s.io/apimachinery/pkg/api/equality"
 	"k8s.io/client-go/util/workqueue"
 	controllerruntime "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -30,48 +29,60 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/samber/lo"
-
 	corecontroller "github.com/aws/karpenter-core/pkg/operator/controller"
+	"github.com/aws/karpenter-core/pkg/utils/result"
 	"github.com/aws/karpenter/pkg/apis/v1alpha1"
 	"github.com/aws/karpenter/pkg/providers/amifamily"
 	"github.com/aws/karpenter/pkg/providers/securitygroup"
 	"github.com/aws/karpenter/pkg/providers/subnet"
 )
 
+type nodeTemplateReconciler interface {
+	Reconcile(context.Context, *v1alpha1.AWSNodeTemplate) (reconcile.Result, error)
+}
+
 var _ corecontroller.TypedController[*v1alpha1.AWSNodeTemplate] = (*Controller)(nil)
 
 type Controller struct {
-	kubeClient            client.Client
-	subnetProvider        *subnet.Provider
-	securityGroupProvider *securitygroup.Provider
-	amiProvider           *amifamily.Provider
+	kubeClient client.Client
+	status     *Status
+	hash       *Hash
 }
 
-func NewController(kubeClient client.Client, subnetProvider *subnet.Provider, securityGroups *securitygroup.Provider, amiprovider *amifamily.Provider) corecontroller.Controller {
+func NewController(kubeClient client.Client, subnetProvider *subnet.Provider, securityGroupProvider *securitygroup.Provider, amiProvider *amifamily.Provider) corecontroller.Controller {
 	return corecontroller.Typed[*v1alpha1.AWSNodeTemplate](kubeClient, &Controller{
-		kubeClient:            kubeClient,
-		subnetProvider:        subnetProvider,
-		securityGroupProvider: securityGroups,
-		amiProvider:           amiprovider,
+		kubeClient: kubeClient,
+		status:     &Status{subnetProvider: subnetProvider, securityGroupProvider: securityGroupProvider, amiProvider: amiProvider},
+		hash:       &Hash{},
 	})
 }
 
 func (c *Controller) Reconcile(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) (reconcile.Result, error) {
 	stored := nodeTemplate.DeepCopy()
 
-	err := multierr.Combine(
-		c.resolveSubnets(ctx, nodeTemplate),
-		c.resolveSecurityGroups(ctx, nodeTemplate),
-		c.resolveAMIs(ctx, nodeTemplate),
-	)
+	var results []reconcile.Result
+	var errs error
+	reconcilers := []nodeTemplateReconciler{
+		c.status,
+		c.hash,
+	}
+	for _, reconciler := range reconcilers {
+		res, err := reconciler.Reconcile(ctx, nodeTemplate)
+		errs = multierr.Append(errs, err)
+		results = append(results, res)
+	}
 
-	if patchErr := c.kubeClient.Status().Patch(ctx, nodeTemplate, client.MergeFrom(stored)); patchErr != nil {
-		err = multierr.Append(err, client.IgnoreNotFound(patchErr))
+	if !equality.Semantic.DeepEqual(stored, nodeTemplate) {
+		statusCopy := nodeTemplate.DeepCopy()
+		if patchErr := c.kubeClient.Patch(ctx, nodeTemplate, client.MergeFrom(stored)); patchErr != nil {
+			errs = multierr.Append(errs, patchErr)
+		}
+		if patchErr := c.kubeClient.Status().Patch(ctx, statusCopy, client.MergeFrom(stored)); patchErr != nil {
+			errs = multierr.Append(errs, patchErr)
+		}
 	}
 
-	return reconcile.Result{RequeueAfter: 5 * time.Minute}, err
+	return result.Min(results...), errs
 }
 
 func (c *Controller) Name() string {
@@ -92,68 +103,3 @@ func (c *Controller) Builder(_ context.Context, m manager.Manager) corecontrolle
 			MaxConcurrentReconciles: 10,
 		}))
 }
-
-func (c *Controller) resolveSubnets(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) error {
-	subnetList, err := c.subnetProvider.List(ctx, nodeTemplate)
-	if err != nil {
-		return err
-	}
-	if len(subnetList) == 0 {
-		nodeTemplate.Status.Subnets = nil
-		return fmt.Errorf("no subnets exist given constraints")
-	}
-
-	sort.Slice(subnetList, func(i, j int) bool {
-		return int(*subnetList[i].AvailableIpAddressCount) > int(*subnetList[j].AvailableIpAddressCount)
-	})
-
-	nodeTemplate.Status.Subnets = lo.Map(subnetList, func(ec2subnet *ec2.Subnet, _ int) v1alpha1.Subnet {
-		return v1alpha1.Subnet{
-			ID:   *ec2subnet.SubnetId,
-			Zone: *ec2subnet.AvailabilityZone,
-		}
-	})
-
-	return nil
-}
-
-func (c *Controller) resolveSecurityGroups(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) error {
-	securityGroups, err := c.securityGroupProvider.List(ctx, nodeTemplate)
-	if err != nil {
-		return err
-	}
-	if len(securityGroups) == 0 && nodeTemplate.Spec.SecurityGroupSelector != nil {
-		nodeTemplate.Status.SecurityGroups = nil
-		return fmt.Errorf("no security groups exist given constraints")
-	}
-
-	nodeTemplate.Status.SecurityGroups = lo.Map(securityGroups, func(securityGroup *ec2.SecurityGroup, _ int) v1alpha1.SecurityGroup {
-		return v1alpha1.SecurityGroup{
-			ID:   *securityGroup.GroupId,
-			Name: *securityGroup.GroupName,
-		}
-	})
-
-	return nil
-}
-
-func (c *Controller) resolveAMIs(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) error {
-	amis, err := c.amiProvider.Get(ctx, nodeTemplate, &amifamily.Options{})
-	if err != nil {
-		return err
-	}
-	if len(amis) == 0 {
-		nodeTemplate.Status.AMIs = nil
-		return fmt.Errorf("no amis exist given constraints")
-	}
-
-	nodeTemplate.Status.AMIs = lo.Map(amis, func(ami amifamily.AMI, _ int) v1alpha1.AMI {
-		return v1alpha1.AMI{
-			Name:         ami.Name,
-			ID:           ami.AmiID,
-			Requirements: ami.Requirements.NodeSelectorRequirements(),
-		}
-	})
-
-	return nil
-}

pkg/controllers/nodetemplate/hash.go

@@ -0,0 +1,32 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nodetemplate
+
+import (
+	"context"
+
+	"github.com/aws/karpenter/pkg/apis/v1alpha1"
+	"github.com/samber/lo"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+type Hash struct{}
+
+func (s *Hash) Reconcile(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) (reconcile.Result, error) {
+	nodeTemplateHash := nodeTemplate.Hash()
+	nodeTemplate.Annotations = lo.Assign(nodeTemplate.ObjectMeta.Annotations, map[string]string{v1alpha1.AnnotationNodeTemplateHash: nodeTemplateHash})
+
+	return reconcile.Result{}, nil
+}

pkg/controllers/nodetemplate/hash_test.go

@@ -0,0 +1,61 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nodetemplate_test
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/aws/aws-sdk-go/aws"
+	. "github.com/aws/karpenter-core/pkg/test/expectations"
+	"github.com/aws/karpenter/pkg/apis/v1alpha1"
+)
+
+var _ = Describe("AWSNodeTemplate Static Drift Hash", func() {
+	It("should update the static drift hash when provisioner static field is updated", func() {
+		ExpectApplied(ctx, env.Client, nodeTemplate)
+		ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate))
+		nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate)
+
+		expectedHash := nodeTemplate.Hash()
+		Expect(nodeTemplate.ObjectMeta.Annotations[v1alpha1.AnnotationNodeTemplateHash]).To(Equal(expectedHash))
+
+		nodeTemplate.Spec.UserData = aws.String("test-userData-2")
+		ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate))
+		nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate)
+
+		expectedHashTwo := nodeTemplate.Hash()
+		Expect(nodeTemplate.ObjectMeta.Annotations[v1alpha1.AnnotationNodeTemplateHash]).To(Equal(expectedHashTwo))
+	})
+	It("should not update the static drift hash when provisioner behavior/dynamic field is updated", func() {
+		ExpectApplied(ctx, env.Client, nodeTemplate)
+		ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate))
+		nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate)
+
+		expectedHash := nodeTemplate.Hash()
+		Expect(nodeTemplate.ObjectMeta.Annotations[v1alpha1.AnnotationNodeTemplateHash]).To(Equal(expectedHash))
+
+		nodeTemplate.Spec.SubnetSelector = map[string]string{"subnet-test-key": "subnet-test-value"}
+		nodeTemplate.Spec.SecurityGroupSelector = map[string]string{"sg-test-key": "sg-test-value"}
+		nodeTemplate.Spec.AMIFamily = aws.String("test-family")
+		nodeTemplate.Spec.AMISelector = map[string]string{"ami-test-key": "ami-test-value"}
+
+		ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate))
+		nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate)
+
+		Expect(nodeTemplate.ObjectMeta.Annotations[v1alpha1.AnnotationNodeTemplateHash]).To(Equal(expectedHash))
+	})
+})

pkg/controllers/nodetemplate/status.go

@@ -0,0 +1,112 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nodetemplate
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/karpenter/pkg/apis/v1alpha1"
+	"github.com/aws/karpenter/pkg/providers/amifamily"
+	"github.com/aws/karpenter/pkg/providers/securitygroup"
+	"github.com/aws/karpenter/pkg/providers/subnet"
+	"github.com/samber/lo"
+	"go.uber.org/multierr"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+type Status struct {
+	subnetProvider        *subnet.Provider
+	securityGroupProvider *securitygroup.Provider
+	amiProvider           *amifamily.Provider
+}
+
+func (s *Status) Reconcile(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) (reconcile.Result, error) {
+	err := multierr.Combine(
+		s.resolveSubnets(ctx, nodeTemplate),
+		s.resolveSecurityGroups(ctx, nodeTemplate),
+		s.resolveAMIs(ctx, nodeTemplate),
+	)
+
+	return reconcile.Result{RequeueAfter: 5 * time.Minute}, err
+}
+
+func (c *Status) resolveSubnets(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) error {
+	subnetList, err := c.subnetProvider.List(ctx, nodeTemplate)
+	if err != nil {
+		return err
+	}
+	if len(subnetList) == 0 {
+		nodeTemplate.Status.Subnets = nil
+		return fmt.Errorf("no subnets exist given constraints")
+	}
+
+	sort.Slice(subnetList, func(i, j int) bool {
+		return int(*subnetList[i].AvailableIpAddressCount) > int(*subnetList[j].AvailableIpAddressCount)
+	})
+
+	nodeTemplate.Status.Subnets = lo.Map(subnetList, func(ec2subnet *ec2.Subnet, _ int) v1alpha1.Subnet {
+		return v1alpha1.Subnet{
+			ID:   *ec2subnet.SubnetId,
+			Zone: *ec2subnet.AvailabilityZone,
+		}
+	})
+
+	return nil
+}
+
+func (c *Status) resolveSecurityGroups(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) error {
+	securityGroups, err := c.securityGroupProvider.List(ctx, nodeTemplate)
+	if err != nil {
+		return err
+	}
+	if len(securityGroups) == 0 && nodeTemplate.Spec.SecurityGroupSelector != nil {
+		nodeTemplate.Status.SecurityGroups = nil
+		return fmt.Errorf("no security groups exist given constraints")
+	}
+
+	nodeTemplate.Status.SecurityGroups = lo.Map(securityGroups, func(securityGroup *ec2.SecurityGroup, _ int) v1alpha1.SecurityGroup {
+		return v1alpha1.SecurityGroup{
+			ID:   *securityGroup.GroupId,
+			Name: *securityGroup.GroupName,
+		}
+	})
+
+	return nil
+}
+
+func (c *Status) resolveAMIs(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) error {
+	amis, err := c.amiProvider.Get(ctx, nodeTemplate, &amifamily.Options{})
+	if err != nil {
+		return err
+	}
+	if len(amis) == 0 {
+		nodeTemplate.Status.AMIs = nil
+		return fmt.Errorf("no amis exist given constraints")
+	}
+
+	nodeTemplate.Status.AMIs = lo.Map(amis, func(ami amifamily.AMI, _ int) v1alpha1.AMI {
+		return v1alpha1.AMI{
+			Name:         ami.Name,
+			ID:           ami.AmiID,
+			Requirements: ami.Requirements.NodeSelectorRequirements(),
+		}
+	})
+
+	return nil
+}