From 2df1b5d3f5d2a1b569ea63fc04a3df5f73f95393 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Wed, 19 Jul 2023 17:46:14 -0700 Subject: [PATCH] Update Controller to Include Hash --- charts/karpenter/templates/clusterrole.yaml | 2 +- pkg/controllers/nodetemplate/controller.go | 120 +--- pkg/controllers/nodetemplate/hash.go | 32 + pkg/controllers/nodetemplate/hash_test.go | 61 ++ pkg/controllers/nodetemplate/status.go | 112 ++++ pkg/controllers/nodetemplate/status_test.go | 668 ++++++++++++++++++++ pkg/controllers/nodetemplate/suite_test.go | 644 +------------------ 7 files changed, 908 insertions(+), 731 deletions(-) create mode 100644 pkg/controllers/nodetemplate/hash.go create mode 100644 pkg/controllers/nodetemplate/hash_test.go create mode 100644 pkg/controllers/nodetemplate/status.go create mode 100644 pkg/controllers/nodetemplate/status_test.go diff --git a/charts/karpenter/templates/clusterrole.yaml b/charts/karpenter/templates/clusterrole.yaml index 6b506e32b60f..92146277405e 100644 --- a/charts/karpenter/templates/clusterrole.yaml +++ b/charts/karpenter/templates/clusterrole.yaml @@ -42,5 +42,5 @@ rules: resourceNames: ["defaulting.webhook.karpenter.k8s.aws"] # Write - apiGroups: ["karpenter.k8s.aws"] - resources: ["awsnodetemplates/status"] + resources: ["awsnodetemplates", "awsnodetemplates/status"] verbs: ["patch", "update"] diff --git a/pkg/controllers/nodetemplate/controller.go b/pkg/controllers/nodetemplate/controller.go index 251057364959..a0c7f83c03af 100644 --- a/pkg/controllers/nodetemplate/controller.go +++ b/pkg/controllers/nodetemplate/controller.go @@ -16,12 +16,11 @@ package nodetemplate import ( "context" - "fmt" - "sort" "time" "go.uber.org/multierr" "golang.org/x/time/rate" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/client-go/util/workqueue" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -30,48 +29,60 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/samber/lo" - corecontroller "github.com/aws/karpenter-core/pkg/operator/controller" + "github.com/aws/karpenter-core/pkg/utils/result" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter/pkg/providers/securitygroup" "github.com/aws/karpenter/pkg/providers/subnet" ) +type nodeTemplateReconciler interface { + Reconcile(context.Context, *v1alpha1.AWSNodeTemplate) (reconcile.Result, error) +} + var _ corecontroller.TypedController[*v1alpha1.AWSNodeTemplate] = (*Controller)(nil) type Controller struct { - kubeClient client.Client - subnetProvider *subnet.Provider - securityGroupProvider *securitygroup.Provider - amiProvider *amifamily.Provider + kubeClient client.Client + status *Status + hash *Hash } -func NewController(kubeClient client.Client, subnetProvider *subnet.Provider, securityGroups *securitygroup.Provider, amiprovider *amifamily.Provider) corecontroller.Controller { +func NewController(kubeClient client.Client, subnetProvider *subnet.Provider, securityGroupProvider *securitygroup.Provider, amiProvider *amifamily.Provider) corecontroller.Controller { return corecontroller.Typed[*v1alpha1.AWSNodeTemplate](kubeClient, &Controller{ - kubeClient: kubeClient, - subnetProvider: subnetProvider, - securityGroupProvider: securityGroups, - amiProvider: amiprovider, + kubeClient: kubeClient, + status: &Status{subnetProvider: subnetProvider, securityGroupProvider: securityGroupProvider, amiProvider: amiProvider}, + hash: &Hash{}, }) } func (c *Controller) Reconcile(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) (reconcile.Result, error) { stored := nodeTemplate.DeepCopy() - err := multierr.Combine( - c.resolveSubnets(ctx, nodeTemplate), - c.resolveSecurityGroups(ctx, nodeTemplate), - c.resolveAMIs(ctx, nodeTemplate), - ) + var results []reconcile.Result + var errs error + reconcilers := []nodeTemplateReconciler{ + c.status, + c.hash, + } + for _, reconciler := range reconcilers { + res, err := reconciler.Reconcile(ctx, nodeTemplate) + errs = multierr.Append(errs, err) + results = append(results, res) + } - if patchErr := c.kubeClient.Status().Patch(ctx, nodeTemplate, client.MergeFrom(stored)); patchErr != nil { - err = multierr.Append(err, client.IgnoreNotFound(patchErr)) + if !equality.Semantic.DeepEqual(stored, nodeTemplate) { + statusCopy := nodeTemplate.DeepCopy() + if patchErr := c.kubeClient.Patch(ctx, nodeTemplate, client.MergeFrom(stored)); patchErr != nil { + errs = multierr.Append(errs, patchErr) + } + if patchErr := c.kubeClient.Status().Patch(ctx, statusCopy, client.MergeFrom(stored)); patchErr != nil { + errs = multierr.Append(errs, patchErr) + } } - return reconcile.Result{RequeueAfter: 5 * time.Minute}, err + return result.Min(results...), errs } func (c *Controller) Name() string { @@ -92,68 +103,3 @@ func (c *Controller) Builder(_ context.Context, m manager.Manager) corecontrolle MaxConcurrentReconciles: 10, })) } - -func (c *Controller) resolveSubnets(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) error { - subnetList, err := c.subnetProvider.List(ctx, nodeTemplate) - if err != nil { - return err - } - if len(subnetList) == 0 { - nodeTemplate.Status.Subnets = nil - return fmt.Errorf("no subnets exist given constraints") - } - - sort.Slice(subnetList, func(i, j int) bool { - return int(*subnetList[i].AvailableIpAddressCount) > int(*subnetList[j].AvailableIpAddressCount) - }) - - nodeTemplate.Status.Subnets = lo.Map(subnetList, func(ec2subnet *ec2.Subnet, _ int) v1alpha1.Subnet { - return v1alpha1.Subnet{ - ID: *ec2subnet.SubnetId, - Zone: *ec2subnet.AvailabilityZone, - } - }) - - return nil -} - -func (c *Controller) resolveSecurityGroups(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) error { - securityGroups, err := c.securityGroupProvider.List(ctx, nodeTemplate) - if err != nil { - return err - } - if len(securityGroups) == 0 && nodeTemplate.Spec.SecurityGroupSelector != nil { - nodeTemplate.Status.SecurityGroups = nil - return fmt.Errorf("no security groups exist given constraints") - } - - nodeTemplate.Status.SecurityGroups = lo.Map(securityGroups, func(securityGroup *ec2.SecurityGroup, _ int) v1alpha1.SecurityGroup { - return v1alpha1.SecurityGroup{ - ID: *securityGroup.GroupId, - Name: *securityGroup.GroupName, - } - }) - - return nil -} - -func (c *Controller) resolveAMIs(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) error { - amis, err := c.amiProvider.Get(ctx, nodeTemplate, &amifamily.Options{}) - if err != nil { - return err - } - if len(amis) == 0 { - nodeTemplate.Status.AMIs = nil - return fmt.Errorf("no amis exist given constraints") - } - - nodeTemplate.Status.AMIs = lo.Map(amis, func(ami amifamily.AMI, _ int) v1alpha1.AMI { - return v1alpha1.AMI{ - Name: ami.Name, - ID: ami.AmiID, - Requirements: ami.Requirements.NodeSelectorRequirements(), - } - }) - - return nil -} diff --git a/pkg/controllers/nodetemplate/hash.go b/pkg/controllers/nodetemplate/hash.go new file mode 100644 index 000000000000..589c7bcbfdc3 --- /dev/null +++ b/pkg/controllers/nodetemplate/hash.go @@ -0,0 +1,32 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodetemplate + +import ( + "context" + + "github.com/aws/karpenter/pkg/apis/v1alpha1" + "github.com/samber/lo" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +type Hash struct{} + +func (s *Hash) Reconcile(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) (reconcile.Result, error) { + nodeTemplateHash := nodeTemplate.Hash() + nodeTemplate.Annotations = lo.Assign(nodeTemplate.ObjectMeta.Annotations, map[string]string{v1alpha1.AnnotationNodeTemplateHash: nodeTemplateHash}) + + return reconcile.Result{}, nil +} diff --git a/pkg/controllers/nodetemplate/hash_test.go b/pkg/controllers/nodetemplate/hash_test.go new file mode 100644 index 000000000000..617963c04088 --- /dev/null +++ b/pkg/controllers/nodetemplate/hash_test.go @@ -0,0 +1,61 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodetemplate_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/aws/aws-sdk-go/aws" + . "github.com/aws/karpenter-core/pkg/test/expectations" + "github.com/aws/karpenter/pkg/apis/v1alpha1" +) + +var _ = Describe("AWSNodeTemplate Static Drift Hash", func() { + It("should update the static drift hash when provisioner static field is updated", func() { + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + + expectedHash := nodeTemplate.Hash() + Expect(nodeTemplate.ObjectMeta.Annotations[v1alpha1.AnnotationNodeTemplateHash]).To(Equal(expectedHash)) + + nodeTemplate.Spec.UserData = aws.String("test-userData-2") + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + + expectedHashTwo := nodeTemplate.Hash() + Expect(nodeTemplate.ObjectMeta.Annotations[v1alpha1.AnnotationNodeTemplateHash]).To(Equal(expectedHashTwo)) + }) + It("should not update the static drift hash when provisioner behavior/dynamic field is updated", func() { + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + + expectedHash := nodeTemplate.Hash() + Expect(nodeTemplate.ObjectMeta.Annotations[v1alpha1.AnnotationNodeTemplateHash]).To(Equal(expectedHash)) + + nodeTemplate.Spec.SubnetSelector = map[string]string{"subnet-test-key": "subnet-test-value"} + nodeTemplate.Spec.SecurityGroupSelector = map[string]string{"sg-test-key": "sg-test-value"} + nodeTemplate.Spec.AMIFamily = aws.String("test-family") + nodeTemplate.Spec.AMISelector = map[string]string{"ami-test-key": "ami-test-value"} + + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + + Expect(nodeTemplate.ObjectMeta.Annotations[v1alpha1.AnnotationNodeTemplateHash]).To(Equal(expectedHash)) + }) +}) diff --git a/pkg/controllers/nodetemplate/status.go b/pkg/controllers/nodetemplate/status.go new file mode 100644 index 000000000000..d19273798316 --- /dev/null +++ b/pkg/controllers/nodetemplate/status.go @@ -0,0 +1,112 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodetemplate + +import ( + "context" + "fmt" + "sort" + "time" + + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/karpenter/pkg/apis/v1alpha1" + "github.com/aws/karpenter/pkg/providers/amifamily" + "github.com/aws/karpenter/pkg/providers/securitygroup" + "github.com/aws/karpenter/pkg/providers/subnet" + "github.com/samber/lo" + "go.uber.org/multierr" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +type Status struct { + subnetProvider *subnet.Provider + securityGroupProvider *securitygroup.Provider + amiProvider *amifamily.Provider +} + +func (s *Status) Reconcile(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) (reconcile.Result, error) { + err := multierr.Combine( + s.resolveSubnets(ctx, nodeTemplate), + s.resolveSecurityGroups(ctx, nodeTemplate), + s.resolveAMIs(ctx, nodeTemplate), + ) + + return reconcile.Result{RequeueAfter: 5 * time.Minute}, err +} + +func (c *Status) resolveSubnets(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) error { + subnetList, err := c.subnetProvider.List(ctx, nodeTemplate) + if err != nil { + return err + } + if len(subnetList) == 0 { + nodeTemplate.Status.Subnets = nil + return fmt.Errorf("no subnets exist given constraints") + } + + sort.Slice(subnetList, func(i, j int) bool { + return int(*subnetList[i].AvailableIpAddressCount) > int(*subnetList[j].AvailableIpAddressCount) + }) + + nodeTemplate.Status.Subnets = lo.Map(subnetList, func(ec2subnet *ec2.Subnet, _ int) v1alpha1.Subnet { + return v1alpha1.Subnet{ + ID: *ec2subnet.SubnetId, + Zone: *ec2subnet.AvailabilityZone, + } + }) + + return nil +} + +func (c *Status) resolveSecurityGroups(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) error { + securityGroups, err := c.securityGroupProvider.List(ctx, nodeTemplate) + if err != nil { + return err + } + if len(securityGroups) == 0 && nodeTemplate.Spec.SecurityGroupSelector != nil { + nodeTemplate.Status.SecurityGroups = nil + return fmt.Errorf("no security groups exist given constraints") + } + + nodeTemplate.Status.SecurityGroups = lo.Map(securityGroups, func(securityGroup *ec2.SecurityGroup, _ int) v1alpha1.SecurityGroup { + return v1alpha1.SecurityGroup{ + ID: *securityGroup.GroupId, + Name: *securityGroup.GroupName, + } + }) + + return nil +} + +func (c *Status) resolveAMIs(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) error { + amis, err := c.amiProvider.Get(ctx, nodeTemplate, &amifamily.Options{}) + if err != nil { + return err + } + if len(amis) == 0 { + nodeTemplate.Status.AMIs = nil + return fmt.Errorf("no amis exist given constraints") + } + + nodeTemplate.Status.AMIs = lo.Map(amis, func(ami amifamily.AMI, _ int) v1alpha1.AMI { + return v1alpha1.AMI{ + Name: ami.Name, + ID: ami.AmiID, + Requirements: ami.Requirements.NodeSelectorRequirements(), + } + }) + + return nil +} diff --git a/pkg/controllers/nodetemplate/status_test.go b/pkg/controllers/nodetemplate/status_test.go new file mode 100644 index 000000000000..696563a753f1 --- /dev/null +++ b/pkg/controllers/nodetemplate/status_test.go @@ -0,0 +1,668 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodetemplate_test + +import ( + "fmt" + "sort" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" + _ "knative.dev/pkg/system/testing" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + . "github.com/aws/karpenter-core/pkg/test/expectations" + + "github.com/aws/karpenter/pkg/apis/v1alpha1" +) + +var _ = Describe("AWSNodeTemplate Status", func() { + Context("Subnet Status", func() { + It("Should update AWSNodeTemplate status for Subnets", func() { + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.Subnets).To(Equal([]v1alpha1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + }, + })) + }) + It("Should have the correct ordering for the Subnets", func() { + awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{ + {SubnetId: aws.String("subnet-test1"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(20)}, + {SubnetId: aws.String("subnet-test2"), AvailabilityZone: aws.String("test-zone-1b"), AvailableIpAddressCount: aws.Int64(100)}, + {SubnetId: aws.String("subnet-test3"), AvailabilityZone: aws.String("test-zone-1c"), AvailableIpAddressCount: aws.Int64(50)}, + }}) + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.Subnets).To(Equal([]v1alpha1.Subnet{ + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + }, + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + })) + }) + It("Should resolve a valid selectors for Subnet by tags", func() { + nodeTemplate.Spec.SubnetSelector = map[string]string{`Name`: `test-subnet-1,test-subnet-2`} + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.Subnets).To(Equal([]v1alpha1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + })) + }) + It("Should resolve a valid selectors for Subnet by ids", func() { + nodeTemplate.Spec.SubnetSelector = map[string]string{`aws-ids`: `subnet-test1`} + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.Subnets).To(Equal([]v1alpha1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + })) + }) + It("Should update Subnet status when the Subnet selector gets updated by tags", func() { + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.Subnets).To(Equal([]v1alpha1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + }, + })) + + nodeTemplate.Spec.SubnetSelector = map[string]string{`Name`: `test-subnet-1,test-subnet-2`} + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.Subnets).To(Equal([]v1alpha1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + })) + }) + It("Should update Subnet status when the Subnet selector gets updated by ids", func() { + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.Subnets).To(Equal([]v1alpha1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + }, + })) + + nodeTemplate.Spec.SubnetSelector = map[string]string{`aws-ids`: `subnet-test1`} + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.Subnets).To(Equal([]v1alpha1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + })) + }) + It("Should not resolve a invalid selectors for Subnet", func() { + nodeTemplate.Spec.SubnetSelector = map[string]string{`foo`: `invalid`} + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileFailed(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.Subnets).To(BeNil()) + }) + It("Should not resolve a invalid selectors for an updated Subnet selectors", func() { + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.Subnets).To(Equal([]v1alpha1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + }, + })) + + nodeTemplate.Spec.SubnetSelector = map[string]string{`foo`: `invalid`} + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileFailed(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.Subnets).To(BeNil()) + }) + }) + Context("Security Groups Status", func() { + It("Should expect no errors when security groups are not in the AWSNodeTemplate", func() { + // TODO: Remove test for v1beta1, as security groups will be required + nodeTemplate.Spec.SecurityGroupSelector = nil + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + Expect(nodeTemplate.Status.SecurityGroups).To(BeNil()) + }) + It("Should update AWSNodeTemplate status for Security Groups", func() { + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.SecurityGroups).To(Equal([]v1alpha1.SecurityGroup{ + { + ID: "sg-test1", + Name: "securityGroup-test1", + }, + { + ID: "sg-test2", + Name: "securityGroup-test2", + }, + { + ID: "sg-test3", + Name: "securityGroup-test3", + }, + })) + }) + It("Should resolve a valid selectors for Security Groups by tags", func() { + nodeTemplate.Spec.SecurityGroupSelector = map[string]string{`Name`: `test-security-group-1,test-security-group-2`} + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.SecurityGroups).To(Equal([]v1alpha1.SecurityGroup{ + { + ID: "sg-test1", + Name: "securityGroup-test1", + }, + { + ID: "sg-test2", + Name: "securityGroup-test2", + }, + })) + }) + It("Should resolve a valid selectors for Security Groups by ids", func() { + nodeTemplate.Spec.SecurityGroupSelector = map[string]string{`aws-ids`: `sg-test1`} + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.SecurityGroups).To(Equal([]v1alpha1.SecurityGroup{ + { + ID: "sg-test1", + Name: "securityGroup-test1", + }, + })) + }) + It("Should update Security Groups status when the Security Groups selector gets updated by tags", func() { + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.SecurityGroups).To(Equal([]v1alpha1.SecurityGroup{ + { + ID: "sg-test1", + Name: "securityGroup-test1", + }, + { + ID: "sg-test2", + Name: "securityGroup-test2", + }, + { + ID: "sg-test3", + Name: "securityGroup-test3", + }, + })) + + nodeTemplate.Spec.SecurityGroupSelector = map[string]string{`Name`: `test-security-group-1,test-security-group-2`} + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.SecurityGroups).To(Equal([]v1alpha1.SecurityGroup{ + { + ID: "sg-test1", + Name: "securityGroup-test1", + }, + { + ID: "sg-test2", + Name: "securityGroup-test2", + }, + })) + }) + It("Should update Security Groups status when the Security Groups selector gets updated by ids", func() { + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.SecurityGroups).To(Equal([]v1alpha1.SecurityGroup{ + { + ID: "sg-test1", + Name: "securityGroup-test1", + }, + { + ID: "sg-test2", + Name: "securityGroup-test2", + }, + { + ID: "sg-test3", + Name: "securityGroup-test3", + }, + })) + + nodeTemplate.Spec.SecurityGroupSelector = map[string]string{`aws-ids`: `sg-test1`} + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.SecurityGroups).To(Equal([]v1alpha1.SecurityGroup{ + { + ID: "sg-test1", + Name: "securityGroup-test1", + }, + })) + }) + It("Should not resolve a invalid selectors for Security Groups", func() { + nodeTemplate.Spec.SecurityGroupSelector = map[string]string{`foo`: `invalid`} + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileFailed(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.SecurityGroups).To(BeNil()) + }) + It("Should not resolve a invalid selectors for an updated Security Groups selector", func() { + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.SecurityGroups).To(Equal([]v1alpha1.SecurityGroup{ + { + ID: "sg-test1", + Name: "securityGroup-test1", + }, + { + ID: "sg-test2", + Name: "securityGroup-test2", + }, + { + ID: "sg-test3", + Name: "securityGroup-test3", + }, + })) + + nodeTemplate.Spec.SecurityGroupSelector = map[string]string{`foo`: `invalid`} + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileFailed(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.SecurityGroups).To(BeNil()) + }) + }) + Context("AMI Status", func() { + BeforeEach(func() { + awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{ + Images: []*ec2.Image{ + { + Name: aws.String("test-ami-1"), + ImageId: aws.String("ami-test1"), + CreationDate: aws.String(time.Now().Format(time.RFC3339)), + Architecture: aws.String("x86_64"), + Tags: []*ec2.Tag{ + {Key: aws.String("Name"), Value: aws.String("test-ami-1")}, + {Key: aws.String("foo"), Value: aws.String("bar")}, + }, + }, + { + Name: aws.String("test-ami-2"), + ImageId: aws.String("ami-test2"), + CreationDate: aws.String(time.Now().Add(time.Minute).Format(time.RFC3339)), + Architecture: aws.String("x86_64"), + Tags: []*ec2.Tag{ + {Key: aws.String("Name"), Value: aws.String("test-ami-2")}, + {Key: aws.String("foo"), Value: aws.String("bar")}, + }, + }, + { + Name: aws.String("test-ami-3"), + ImageId: aws.String("ami-test3"), + CreationDate: aws.String(time.Now().Add(2 * time.Minute).Format(time.RFC3339)), + Architecture: aws.String("x86_64"), + Tags: []*ec2.Tag{ + {Key: aws.String("Name"), Value: aws.String("test-ami-3")}, + {Key: aws.String("foo"), Value: aws.String("bar")}, + }, + }, + }, + }) + }) + It("should resolve amiSelector AMIs and requirements into status", func() { + version := lo.Must(awsEnv.AMIProvider.KubeServerVersion(ctx)) + + awsEnv.SSMAPI.Parameters = map[string]string{ + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", version): "ami-id-123", + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-gpu/recommended/image_id", version): "ami-id-456", + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, fmt.Sprintf("-%s", v1alpha5.ArchitectureArm64)): "ami-id-789", + } + + awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{ + Images: []*ec2.Image{ + { + Name: aws.String("test-ami-1"), + ImageId: aws.String("ami-id-123"), + CreationDate: aws.String(time.Now().Format(time.RFC3339)), + Architecture: aws.String("x86_64"), + Tags: []*ec2.Tag{ + {Key: aws.String("Name"), Value: aws.String("test-ami-1")}, + {Key: aws.String("foo"), Value: aws.String("bar")}, + }, + }, + { + Name: aws.String("test-ami-2"), + ImageId: aws.String("ami-id-456"), + CreationDate: aws.String(time.Now().Add(time.Minute).Format(time.RFC3339)), + Architecture: aws.String("x86_64"), + Tags: []*ec2.Tag{ + {Key: aws.String("Name"), Value: aws.String("test-ami-2")}, + {Key: aws.String("foo"), Value: aws.String("bar")}, + }, + }, + { + Name: aws.String("test-ami-3"), + ImageId: aws.String("ami-id-789"), + CreationDate: aws.String(time.Now().Add(2 * time.Minute).Format(time.RFC3339)), + Architecture: aws.String("x86_64"), + Tags: []*ec2.Tag{ + {Key: aws.String("Name"), Value: aws.String("test-ami-3")}, + {Key: aws.String("foo"), Value: aws.String("bar")}, + }, + }, + }, + }) + nodeTemplate.Spec.AMISelector = nil + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + sortRequirements(nodeTemplate.Status.AMIs) + Expect(nodeTemplate.Status.AMIs).To(ContainElements([]v1alpha1.AMI{ + { + Name: "test-ami-1", + ID: "ami-id-123", + Requirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureAmd64}, + }, + { + Key: v1alpha1.LabelInstanceGPUCount, + Operator: v1.NodeSelectorOpDoesNotExist, + }, + { + Key: v1alpha1.LabelInstanceAcceleratorCount, + Operator: v1.NodeSelectorOpDoesNotExist, + }, + }, + }, + { + Name: "test-ami-3", + ID: "ami-id-789", + Requirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureArm64}, + }, + { + Key: v1alpha1.LabelInstanceGPUCount, + Operator: v1.NodeSelectorOpDoesNotExist, + }, + { + Key: v1alpha1.LabelInstanceAcceleratorCount, + Operator: v1.NodeSelectorOpDoesNotExist, + }, + }, + }, + { + Name: "test-ami-2", + ID: "ami-id-456", + Requirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureAmd64}, + }, + { + Key: v1alpha1.LabelInstanceGPUCount, + Operator: v1.NodeSelectorOpExists, + }, + }, + }, + { + Name: "test-ami-2", + ID: "ami-id-456", + Requirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureAmd64}, + }, + { + Key: v1alpha1.LabelInstanceAcceleratorCount, + Operator: v1.NodeSelectorOpExists, + }, + }, + }, + })) + }) + It("should resolve amiSelector AMis and requirements into status when all SSM aliases don't resolve", func() { + version := lo.Must(awsEnv.AMIProvider.KubeServerVersion(ctx)) + // This parameter set doesn't include any of the Nvidia AMIs + awsEnv.SSMAPI.Parameters = map[string]string{ + fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/x86_64/latest/image_id", version): "ami-id-123", + fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/arm64/latest/image_id", version): "ami-id-456", + } + nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket + nodeTemplate.Spec.AMISelector = nil + awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{ + Images: []*ec2.Image{ + { + Name: aws.String("test-ami-1"), + ImageId: aws.String("ami-id-123"), + CreationDate: aws.String(time.Now().Format(time.RFC3339)), + Architecture: aws.String("x86_64"), + Tags: []*ec2.Tag{ + {Key: aws.String("Name"), Value: aws.String("test-ami-1")}, + {Key: aws.String("foo"), Value: aws.String("bar")}, + }, + }, + { + Name: aws.String("test-ami-2"), + ImageId: aws.String("ami-id-456"), + CreationDate: aws.String(time.Now().Add(time.Minute).Format(time.RFC3339)), + Architecture: aws.String("arm64"), + Tags: []*ec2.Tag{ + {Key: aws.String("Name"), Value: aws.String("test-ami-2")}, + {Key: aws.String("foo"), Value: aws.String("bar")}, + }, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + sortRequirements(nodeTemplate.Status.AMIs) + Expect(nodeTemplate.Status.AMIs).To(ContainElements([]v1alpha1.AMI{ + { + Name: "test-ami-1", + ID: "ami-id-123", + Requirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureAmd64}, + }, + { + Key: v1alpha1.LabelInstanceGPUCount, + Operator: v1.NodeSelectorOpDoesNotExist, + }, + { + Key: v1alpha1.LabelInstanceAcceleratorCount, + Operator: v1.NodeSelectorOpDoesNotExist, + }, + }, + }, + { + Name: "test-ami-2", + ID: "ami-id-456", + Requirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureArm64}, + }, + { + Key: v1alpha1.LabelInstanceGPUCount, + Operator: v1.NodeSelectorOpDoesNotExist, + }, + { + Key: v1alpha1.LabelInstanceAcceleratorCount, + Operator: v1.NodeSelectorOpDoesNotExist, + }, + }, + }, + })) + }) + It("Should resolve a valid AMI selector", func() { + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + Expect(nodeTemplate.Status.AMIs).To(ContainElements( + []v1alpha1.AMI{ + { + Name: "test-ami-3", + ID: "ami-test3", + Requirements: []v1.NodeSelectorRequirement{ + { + Key: "kubernetes.io/arch", + Operator: "In", + Values: []string{ + "amd64", + }, + }, + }, + }, + }, + )) + }) + It("should resolve amiSelector AMIs that have well-known tags as AMI requirements into status", func() { + awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{ + Images: []*ec2.Image{ + { + Name: aws.String("test-ami-4"), + ImageId: aws.String("ami-test4"), + CreationDate: aws.String(time.Now().Add(2 * time.Minute).Format(time.RFC3339)), + Architecture: aws.String("x86_64"), + Tags: []*ec2.Tag{ + {Key: aws.String("Name"), Value: aws.String("test-ami-3")}, + {Key: aws.String("foo"), Value: aws.String("bar")}, + {Key: aws.String("kubernetes.io/os"), Value: aws.String("test-requirement-1")}, + }, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) + nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) + sortRequirements(nodeTemplate.Status.AMIs) + Expect(nodeTemplate.Status.AMIs).To(ContainElements([]v1alpha1.AMI{ + { + Name: "test-ami-4", + ID: "ami-test4", + Requirements: []v1.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: "In", + Values: []string{ + "test-requirement-1", + }, + }, + { + Key: "kubernetes.io/arch", + Operator: "In", + Values: []string{ + "amd64", + }, + }, + }, + }, + }, + )) + }) + }) +}) + +func sortRequirements(amis []v1alpha1.AMI) { + for i := range amis { + sort.Slice(amis[i].Requirements, func(p, q int) bool { + return amis[i].Requirements[p].Key > amis[i].Requirements[q].Key + }) + } +} diff --git a/pkg/controllers/nodetemplate/suite_test.go b/pkg/controllers/nodetemplate/suite_test.go index c1c43222103c..f5adb61b9f56 100644 --- a/pkg/controllers/nodetemplate/suite_test.go +++ b/pkg/controllers/nodetemplate/suite_test.go @@ -16,24 +16,15 @@ package nodetemplate_test import ( "context" - "fmt" - "sort" "testing" - "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/samber/lo" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "knative.dev/pkg/logging/testing" _ "knative.dev/pkg/system/testing" - "sigs.k8s.io/controller-runtime/pkg/client" coresettings "github.com/aws/karpenter-core/pkg/apis/settings" - "github.com/aws/karpenter-core/pkg/apis/v1alpha5" corecontroller "github.com/aws/karpenter-core/pkg/operator/controller" "github.com/aws/karpenter-core/pkg/operator/injection" "github.com/aws/karpenter-core/pkg/operator/options" @@ -58,7 +49,7 @@ var controller corecontroller.Controller func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) RegisterFailHandler(Fail) - RunSpecs(t, "AWSNodeTemplateController") + RunSpecs(t, "AWSNodeTemplate") } var _ = BeforeSuite(func() { @@ -96,636 +87,3 @@ var _ = BeforeEach(func() { var _ = AfterEach(func() { ExpectCleanedUp(ctx, env.Client) }) - -var _ = Describe("AWSNodeTemplateController", func() { - Context("Subnet Status", func() { - It("Should update AWSNodeTemplate status for Subnets", func() { - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.Subnets).To(Equal([]v1alpha1.Subnet{ - { - ID: "subnet-test1", - Zone: "test-zone-1a", - }, - { - ID: "subnet-test2", - Zone: "test-zone-1b", - }, - { - ID: "subnet-test3", - Zone: "test-zone-1c", - }, - })) - }) - It("Should have the correct ordering for the Subnets", func() { - awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{ - {SubnetId: aws.String("subnet-test1"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(20)}, - {SubnetId: aws.String("subnet-test2"), AvailabilityZone: aws.String("test-zone-1b"), AvailableIpAddressCount: aws.Int64(100)}, - {SubnetId: aws.String("subnet-test3"), AvailabilityZone: aws.String("test-zone-1c"), AvailableIpAddressCount: aws.Int64(50)}, - }}) - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.Subnets).To(Equal([]v1alpha1.Subnet{ - { - ID: "subnet-test2", - Zone: "test-zone-1b", - }, - { - ID: "subnet-test3", - Zone: "test-zone-1c", - }, - { - ID: "subnet-test1", - Zone: "test-zone-1a", - }, - })) - }) - It("Should resolve a valid selectors for Subnet by tags", func() { - nodeTemplate.Spec.SubnetSelector = map[string]string{`Name`: `test-subnet-1,test-subnet-2`} - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.Subnets).To(Equal([]v1alpha1.Subnet{ - { - ID: "subnet-test1", - Zone: "test-zone-1a", - }, - { - ID: "subnet-test2", - Zone: "test-zone-1b", - }, - })) - }) - It("Should resolve a valid selectors for Subnet by ids", func() { - nodeTemplate.Spec.SubnetSelector = map[string]string{`aws-ids`: `subnet-test1`} - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.Subnets).To(Equal([]v1alpha1.Subnet{ - { - ID: "subnet-test1", - Zone: "test-zone-1a", - }, - })) - }) - It("Should update Subnet status when the Subnet selector gets updated by tags", func() { - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.Subnets).To(Equal([]v1alpha1.Subnet{ - { - ID: "subnet-test1", - Zone: "test-zone-1a", - }, - { - ID: "subnet-test2", - Zone: "test-zone-1b", - }, - { - ID: "subnet-test3", - Zone: "test-zone-1c", - }, - })) - - nodeTemplate.Spec.SubnetSelector = map[string]string{`Name`: `test-subnet-1,test-subnet-2`} - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.Subnets).To(Equal([]v1alpha1.Subnet{ - { - ID: "subnet-test1", - Zone: "test-zone-1a", - }, - { - ID: "subnet-test2", - Zone: "test-zone-1b", - }, - })) - }) - It("Should update Subnet status when the Subnet selector gets updated by ids", func() { - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.Subnets).To(Equal([]v1alpha1.Subnet{ - { - ID: "subnet-test1", - Zone: "test-zone-1a", - }, - { - ID: "subnet-test2", - Zone: "test-zone-1b", - }, - { - ID: "subnet-test3", - Zone: "test-zone-1c", - }, - })) - - nodeTemplate.Spec.SubnetSelector = map[string]string{`aws-ids`: `subnet-test1`} - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.Subnets).To(Equal([]v1alpha1.Subnet{ - { - ID: "subnet-test1", - Zone: "test-zone-1a", - }, - })) - }) - It("Should not resolve a invalid selectors for Subnet", func() { - nodeTemplate.Spec.SubnetSelector = map[string]string{`foo`: `invalid`} - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileFailed(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.Subnets).To(BeNil()) - }) - It("Should not resolve a invalid selectors for an updated Subnet selectors", func() { - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.Subnets).To(Equal([]v1alpha1.Subnet{ - { - ID: "subnet-test1", - Zone: "test-zone-1a", - }, - { - ID: "subnet-test2", - Zone: "test-zone-1b", - }, - { - ID: "subnet-test3", - Zone: "test-zone-1c", - }, - })) - - nodeTemplate.Spec.SubnetSelector = map[string]string{`foo`: `invalid`} - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileFailed(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.Subnets).To(BeNil()) - }) - }) - Context("Security Groups Status", func() { - It("Should expect no errors when security groups are not in the AWSNodeTemplate", func() { - // TODO: Remove test for v1beta1, as security groups will be required - nodeTemplate.Spec.SecurityGroupSelector = nil - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - Expect(nodeTemplate.Status.SecurityGroups).To(BeNil()) - }) - It("Should update AWSNodeTemplate status for Security Groups", func() { - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.SecurityGroups).To(Equal([]v1alpha1.SecurityGroup{ - { - ID: "sg-test1", - Name: "securityGroup-test1", - }, - { - ID: "sg-test2", - Name: "securityGroup-test2", - }, - { - ID: "sg-test3", - Name: "securityGroup-test3", - }, - })) - }) - It("Should resolve a valid selectors for Security Groups by tags", func() { - nodeTemplate.Spec.SecurityGroupSelector = map[string]string{`Name`: `test-security-group-1,test-security-group-2`} - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.SecurityGroups).To(Equal([]v1alpha1.SecurityGroup{ - { - ID: "sg-test1", - Name: "securityGroup-test1", - }, - { - ID: "sg-test2", - Name: "securityGroup-test2", - }, - })) - }) - It("Should resolve a valid selectors for Security Groups by ids", func() { - nodeTemplate.Spec.SecurityGroupSelector = map[string]string{`aws-ids`: `sg-test1`} - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.SecurityGroups).To(Equal([]v1alpha1.SecurityGroup{ - { - ID: "sg-test1", - Name: "securityGroup-test1", - }, - })) - }) - It("Should update Security Groups status when the Security Groups selector gets updated by tags", func() { - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.SecurityGroups).To(Equal([]v1alpha1.SecurityGroup{ - { - ID: "sg-test1", - Name: "securityGroup-test1", - }, - { - ID: "sg-test2", - Name: "securityGroup-test2", - }, - { - ID: "sg-test3", - Name: "securityGroup-test3", - }, - })) - - nodeTemplate.Spec.SecurityGroupSelector = map[string]string{`Name`: `test-security-group-1,test-security-group-2`} - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.SecurityGroups).To(Equal([]v1alpha1.SecurityGroup{ - { - ID: "sg-test1", - Name: "securityGroup-test1", - }, - { - ID: "sg-test2", - Name: "securityGroup-test2", - }, - })) - }) - It("Should update Security Groups status when the Security Groups selector gets updated by ids", func() { - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.SecurityGroups).To(Equal([]v1alpha1.SecurityGroup{ - { - ID: "sg-test1", - Name: "securityGroup-test1", - }, - { - ID: "sg-test2", - Name: "securityGroup-test2", - }, - { - ID: "sg-test3", - Name: "securityGroup-test3", - }, - })) - - nodeTemplate.Spec.SecurityGroupSelector = map[string]string{`aws-ids`: `sg-test1`} - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.SecurityGroups).To(Equal([]v1alpha1.SecurityGroup{ - { - ID: "sg-test1", - Name: "securityGroup-test1", - }, - })) - }) - It("Should not resolve a invalid selectors for Security Groups", func() { - nodeTemplate.Spec.SecurityGroupSelector = map[string]string{`foo`: `invalid`} - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileFailed(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.SecurityGroups).To(BeNil()) - }) - It("Should not resolve a invalid selectors for an updated Security Groups selector", func() { - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.SecurityGroups).To(Equal([]v1alpha1.SecurityGroup{ - { - ID: "sg-test1", - Name: "securityGroup-test1", - }, - { - ID: "sg-test2", - Name: "securityGroup-test2", - }, - { - ID: "sg-test3", - Name: "securityGroup-test3", - }, - })) - - nodeTemplate.Spec.SecurityGroupSelector = map[string]string{`foo`: `invalid`} - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileFailed(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.SecurityGroups).To(BeNil()) - }) - }) - Context("AMI Status", func() { - BeforeEach(func() { - awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{ - Images: []*ec2.Image{ - { - Name: aws.String("test-ami-1"), - ImageId: aws.String("ami-test1"), - CreationDate: aws.String(time.Now().Format(time.RFC3339)), - Architecture: aws.String("x86_64"), - Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String("test-ami-1")}, - {Key: aws.String("foo"), Value: aws.String("bar")}, - }, - }, - { - Name: aws.String("test-ami-2"), - ImageId: aws.String("ami-test2"), - CreationDate: aws.String(time.Now().Add(time.Minute).Format(time.RFC3339)), - Architecture: aws.String("x86_64"), - Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String("test-ami-2")}, - {Key: aws.String("foo"), Value: aws.String("bar")}, - }, - }, - { - Name: aws.String("test-ami-3"), - ImageId: aws.String("ami-test3"), - CreationDate: aws.String(time.Now().Add(2 * time.Minute).Format(time.RFC3339)), - Architecture: aws.String("x86_64"), - Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String("test-ami-3")}, - {Key: aws.String("foo"), Value: aws.String("bar")}, - }, - }, - }, - }) - }) - It("should resolve amiSelector AMIs and requirements into status", func() { - version := lo.Must(awsEnv.AMIProvider.KubeServerVersion(ctx)) - - awsEnv.SSMAPI.Parameters = map[string]string{ - fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", version): "ami-id-123", - fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-gpu/recommended/image_id", version): "ami-id-456", - fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, fmt.Sprintf("-%s", v1alpha5.ArchitectureArm64)): "ami-id-789", - } - - awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{ - Images: []*ec2.Image{ - { - Name: aws.String("test-ami-1"), - ImageId: aws.String("ami-id-123"), - CreationDate: aws.String(time.Now().Format(time.RFC3339)), - Architecture: aws.String("x86_64"), - Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String("test-ami-1")}, - {Key: aws.String("foo"), Value: aws.String("bar")}, - }, - }, - { - Name: aws.String("test-ami-2"), - ImageId: aws.String("ami-id-456"), - CreationDate: aws.String(time.Now().Add(time.Minute).Format(time.RFC3339)), - Architecture: aws.String("x86_64"), - Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String("test-ami-2")}, - {Key: aws.String("foo"), Value: aws.String("bar")}, - }, - }, - { - Name: aws.String("test-ami-3"), - ImageId: aws.String("ami-id-789"), - CreationDate: aws.String(time.Now().Add(2 * time.Minute).Format(time.RFC3339)), - Architecture: aws.String("x86_64"), - Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String("test-ami-3")}, - {Key: aws.String("foo"), Value: aws.String("bar")}, - }, - }, - }, - }) - nodeTemplate.Spec.AMISelector = nil - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - sortRequirements(nodeTemplate.Status.AMIs) - Expect(nodeTemplate.Status.AMIs).To(ContainElements([]v1alpha1.AMI{ - { - Name: "test-ami-1", - ID: "ami-id-123", - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureAmd64}, - }, - { - Key: v1alpha1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, - { - Key: v1alpha1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, - }, - }, - { - Name: "test-ami-3", - ID: "ami-id-789", - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureArm64}, - }, - { - Key: v1alpha1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, - { - Key: v1alpha1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, - }, - }, - { - Name: "test-ami-2", - ID: "ami-id-456", - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureAmd64}, - }, - { - Key: v1alpha1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpExists, - }, - }, - }, - { - Name: "test-ami-2", - ID: "ami-id-456", - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureAmd64}, - }, - { - Key: v1alpha1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpExists, - }, - }, - }, - })) - }) - It("should resolve amiSelector AMis and requirements into status when all SSM aliases don't resolve", func() { - version := lo.Must(awsEnv.AMIProvider.KubeServerVersion(ctx)) - // This parameter set doesn't include any of the Nvidia AMIs - awsEnv.SSMAPI.Parameters = map[string]string{ - fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/x86_64/latest/image_id", version): "ami-id-123", - fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/arm64/latest/image_id", version): "ami-id-456", - } - nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - nodeTemplate.Spec.AMISelector = nil - awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{ - Images: []*ec2.Image{ - { - Name: aws.String("test-ami-1"), - ImageId: aws.String("ami-id-123"), - CreationDate: aws.String(time.Now().Format(time.RFC3339)), - Architecture: aws.String("x86_64"), - Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String("test-ami-1")}, - {Key: aws.String("foo"), Value: aws.String("bar")}, - }, - }, - { - Name: aws.String("test-ami-2"), - ImageId: aws.String("ami-id-456"), - CreationDate: aws.String(time.Now().Add(time.Minute).Format(time.RFC3339)), - Architecture: aws.String("arm64"), - Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String("test-ami-2")}, - {Key: aws.String("foo"), Value: aws.String("bar")}, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - sortRequirements(nodeTemplate.Status.AMIs) - Expect(nodeTemplate.Status.AMIs).To(ContainElements([]v1alpha1.AMI{ - { - Name: "test-ami-1", - ID: "ami-id-123", - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureAmd64}, - }, - { - Key: v1alpha1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, - { - Key: v1alpha1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, - }, - }, - { - Name: "test-ami-2", - ID: "ami-id-456", - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{v1alpha5.ArchitectureArm64}, - }, - { - Key: v1alpha1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, - { - Key: v1alpha1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, - }, - }, - })) - }) - It("Should resolve a valid AMI selector", func() { - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - Expect(nodeTemplate.Status.AMIs).To(ContainElements( - []v1alpha1.AMI{ - { - Name: "test-ami-3", - ID: "ami-test3", - Requirements: []v1.NodeSelectorRequirement{ - { - Key: "kubernetes.io/arch", - Operator: "In", - Values: []string{ - "amd64", - }, - }, - }, - }, - }, - )) - }) - It("should resolve amiSelector AMIs that have well-known tags as AMI requirements into status", func() { - awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{ - Images: []*ec2.Image{ - { - Name: aws.String("test-ami-4"), - ImageId: aws.String("ami-test4"), - CreationDate: aws.String(time.Now().Add(2 * time.Minute).Format(time.RFC3339)), - Architecture: aws.String("x86_64"), - Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String("test-ami-3")}, - {Key: aws.String("foo"), Value: aws.String("bar")}, - {Key: aws.String("kubernetes.io/os"), Value: aws.String("test-requirement-1")}, - }, - }, - }, - }) - ExpectApplied(ctx, env.Client, nodeTemplate) - ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) - nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - sortRequirements(nodeTemplate.Status.AMIs) - Expect(nodeTemplate.Status.AMIs).To(ContainElements([]v1alpha1.AMI{ - { - Name: "test-ami-4", - ID: "ami-test4", - Requirements: []v1.NodeSelectorRequirement{ - { - Key: "kubernetes.io/os", - Operator: "In", - Values: []string{ - "test-requirement-1", - }, - }, - { - Key: "kubernetes.io/arch", - Operator: "In", - Values: []string{ - "amd64", - }, - }, - }, - }, - }, - )) - }) - }) -}) - -func sortRequirements(amis []v1alpha1.AMI) { - for i := range amis { - sort.Slice(amis[i].Requirements, func(p, q int) bool { - return amis[i].Requirements[p].Key > amis[i].Requirements[q].Key - }) - } -}