From 5b2a3dc70774a4d013c141a18474c2252d0fa905 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Tue, 23 Apr 2024 01:06:30 -0700 Subject: [PATCH] Use status values for AMI resolution --- pkg/cloudprovider/drift.go | 8 +- pkg/cloudprovider/suite_test.go | 76 +++++++++++++++ pkg/providers/amifamily/ami.go | 21 ++-- pkg/providers/amifamily/resolver.go | 10 +- pkg/providers/instance/suite_test.go | 34 +++++++ pkg/providers/instancetype/suite_test.go | 42 ++++++++ pkg/providers/launchtemplate/suite_test.go | 108 +++++++++++++++------ 7 files changed, 248 insertions(+), 51 deletions(-) diff --git a/pkg/cloudprovider/drift.go b/pkg/cloudprovider/drift.go index 69dc5aad1783..75e46faeceb1 100644 --- a/pkg/cloudprovider/drift.go +++ b/pkg/cloudprovider/drift.go @@ -80,14 +80,10 @@ func (c *CloudProvider) isAMIDrifted(ctx context.Context, nodeClaim *corev1beta1 if !found { return "", fmt.Errorf(`finding node instance type "%s"`, nodeClaim.Labels[v1.LabelInstanceTypeStable]) } - amis, err := c.amiProvider.Get(ctx, nodeClass, &amifamily.Options{}) - if err != nil { - return "", fmt.Errorf("getting amis, %w", err) - } - if len(amis) == 0 { + if len(nodeClass.Status.AMIs) == 0 { return "", fmt.Errorf("no amis exist given constraints") } - mappedAMIs := amis.MapToInstanceTypes([]*cloudprovider.InstanceType{nodeInstanceType}) + mappedAMIs := amifamily.MapToInstanceTypes([]*cloudprovider.InstanceType{nodeInstanceType}, nodeClass.Status.AMIs) if !lo.Contains(lo.Keys(mappedAMIs), instance.ImageID) { return AMIDrift, nil } diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index b9d255ed3808..dbecedb55e16 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -50,6 +50,7 @@ import ( "sigs.k8s.io/karpenter/pkg/events" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/operator/scheme" + "sigs.k8s.io/karpenter/pkg/scheduling" coretest "sigs.k8s.io/karpenter/pkg/test" . "github.com/onsi/ginkgo/v2" @@ -138,6 +139,40 @@ var _ = Describe("CloudProvider", func() { }, }, }) + amdRequirements := scheduling.NewRequirements() + amdRequirements.Add(scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64)) + nodeClass.Status.AMIs = []v1beta1.AMI{ + { + ID: "ami-test1", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpDoesNotExist), + ).NodeSelectorRequirements(), + }, + { + ID: "ami-test2", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpExists), + ).NodeSelectorRequirements(), + }, + { + ID: "ami-test3", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpExists), + ).NodeSelectorRequirements(), + }, + { + ID: "ami-test4", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64), + scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpDoesNotExist), + ).NodeSelectorRequirements(), + }, + } Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) }) @@ -586,6 +621,20 @@ var _ = Describe("CloudProvider", func() { }, }, }) + nodeClass.Status.AMIs = []v1beta1.AMI{ + { + ID: armAMIID, + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64), + ).NodeSelectorRequirements(), + }, + { + ID: amdAMIID, + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + ).NodeSelectorRequirements(), + }, + } ExpectApplied(ctx, env.Client, nodePool, nodeClass) instanceTypes, err := cloudProvider.GetInstanceTypes(ctx, nodePool) Expect(err).ToNot(HaveOccurred()) @@ -733,6 +782,14 @@ var _ = Describe("CloudProvider", func() { }) It("should return drifted if the AMI no longer matches the existing NodeClaims instance type", func() { nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: amdAMIID}} + nodeClass.Status.AMIs = []v1beta1.AMI{ + { + ID: amdAMIID, + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + ).NodeSelectorRequirements(), + }, + } ExpectApplied(ctx, env.Client, nodeClass) isDrifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) Expect(err).ToNot(HaveOccurred()) @@ -740,6 +797,10 @@ var _ = Describe("CloudProvider", func() { }) Context("Static Drift Detection", func() { BeforeEach(func() { + armRequirements := scheduling.NewRequirements() + armRequirements.Add(scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, "arm64")) + amdRequirements := scheduling.NewRequirements() + amdRequirements.Add(scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, "x86_64")) nodeClass = &v1beta1.EC2NodeClass{ ObjectMeta: nodeClass.ObjectMeta, Spec: v1beta1.EC2NodeClassSpec{ @@ -777,6 +838,18 @@ var _ = Describe("CloudProvider", func() { }, }, }, + Status: v1beta1.EC2NodeClassStatus{ + AMIs: []v1beta1.AMI{ + { + ID: armAMIID, + Requirements: armRequirements.NodeSelectorRequirements(), + }, + { + ID: amdAMIID, + Requirements: amdRequirements.NodeSelectorRequirements(), + }, + }, + }, } nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) nodeClaim.Annotations = lo.Assign(nodeClaim.Annotations, map[string]string{v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) @@ -1013,6 +1086,9 @@ var _ = Describe("CloudProvider", func() { }, }, }, + Status: v1beta1.EC2NodeClassStatus{ + AMIs: nodeClass.Status.AMIs, + }, }) nodePool2 := coretest.NodePool(corev1beta1.NodePool{ Spec: corev1beta1.NodePoolSpec{ diff --git a/pkg/providers/amifamily/ami.go b/pkg/providers/amifamily/ami.go index fd89ced04dcb..4034d16d7f1a 100644 --- a/pkg/providers/amifamily/ami.go +++ b/pkg/providers/amifamily/ami.go @@ -19,6 +19,7 @@ import ( "fmt" "sort" "strings" + "sync" "time" "github.com/aws/aws-sdk-go/aws" @@ -45,6 +46,7 @@ type Provider interface { } type DefaultProvider struct { + sync.RWMutex cache *cache.Cache ssm ssmiface.SSMAPI ec2api ec2iface.EC2API @@ -79,12 +81,12 @@ func (a AMIs) Sort() { }) } -func (a AMIs) String() string { +func String(amis []v1beta1.AMI) string { var sb strings.Builder - ids := lo.Map(a, func(a AMI, _ int) string { return a.AmiID }) - if len(a) > 25 { + ids := lo.Map(amis, func(a v1beta1.AMI, _ int) string { return a.ID }) + if len(amis) > 25 { sb.WriteString(strings.Join(ids[:25], ", ")) - sb.WriteString(fmt.Sprintf(" and %d other(s)", len(a)-25)) + sb.WriteString(fmt.Sprintf(" and %d other(s)", len(amis)-25)) } else { sb.WriteString(strings.Join(ids, ", ")) } @@ -92,12 +94,12 @@ func (a AMIs) String() string { } // MapToInstanceTypes returns a map of AMIIDs that are the most recent on creationDate to compatible instancetypes -func (a AMIs) MapToInstanceTypes(instanceTypes []*cloudprovider.InstanceType) map[string][]*cloudprovider.InstanceType { +func MapToInstanceTypes(instanceTypes []*cloudprovider.InstanceType, amis []v1beta1.AMI) map[string][]*cloudprovider.InstanceType { amiIDs := map[string][]*cloudprovider.InstanceType{} for _, instanceType := range instanceTypes { - for _, ami := range a { - if err := instanceType.Requirements.Compatible(ami.Requirements, scheduling.AllowUndefinedWellKnownLabels); err == nil { - amiIDs[ami.AmiID] = append(amiIDs[ami.AmiID], instanceType) + for _, ami := range amis { + if err := instanceType.Requirements.Compatible(scheduling.NewNodeSelectorRequirementsWithMinValues(ami.Requirements...), scheduling.AllowUndefinedWellKnownLabels); err == nil { + amiIDs[ami.ID] = append(amiIDs[ami.ID], instanceType) break } } @@ -117,6 +119,9 @@ func NewDefaultProvider(versionProvider version.Provider, ssm ssmiface.SSMAPI, e // Get Returning a list of AMIs with its associated requirements func (p *DefaultProvider) Get(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, options *Options) (AMIs, error) { + p.Lock() + defer p.Unlock() + var err error var amis AMIs if len(nodeClass.Spec.AMISelectorTerms) == 0 { diff --git a/pkg/providers/amifamily/resolver.go b/pkg/providers/amifamily/resolver.go index 65adec3a9abb..8f63d3c2a8df 100644 --- a/pkg/providers/amifamily/resolver.go +++ b/pkg/providers/amifamily/resolver.go @@ -122,16 +122,12 @@ func NewResolver(amiProvider Provider) *Resolver { // Multiple ResolvedTemplates are returned based on the instanceTypes passed in to support special AMIs for certain instance types like GPUs. func (r Resolver) Resolve(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, capacityType string, options *Options) ([]*LaunchTemplate, error) { amiFamily := GetAMIFamily(nodeClass.Spec.AMIFamily, options) - amis, err := r.amiProvider.Get(ctx, nodeClass, options) - if err != nil { - return nil, err - } - if len(amis) == 0 { + if len(nodeClass.Status.AMIs) == 0 { return nil, fmt.Errorf("no amis exist given constraints") } - mappedAMIs := amis.MapToInstanceTypes(instanceTypes) + mappedAMIs := MapToInstanceTypes(instanceTypes, nodeClass.Status.AMIs) if len(mappedAMIs) == 0 { - return nil, fmt.Errorf("no instance types satisfy requirements of amis %v", amis) + return nil, fmt.Errorf("no instance types satisfy requirements of amis %v", String(nodeClass.Status.AMIs)) } var resolvedTemplates []*LaunchTemplate for amiID, instanceTypes := range mappedAMIs { diff --git a/pkg/providers/instance/suite_test.go b/pkg/providers/instance/suite_test.go index 1693e2eadf0d..f3cd735fa1fe 100644 --- a/pkg/providers/instance/suite_test.go +++ b/pkg/providers/instance/suite_test.go @@ -23,6 +23,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/samber/lo" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" @@ -31,6 +32,7 @@ import ( "sigs.k8s.io/karpenter/pkg/events" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/operator/scheme" + "sigs.k8s.io/karpenter/pkg/scheduling" coretest "sigs.k8s.io/karpenter/pkg/test" "github.com/aws/karpenter-provider-aws/pkg/apis" @@ -106,6 +108,38 @@ var _ = Describe("InstanceProvider", func() { }, }, }) + nodeClass.Status.AMIs = []v1beta1.AMI{ + { + ID: "ami-test1", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpDoesNotExist), + ).NodeSelectorRequirements(), + }, + { + ID: "ami-test2", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpExists), + ).NodeSelectorRequirements(), + }, + { + ID: "ami-test3", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpExists), + ).NodeSelectorRequirements(), + }, + { + ID: "ami-test4", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64), + scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpDoesNotExist), + ).NodeSelectorRequirements(), + }, + } Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) }) diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index 0e270e24a70b..404d2e01fdd1 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -155,6 +155,48 @@ var _ = Describe("InstanceTypeProvider", func() { }, }, }) + nodeClass.Status.AMIs = []v1beta1.AMI{ + { + ID: "ami-test1", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpDoesNotExist), + ).NodeSelectorRequirements(), + }, + { + ID: "ami-test2", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpExists), + ).NodeSelectorRequirements(), + }, + { + ID: "ami-test3", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpExists), + ).NodeSelectorRequirements(), + }, + { + ID: "ami-test4", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64), + scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpDoesNotExist), + ).NodeSelectorRequirements(), + }, + } + windowsNodeClass.Status.AMIs = []v1beta1.AMI{ + { + ID: "ami-window-test1", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1.LabelOSStable, v1.NodeSelectorOpIn, string(v1.Windows)), + scheduling.NewRequirement(v1.LabelWindowsBuild, v1.NodeSelectorOpIn, v1beta1.Windows2022Build), + ).NodeSelectorRequirements(), + }, + } Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) }) diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index 6799b9ed3ab8..e146749e5aab 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -51,12 +51,14 @@ import ( "sigs.k8s.io/karpenter/pkg/events" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/operator/scheme" + "sigs.k8s.io/karpenter/pkg/scheduling" coretest "sigs.k8s.io/karpenter/pkg/test" . "sigs.k8s.io/karpenter/pkg/test/expectations" "github.com/aws/karpenter-provider-aws/pkg/apis" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" + "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/status" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" @@ -146,6 +148,38 @@ var _ = Describe("LaunchTemplate Provider", func() { }, }, }) + nodeClass.Status.AMIs = []v1beta1.AMI{ + { + ID: "ami-test1", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpDoesNotExist), + ).NodeSelectorRequirements(), + }, + { + ID: "ami-test2", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpExists), + ).NodeSelectorRequirements(), + }, + { + ID: "ami-test3", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpExists), + ).NodeSelectorRequirements(), + }, + { + ID: "ami-test4", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64), + scheduling.NewRequirement(v1beta1.LabelInstanceGPUCount, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1beta1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpDoesNotExist), + ).NodeSelectorRequirements(), + }, + } Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) }) @@ -171,6 +205,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }, }, }) + nodeClass2.Status.AMIs = nodeClass.Status.AMIs pods := []*v1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ { @@ -1765,14 +1800,14 @@ var _ = Describe("LaunchTemplate Provider", func() { Context("Custom AMI Selector", func() { It("should use ami selector specified in EC2NodeClass", func() { nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} - awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ + nodeClass.Status.AMIs = []v1beta1.AMI{ { - Name: aws.String(coretest.RandomName()), - ImageId: aws.String("ami-123"), - Architecture: aws.String("x86_64"), - CreationDate: aws.String("2022-08-15T12:00:00Z"), + ID: "ami-123", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + ).NodeSelectorRequirements(), }, - }}) + } ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1786,14 +1821,14 @@ var _ = Describe("LaunchTemplate Provider", func() { nodeClass.Spec.UserData = aws.String("special user data") nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom - awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ + nodeClass.Status.AMIs = []v1beta1.AMI{ { - Name: aws.String(coretest.RandomName()), - ImageId: aws.String("ami-123"), - Architecture: aws.String("x86_64"), - CreationDate: aws.String("2022-08-15T12:00:00Z"), + ID: "ami-123", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + ).NodeSelectorRequirements(), }, - }}) + } ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1822,6 +1857,7 @@ var _ = Describe("LaunchTemplate Provider", func() { pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) + awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(BeNumerically(">=", 2)) actualFilter := awsEnv.EC2API.CalledWithDescribeImagesInput.Pop().Filters expectedFilter := []*ec2.Filter{ @@ -1833,21 +1869,21 @@ var _ = Describe("LaunchTemplate Provider", func() { Expect(actualFilter).To(Equal(expectedFilter)) }) It("should create multiple launch templates when multiple amis are discovered with non-equivalent requirements", func() { - awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + nodeClass.Status.AMIs = []v1beta1.AMI{ { - Name: aws.String(coretest.RandomName()), - ImageId: aws.String("ami-123"), - Architecture: aws.String("x86_64"), - CreationDate: aws.String("2022-08-15T12:00:00Z"), + ID: "ami-123", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + ).NodeSelectorRequirements(), }, { - Name: aws.String(coretest.RandomName()), - ImageId: aws.String("ami-456"), - Architecture: aws.String("arm64"), - CreationDate: aws.String("2022-08-10T12:00:00Z"), + ID: "ami-456", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64), + ).NodeSelectorRequirements(), }, - }}) - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + } ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1860,7 +1896,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) Expect(expectedImageIds.Equal(actualImageIds)).To(BeTrue()) }) - It("should create a launch template with the newest compatible AMI when multiple amis are discovered", func() { + FIt("should create a launch template with the newest compatible AMI when multiple amis are discovered", func() { awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ { Name: aws.String(coretest.RandomName()), @@ -1884,6 +1920,8 @@ var _ = Describe("LaunchTemplate Provider", func() { }}) nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} ExpectApplied(ctx, env.Client, nodeClass) + controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeClass)) nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ { NodeSelectorRequirement: v1.NodeSelectorRequirement{ @@ -1895,6 +1933,7 @@ var _ = Describe("LaunchTemplate Provider", func() { } ExpectApplied(ctx, env.Client, nodePool) pod := coretest.UnschedulablePod() + awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(BeNumerically(">=", 1)) @@ -1906,6 +1945,7 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should fail if no amis match selector.", func() { awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{}}) nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + nodeClass.Status.AMIs = []v1beta1.AMI{} ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1916,6 +1956,14 @@ var _ = Describe("LaunchTemplate Provider", func() { awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ {Name: aws.String(coretest.RandomName()), ImageId: aws.String("ami-123"), Architecture: aws.String("newnew"), CreationDate: aws.String("2022-01-01T12:00:00Z")}}}) nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + nodeClass.Status.AMIs = []v1beta1.AMI{ + { + ID: "ami-123", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, "newnew"), + ).NodeSelectorRequirements(), + }, + } ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1927,14 +1975,14 @@ var _ = Describe("LaunchTemplate Provider", func() { awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", version): "test-ami-123", } - awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ + nodeClass.Status.AMIs = []v1beta1.AMI{ { - Name: aws.String(coretest.RandomName()), - ImageId: aws.String("test-ami-123"), - Architecture: aws.String("x86_64"), - CreationDate: aws.String("2022-08-15T12:00:00Z"), + ID: "test-ami-123", + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + ).NodeSelectorRequirements(), }, - }}) + } ExpectApplied(ctx, env.Client, nodeClass) ExpectApplied(ctx, env.Client, nodePool) pod := coretest.UnschedulablePod()