From c04fa8696e7c0110e51fa90373f139c4d0496113 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Wed, 24 Apr 2024 10:59:14 -0700 Subject: [PATCH] Use Security Groups status for node launch --- pkg/cloudprovider/drift.go | 12 +-- pkg/cloudprovider/suite_test.go | 57 ++++++++--- pkg/providers/instance/suite_test.go | 14 +++ pkg/providers/instancetype/suite_test.go | 51 ++++++++++ .../launchtemplate/launchtemplate.go | 25 +++-- pkg/providers/launchtemplate/suite_test.go | 95 +++++++++++++++++++ 6 files changed, 220 insertions(+), 34 deletions(-) diff --git a/pkg/cloudprovider/drift.go b/pkg/cloudprovider/drift.go index 69dc5aad1783..318af54ac768 100644 --- a/pkg/cloudprovider/drift.go +++ b/pkg/cloudprovider/drift.go @@ -54,7 +54,7 @@ func (c *CloudProvider) isNodeClassDrifted(ctx context.Context, nodeClaim *corev if err != nil { return "", fmt.Errorf("calculating ami drift, %w", err) } - securitygroupDrifted, err := c.areSecurityGroupsDrifted(ctx, instance, nodeClass) + securitygroupDrifted, err := c.areSecurityGroupsDrifted(instance, nodeClass) if err != nil { return "", fmt.Errorf("calculating securitygroup drift, %w", err) } @@ -118,14 +118,10 @@ func (c *CloudProvider) isSubnetDrifted(ctx context.Context, instance *instance. // Checks if the security groups are drifted, by comparing the security groups returned from the SecurityGroupProvider // to the ec2 instance security groups -func (c *CloudProvider) areSecurityGroupsDrifted(ctx context.Context, ec2Instance *instance.Instance, nodeClass *v1beta1.EC2NodeClass) (cloudprovider.DriftReason, error) { - securitygroup, err := c.securityGroupProvider.List(ctx, nodeClass) - if err != nil { - return "", err - } - securityGroupIds := sets.New(lo.Map(securitygroup, func(sg *ec2.SecurityGroup, _ int) string { return aws.StringValue(sg.GroupId) })...) +func (c *CloudProvider) areSecurityGroupsDrifted(ec2Instance *instance.Instance, nodeClass *v1beta1.EC2NodeClass) (cloudprovider.DriftReason, error) { + securityGroupIds := sets.New(lo.Map(nodeClass.Status.SecurityGroups, func(sg v1beta1.SecurityGroup, _ int) string { return sg.ID })...) if len(securityGroupIds) == 0 { - return "", fmt.Errorf("no security groups are discovered") + return "", fmt.Errorf("no security groups are present in the status") } if !securityGroupIds.Equal(sets.New(ec2Instance.SecurityGroupIDs...)) { diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index b9d255ed3808..3617355f1730 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -138,6 +138,20 @@ var _ = Describe("CloudProvider", func() { }, }, }) + nodeClass.Status.SecurityGroups = []v1beta1.SecurityGroup{ + { + ID: "sg-test1", + Name: "securityGroup-test1", + }, + { + ID: "sg-test2", + Name: "securityGroup-test2", + }, + { + ID: "sg-test3", + Name: "securityGroup-test3", + }, + } Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) }) @@ -586,6 +600,11 @@ var _ = Describe("CloudProvider", func() { }, }, }) + nodeClass.Status.SecurityGroups = []v1beta1.SecurityGroup{ + { + ID: validSecurityGroup, + }, + } ExpectApplied(ctx, env.Client, nodePool, nodeClass) instanceTypes, err := cloudProvider.GetInstanceTypes(ctx, nodePool) Expect(err).ToNot(HaveOccurred()) @@ -671,7 +690,8 @@ var _ = Describe("CloudProvider", func() { Expect(isDrifted).To(BeEmpty()) }) It("should return an error if the security groups are empty", func() { - awsEnv.EC2API.DescribeSecurityGroupsOutput.Set(&ec2.DescribeSecurityGroupsOutput{SecurityGroups: []*ec2.SecurityGroup{}}) + nodeClass.Status.SecurityGroups = []v1beta1.SecurityGroup{} + ExpectApplied(ctx, env.Client, nodeClass) // Instance is a reference to what we return in the GetInstances call instance.SecurityGroups = []*ec2.GroupIdentifier{{GroupId: aws.String(fake.SecurityGroupID())}} _, err := cloudProvider.IsDrifted(ctx, nodeClaim) @@ -692,18 +712,17 @@ var _ = Describe("CloudProvider", func() { Expect(isDrifted).To(Equal(cloudprovider.SecurityGroupDrift)) }) It("should return drifted if more security groups are present than instance security groups then discovered from nodeclass", func() { - awsEnv.EC2API.DescribeSecurityGroupsOutput.Set(&ec2.DescribeSecurityGroupsOutput{ - SecurityGroups: []*ec2.SecurityGroup{ - { - GroupId: aws.String(validSecurityGroup), - GroupName: aws.String("test-securitygroup"), - }, - { - GroupId: aws.String(fake.SecurityGroupID()), - GroupName: aws.String("test-securitygroup"), - }, + nodeClass.Status.SecurityGroups = []v1beta1.SecurityGroup{ + { + ID: validSecurityGroup, + Name: "test-securitygroup", }, - }) + { + ID: fake.SecurityGroupID(), + Name: "test-securitygroup", + }, + } + ExpectApplied(ctx, env.Client, nodeClass) isDrifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) Expect(err).ToNot(HaveOccurred()) Expect(isDrifted).To(Equal(cloudprovider.SecurityGroupDrift)) @@ -777,6 +796,13 @@ var _ = Describe("CloudProvider", func() { }, }, }, + Status: v1beta1.EC2NodeClassStatus{ + SecurityGroups: []v1beta1.SecurityGroup{ + { + ID: validSecurityGroup, + }, + }, + }, } nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) nodeClaim.Annotations = lo.Assign(nodeClaim.Annotations, map[string]string{v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) @@ -1013,6 +1039,13 @@ var _ = Describe("CloudProvider", func() { }, }, }, + Status: v1beta1.EC2NodeClassStatus{ + SecurityGroups: []v1beta1.SecurityGroup{ + { + ID: "sg-test1", + }, + }, + }, }) nodePool2 := coretest.NodePool(corev1beta1.NodePool{ Spec: corev1beta1.NodePoolSpec{ diff --git a/pkg/providers/instance/suite_test.go b/pkg/providers/instance/suite_test.go index 1693e2eadf0d..8945e1ae5d9e 100644 --- a/pkg/providers/instance/suite_test.go +++ b/pkg/providers/instance/suite_test.go @@ -26,9 +26,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/events" + "sigs.k8s.io/karpenter/pkg/operator/controller" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" @@ -36,6 +38,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" + "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/status" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/instance" @@ -51,6 +54,7 @@ var ctx context.Context var env *coretest.Environment var awsEnv *test.Environment var cloudProvider *cloudprovider.CloudProvider +var statusController controller.Controller func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -65,6 +69,14 @@ var _ = BeforeSuite(func() { awsEnv = test.NewEnvironment(ctx, env) cloudProvider = cloudprovider.New(awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, awsEnv.AMIProvider, awsEnv.SecurityGroupProvider, awsEnv.SubnetProvider) + statusController = status.NewController( + env.Client, + awsEnv.SubnetProvider, + awsEnv.SecurityGroupProvider, + awsEnv.AMIProvider, + awsEnv.InstanceProfileProvider, + awsEnv.LaunchTemplateProvider, + ) }) var _ = AfterSuite(func() { @@ -111,6 +123,8 @@ var _ = Describe("InstanceProvider", func() { }) It("should return an ICE error when all attempted instance types return an ICE error", func() { ExpectApplied(ctx, env.Client, nodeClaim, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{ {CapacityType: corev1beta1.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, {CapacityType: corev1beta1.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1b"}, diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index 0e270e24a70b..bd711d607c55 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -39,11 +39,13 @@ import ( . "knative.dev/pkg/logging/testing" "knative.dev/pkg/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/controllers/provisioning" "sigs.k8s.io/karpenter/pkg/controllers/state" "sigs.k8s.io/karpenter/pkg/events" + "sigs.k8s.io/karpenter/pkg/operator/controller" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/operator/scheme" "sigs.k8s.io/karpenter/pkg/scheduling" @@ -54,6 +56,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" + "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/status" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" @@ -68,6 +71,7 @@ var fakeClock *clock.FakeClock var prov *provisioning.Provisioner var cluster *state.Cluster var cloudProvider *cloudprovider.CloudProvider +var statusController controller.Controller func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -85,6 +89,14 @@ var _ = BeforeSuite(func() { env.Client, awsEnv.AMIProvider, awsEnv.SecurityGroupProvider, awsEnv.SubnetProvider) cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) prov = provisioning.NewProvisioner(env.Client, events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) + statusController = status.NewController( + env.Client, + awsEnv.SubnetProvider, + awsEnv.SecurityGroupProvider, + awsEnv.AMIProvider, + awsEnv.InstanceProfileProvider, + awsEnv.LaunchTemplateProvider, + ) }) var _ = AfterSuite(func() { @@ -161,6 +173,8 @@ var _ = Describe("InstanceTypeProvider", func() { It("should support individual instance type labels", func() { ExpectApplied(ctx, env.Client, nodePool, windowsNodePool, nodeClass, windowsNodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(windowsNodeClass)) nodeSelector := map[string]string{ // Well known @@ -214,6 +228,7 @@ var _ = Describe("InstanceTypeProvider", func() { }) It("should support combined instance type labels", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) nodeSelector := map[string]string{ // Well known @@ -265,6 +280,7 @@ var _ = Describe("InstanceTypeProvider", func() { }) It("should support instance type labels with accelerator", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) nodeSelector := map[string]string{ // Well known @@ -315,6 +331,7 @@ var _ = Describe("InstanceTypeProvider", func() { }) It("should not launch AWS Pod ENI on a t3", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ NodeSelector: map[string]string{ v1.LabelInstanceTypeStable: "t3.large", @@ -338,6 +355,7 @@ var _ = Describe("InstanceTypeProvider", func() { Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, @@ -396,6 +414,7 @@ var _ = Describe("InstanceTypeProvider", func() { }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(generateSpotPricing(cloudProvider, nodePool)) Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypes(ctx)).To(Succeed()) @@ -459,6 +478,7 @@ var _ = Describe("InstanceTypeProvider", func() { // Apply requirements and schedule pods. ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, @@ -483,6 +503,7 @@ var _ = Describe("InstanceTypeProvider", func() { }) It("should de-prioritize metal", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, @@ -502,6 +523,7 @@ var _ = Describe("InstanceTypeProvider", func() { }) It("should de-prioritize gpu types", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, @@ -528,6 +550,7 @@ var _ = Describe("InstanceTypeProvider", func() { }, }) ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ NodeSelector: map[string]string{ v1beta1.LabelInstanceSize: "metal", @@ -542,6 +565,7 @@ var _ = Describe("InstanceTypeProvider", func() { }) It("should launch AWS Pod ENI on a compatible instance type", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{v1beta1.ResourceAWSPodENI: resource.MustParse("1")}, @@ -560,6 +584,7 @@ var _ = Describe("InstanceTypeProvider", func() { It("should launch instances for Nvidia GPU resource requests", func() { nodeNames := sets.NewString() ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pods := []*v1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ @@ -593,6 +618,7 @@ var _ = Describe("InstanceTypeProvider", func() { It("should launch instances for Habana GPU resource requests", func() { nodeNames := sets.NewString() ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pods := []*v1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ @@ -624,6 +650,7 @@ var _ = Describe("InstanceTypeProvider", func() { It("should launch instances for AWS Neuron resource requests", func() { nodeNames := sets.NewString() ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pods := []*v1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ @@ -666,6 +693,7 @@ var _ = Describe("InstanceTypeProvider", func() { }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pods := []*v1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ @@ -693,6 +721,7 @@ var _ = Describe("InstanceTypeProvider", func() { }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pods := []*v1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ @@ -718,6 +747,7 @@ var _ = Describe("InstanceTypeProvider", func() { }) It("should not launch instances w/ instance storage for ephemeral storage resource requests when exceeding blockDeviceMapping", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{v1.ResourceEphemeralStorage: resource.MustParse("5000Gi")}, @@ -729,6 +759,7 @@ var _ = Describe("InstanceTypeProvider", func() { It("should launch instances w/ instance storage for ephemeral storage resource requests when disks are mounted for ephemeral-storage", func() { nodeClass.Spec.InstanceStorePolicy = lo.ToPtr(v1beta1.InstanceStorePolicyRAID0) ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{v1.ResourceEphemeralStorage: resource.MustParse("5000Gi")}, @@ -852,6 +883,7 @@ var _ = Describe("InstanceTypeProvider", func() { }) It("should launch instances in local zones", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ NodeRequirements: []v1.NodeSelectorRequirement{{ Key: v1.LabelTopologyZone, @@ -1673,6 +1705,7 @@ var _ = Describe("InstanceTypeProvider", func() { }) ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) its, err := cloudProvider.GetInstanceTypes(ctx, nodePool) Expect(err).To(BeNil()) @@ -1726,6 +1759,7 @@ var _ = Describe("InstanceTypeProvider", func() { It("should launch instances of different type on second reconciliation attempt with Insufficient Capacity Error Cache fallback", func() { awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{{CapacityType: corev1beta1.CapacityTypeOnDemand, InstanceType: "inf1.6xlarge", Zone: "test-zone-1a"}}) ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pods := []*v1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{ NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}, @@ -1773,6 +1807,7 @@ var _ = Describe("InstanceTypeProvider", func() { }, }}} ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) // it should've tried to pack them in test-zone-1a on a p3.8xlarge then hit insufficient capacity, the next attempt will try test-zone-1b ExpectNotScheduled(ctx, env.Client, pod) @@ -1807,6 +1842,7 @@ var _ = Describe("InstanceTypeProvider", func() { } // Provisions 2 m5.large instances since m5.xlarge was ICE'd ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) for _, pod := range pods { ExpectNotScheduled(ctx, env.Client, pod) @@ -1820,6 +1856,7 @@ var _ = Describe("InstanceTypeProvider", func() { It("should launch instances on later reconciliation attempt with Insufficient Capacity Error Cache expiry", func() { awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{{CapacityType: corev1beta1.CapacityTypeOnDemand, InstanceType: "inf1.6xlarge", Zone: "test-zone-1a"}}) ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "inf1.6xlarge"}, ResourceRequirements: v1.ResourceRequirements{ @@ -1853,6 +1890,7 @@ var _ = Describe("InstanceTypeProvider", func() { }, }}} ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) // it should've tried to pack them in test-zone-1a on a dl1.24xlarge then hit insufficient capacity, the next attempt will try test-zone-1b ExpectNotScheduled(ctx, env.Client, pod) @@ -1876,6 +1914,7 @@ var _ = Describe("InstanceTypeProvider", func() { } // Spot Unavailable ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) @@ -1910,6 +1949,7 @@ var _ = Describe("InstanceTypeProvider", func() { }) ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) for _, ct := range []string{corev1beta1.CapacityTypeOnDemand, corev1beta1.CapacityTypeSpot} { for _, zone := range []string{"test-zone-1a", "test-zone-1b"} { ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, @@ -1941,6 +1981,7 @@ var _ = Describe("InstanceTypeProvider", func() { Context("CapacityType", func() { It("should default to on-demand", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) @@ -1950,6 +1991,7 @@ var _ = Describe("InstanceTypeProvider", func() { nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeSpot, corev1beta1.CapacityTypeOnDemand}}}} ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) @@ -1977,6 +2019,7 @@ var _ = Describe("InstanceTypeProvider", func() { // Instance type with no zonal availability for spot shouldn't be scheduled ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) @@ -2002,6 +2045,7 @@ var _ = Describe("InstanceTypeProvider", func() { } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) @@ -2030,6 +2074,7 @@ var _ = Describe("InstanceTypeProvider", func() { }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) @@ -2043,6 +2088,7 @@ var _ = Describe("InstanceTypeProvider", func() { }) It("should default to EBS defaults when volumeSize is not defined in blockDeviceMappings for AL2 Root volume", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) @@ -2059,6 +2105,7 @@ var _ = Describe("InstanceTypeProvider", func() { awsEnv.LaunchTemplateProvider.CABundle = lo.ToPtr("Y2EtYnVuZGxlCg==") awsEnv.LaunchTemplateProvider.ClusterCIDR.Store(lo.ToPtr("10.100.0.0/16")) ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) @@ -2074,6 +2121,7 @@ var _ = Describe("InstanceTypeProvider", func() { nodeClass.Spec.AMIFamily = aws.String(v1beta1.AMIFamilyBottlerocket) nodeClass.Spec.BlockDeviceMappings[0].DeviceName = aws.String("/dev/xvdb") ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) @@ -2090,6 +2138,7 @@ var _ = Describe("InstanceTypeProvider", func() { nodeClass.Spec.AMIFamily = aws.String(v1beta1.AMIFamilyUbuntu) nodeClass.Spec.BlockDeviceMappings[0].DeviceName = aws.String("/dev/sda1") ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) @@ -2105,6 +2154,7 @@ var _ = Describe("InstanceTypeProvider", func() { Context("Metadata Options", func() { It("should default metadata options on generated launch template", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -2124,6 +2174,7 @@ var _ = Describe("InstanceTypeProvider", func() { HTTPTokens: aws.String(ec2.LaunchTemplateHttpTokensStateOptional), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index f7d692c6f0de..e4cb2f8ee64b 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -163,13 +163,12 @@ func (p *DefaultProvider) createAMIOptions(ctx context.Context, nodeClass *v1bet if err != nil { return nil, err } + // Since we are using security group status, A node is launched and immediately drift due to + // consistency delay of when the security group status is updated + // TODO @aengeda: add observation check to validate that the status is updated before a node is launched // Get constrained security groups - securityGroups, err := p.securityGroupProvider.List(ctx, nodeClass) - if err != nil { - return nil, err - } - if len(securityGroups) == 0 { - return nil, fmt.Errorf("no security groups exist given constraints") + if len(nodeClass.Status.SecurityGroups) == 0 { + return nil, fmt.Errorf("no security groups are present in the status") } options := &amifamily.Options{ ClusterName: options.FromContext(ctx).ClusterName, @@ -177,14 +176,12 @@ func (p *DefaultProvider) createAMIOptions(ctx context.Context, nodeClass *v1bet ClusterCIDR: p.ClusterCIDR.Load(), InstanceProfile: instanceProfile, InstanceStorePolicy: nodeClass.Spec.InstanceStorePolicy, - SecurityGroups: lo.Map(securityGroups, func(s *ec2.SecurityGroup, _ int) v1beta1.SecurityGroup { - return v1beta1.SecurityGroup{ID: aws.StringValue(s.GroupId), Name: aws.StringValue(s.GroupName)} - }), - Tags: tags, - Labels: labels, - CABundle: p.CABundle, - KubeDNSIP: p.KubeDNSIP, - NodeClassName: nodeClass.Name, + SecurityGroups: nodeClass.Status.SecurityGroups, + Tags: tags, + Labels: labels, + CABundle: p.CABundle, + KubeDNSIP: p.KubeDNSIP, + NodeClassName: nodeClass.Name, } if nodeClass.Spec.AssociatePublicIPAddress != nil { options.AssociatePublicIPAddress = nodeClass.Spec.AssociatePublicIPAddress diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index 6799b9ed3ab8..66b696f0d4fe 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -49,6 +49,7 @@ import ( "sigs.k8s.io/karpenter/pkg/controllers/provisioning" "sigs.k8s.io/karpenter/pkg/controllers/state" "sigs.k8s.io/karpenter/pkg/events" + "sigs.k8s.io/karpenter/pkg/operator/controller" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" @@ -57,6 +58,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" + "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/status" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" @@ -75,6 +77,7 @@ var fakeClock *clock.FakeClock var prov *provisioning.Provisioner var cluster *state.Cluster var cloudProvider *cloudprovider.CloudProvider +var statusController controller.Controller func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -94,6 +97,14 @@ var _ = BeforeSuite(func() { env.Client, awsEnv.AMIProvider, awsEnv.SecurityGroupProvider, awsEnv.SubnetProvider) cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) prov = provisioning.NewProvisioner(env.Client, events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) + statusController = status.NewController( + env.Client, + awsEnv.SubnetProvider, + awsEnv.SecurityGroupProvider, + awsEnv.AMIProvider, + awsEnv.InstanceProfileProvider, + awsEnv.LaunchTemplateProvider, + ) }) var _ = AfterSuite(func() { @@ -190,6 +201,8 @@ var _ = Describe("LaunchTemplate Provider", func() { }), } ExpectApplied(ctx, env.Client, nodePool, nodeClass, nodePool2, nodeClass2) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass2)) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...) ltConfigCount := len(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop().LaunchTemplateConfigs) + len(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop().LaunchTemplateConfigs) Expect(ltConfigCount).To(BeNumerically("==", awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len())) @@ -204,6 +217,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should default to a generated launch template", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -222,6 +236,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) }) It("should fail to provision if the instance profile isn't defined", func() { + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) nodeClass.Status.InstanceProfile = "" ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() @@ -232,6 +247,7 @@ var _ = Describe("LaunchTemplate Provider", func() { nodeClass.Spec.Role = "" nodeClass.Spec.InstanceProfile = aws.String("overridden-profile") ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -271,6 +287,7 @@ var _ = Describe("LaunchTemplate Provider", func() { } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod1 := coretest.UnschedulablePod(coretest.PodOptions{ Tolerations: []v1.Toleration{t1, t2, t3}, ResourceRequirements: rr, @@ -302,6 +319,7 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should recover from an out-of-sync launch template cache", func() { nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: aws.Int32(1)} ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -429,6 +447,7 @@ var _ = Describe("LaunchTemplate Provider", func() { Context("Labels", func() { It("should apply labels to the node", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) @@ -444,6 +463,7 @@ var _ = Describe("LaunchTemplate Provider", func() { "tag2": "tag2value", } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -476,6 +496,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -496,6 +517,7 @@ var _ = Describe("LaunchTemplate Provider", func() { "Name": "myname", } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -518,6 +540,7 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should default AL2 block device mappings", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -534,6 +557,7 @@ var _ = Describe("LaunchTemplate Provider", func() { awsEnv.LaunchTemplateProvider.CABundle = lo.ToPtr("Y2EtYnVuZGxlCg==") awsEnv.LaunchTemplateProvider.ClusterCIDR.Store(lo.ToPtr("10.100.0.0/16")) ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -572,6 +596,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -622,6 +647,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -635,6 +661,7 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should default bottlerocket second volume with root volume size", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -655,6 +682,7 @@ var _ = Describe("LaunchTemplate Provider", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -680,6 +708,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -705,12 +734,14 @@ var _ = Describe("LaunchTemplate Provider", func() { v1.ResourceEphemeralStorage: resource.MustParse("1Gi")}}, }}, )) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) }) It("should pack pods with any ephemeral-storage request", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ v1.ResourceEphemeralStorage: resource.MustParse("1G"), @@ -720,6 +751,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should pack pods with large ephemeral-storage request", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ v1.ResourceEphemeralStorage: resource.MustParse("10Gi"), @@ -729,6 +761,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should not pack pods if the sum of pod ephemeral-storage and overhead exceeds node capacity", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ v1.ResourceEphemeralStorage: resource.MustParse("19Gi"), @@ -739,6 +772,7 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should pack pods if the pod's ephemeral-storage exceeds node capacity and instance storage is mounted", func() { nodeClass.Spec.InstanceStorePolicy = lo.ToPtr(v1beta1.InstanceStorePolicyRAID0) ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ // Default node ephemeral-storage capacity is 20Gi @@ -752,6 +786,7 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should launch multiple nodes if sum of pod ephemeral-storage requests exceeds a single nodes capacity", func() { var nodes []*v1.Node ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pods := []*v1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ @@ -774,6 +809,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should only pack pods with ephemeral-storage requests that will fit on an available node", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pods := []*v1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ @@ -794,6 +830,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should not pack pod if no available instance types have enough storage", func() { ExpectApplied(ctx, env.Client, nodePool) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ v1.ResourceEphemeralStorage: resource.MustParse("150Gi"), @@ -819,6 +856,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ v1.ResourceEphemeralStorage: resource.MustParse("25Gi"), @@ -848,6 +886,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ // this pod can only be satisfied if `/dev/xvdb` will house all the pods. @@ -885,6 +924,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ // this pod can only be satisfied if `/dev/xvdb` will house all the pods. @@ -1032,6 +1072,7 @@ var _ = Describe("LaunchTemplate Provider", func() { Context("User Data", func() { It("should specify --use-max-pods=false when using ENI-based pod density", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1040,6 +1081,7 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should specify --use-max-pods=false and --max-pods user value when user specifies maxPods in NodePool", func() { nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: aws.Int32(10)} ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1054,6 +1096,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1081,6 +1124,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1108,6 +1152,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1140,6 +1185,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1172,6 +1218,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1195,6 +1242,7 @@ var _ = Describe("LaunchTemplate Provider", func() { EvictionMaxPodGracePeriod: aws.Int32(300), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1205,6 +1253,7 @@ var _ = Describe("LaunchTemplate Provider", func() { PodsPerCore: aws.Int32(2), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1216,6 +1265,7 @@ var _ = Describe("LaunchTemplate Provider", func() { MaxPods: aws.Int32(100), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1224,6 +1274,7 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should specify --dns-cluster-ip and --ip-family when running in an ipv6 cluster", func() { awsEnv.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("fd4b:121b:812b::a") ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1232,6 +1283,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should specify --dns-cluster-ip when running in an ipv4 cluster", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1242,6 +1294,7 @@ var _ = Describe("LaunchTemplate Provider", func() { ImageGCHighThresholdPercent: aws.Int32(50), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1252,6 +1305,7 @@ var _ = Describe("LaunchTemplate Provider", func() { ImageGCLowThresholdPercent: aws.Int32(50), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1262,6 +1316,7 @@ var _ = Describe("LaunchTemplate Provider", func() { CPUCFSQuota: aws.Bool(false), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1274,6 +1329,7 @@ var _ = Describe("LaunchTemplate Provider", func() { "subdomain." + v1.LabelNamespaceNodeRestriction + "/custom-label": "custom-value", }) ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1283,6 +1339,7 @@ var _ = Describe("LaunchTemplate Provider", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 nodeClass.Spec.InstanceStorePolicy = lo.ToPtr(v1beta1.InstanceStorePolicyRAID0) ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1300,6 +1357,7 @@ var _ = Describe("LaunchTemplate Provider", func() { nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "foo", Value: "bar", Effect: v1.TaintEffectNoExecute}} nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "baz", Value: "bin", Effect: v1.TaintEffectNoExecute}} ExpectApplied(ctx, env.Client, nodeClass, nodePool) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.PodOptions{ Tolerations: []v1.Toleration{{Operator: v1.TolerationOpExists}}, }) @@ -1313,6 +1371,7 @@ var _ = Describe("LaunchTemplate Provider", func() { nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "foo", Value: "bar", Effect: v1.TaintEffectNoExecute}} nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "baz", Value: "bin", Effect: v1.TaintEffectNoExecute}} ExpectApplied(ctx, env.Client, nodeClass, nodePool) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(nodePool), nodePool)).To(Succeed()) pod := coretest.UnschedulablePod(coretest.PodOptions{ Tolerations: []v1.Toleration{{Operator: v1.TolerationOpExists}}, @@ -1326,6 +1385,7 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should not bootstrap when provider ref points to a non-existent EC2NodeClass resource", func() { nodePool.Spec.Template.Spec.NodeClassRef = &corev1beta1.NodeClassReference{Name: "doesnotexist"} ExpectApplied(ctx, env.Client, nodePool) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) // This will not be scheduled since we were pointed to a non-existent EC2NodeClass resource. @@ -1334,6 +1394,7 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should not bootstrap on invalid toml user data", func() { nodeClass.Spec.UserData = aws.String("#/bin/bash\n ./not-toml.sh") ExpectApplied(ctx, env.Client, nodeClass, nodePool) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) // This will not be scheduled since userData cannot be generated for the prospective node. @@ -1341,6 +1402,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should override system reserved values in user data", func() { ExpectApplied(ctx, env.Client, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ SystemReserved: map[string]string{ string(v1.ResourceCPU): "2", @@ -1366,6 +1428,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should override kube reserved values in user data", func() { ExpectApplied(ctx, env.Client, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ KubeReserved: map[string]string{ string(v1.ResourceCPU): "2", @@ -1391,6 +1454,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should override kube reserved values in user data", func() { ExpectApplied(ctx, env.Client, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ EvictionHard: map[string]string{ "memory.available": "10%", @@ -1419,6 +1483,7 @@ var _ = Describe("LaunchTemplate Provider", func() { MaxPods: aws.Int32(10), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1437,6 +1502,7 @@ var _ = Describe("LaunchTemplate Provider", func() { ImageGCHighThresholdPercent: aws.Int32(50), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1457,6 +1523,7 @@ var _ = Describe("LaunchTemplate Provider", func() { ImageGCLowThresholdPercent: aws.Int32(50), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1474,6 +1541,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should pass ClusterDNSIP when discovered", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1492,6 +1560,7 @@ var _ = Describe("LaunchTemplate Provider", func() { CPUCFSQuota: aws.Bool(false), } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1515,6 +1584,7 @@ var _ = Describe("LaunchTemplate Provider", func() { Expect(err).To(BeNil()) nodeClass.Spec.UserData = aws.String(string(content)) ExpectApplied(ctx, env.Client, nodeClass, nodePool) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1528,6 +1598,7 @@ var _ = Describe("LaunchTemplate Provider", func() { Expect(err).To(BeNil()) nodeClass.Spec.UserData = aws.String(string(content)) ExpectApplied(ctx, env.Client, nodeClass, nodePool) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1541,6 +1612,7 @@ var _ = Describe("LaunchTemplate Provider", func() { Expect(err).To(BeNil()) nodeClass.Spec.UserData = aws.String(string(content)) ExpectApplied(ctx, env.Client, nodeClass, nodePool) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1552,6 +1624,7 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should handle empty custom user data", func() { nodeClass.Spec.UserData = nil ExpectApplied(ctx, env.Client, nodeClass, nodePool) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1583,6 +1656,7 @@ var _ = Describe("LaunchTemplate Provider", func() { } nodePool.Spec.Template.Spec.Taints = desiredTaints ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(coretest.UnscheduleablePodOptions(coretest.PodOptions{ Tolerations: []v1.Toleration{{ Operator: v1.TolerationOpExists, @@ -1611,6 +1685,7 @@ var _ = Describe("LaunchTemplate Provider", func() { nodePool.Spec.Template.Labels = desiredLabels ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1631,6 +1706,7 @@ var _ = Describe("LaunchTemplate Provider", func() { func(field string, kc corev1beta1.KubeletConfiguration) { nodePool.Spec.Template.Spec.Kubelet = &kc ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1721,6 +1797,7 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should set LocalDiskStrategy to Raid0 when specified by the InstanceStorePolicy", func() { nodeClass.Spec.InstanceStorePolicy = lo.ToPtr(v1beta1.InstanceStorePolicyRAID0) ExpectApplied(ctx, env.Client, nodeClass, nodePool) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1740,6 +1817,7 @@ var _ = Describe("LaunchTemplate Provider", func() { } nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} ExpectApplied(ctx, env.Client, nodeClass, nodePool) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1754,6 +1832,7 @@ var _ = Describe("LaunchTemplate Provider", func() { Entry("empty", nil, "al2023_userdata_unmerged.golden"), ) It("should fail to create launch templates if cluster CIDR is unresolved", func() { + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) awsEnv.LaunchTemplateProvider.ClusterCIDR.Store(nil) ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() @@ -1774,6 +1853,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }, }}) ExpectApplied(ctx, env.Client, nodeClass, nodePool) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1795,6 +1875,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }, }}) ExpectApplied(ctx, env.Client, nodeClass, nodePool) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1819,6 +1900,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }, }}) ExpectApplied(ctx, env.Client, nodeClass, nodePool) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1849,6 +1931,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }}) nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} ExpectApplied(ctx, env.Client, nodeClass, nodePool) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1884,6 +1967,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }}) nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} ExpectApplied(ctx, env.Client, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ { NodeSelectorRequirement: v1.NodeSelectorRequirement{ @@ -1907,6 +1991,7 @@ var _ = Describe("LaunchTemplate Provider", func() { awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{}}) nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} ExpectApplied(ctx, env.Client, nodeClass, nodePool) + ExpectReconcileFailed(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) @@ -1917,6 +2002,7 @@ var _ = Describe("LaunchTemplate Provider", func() { {Name: aws.String(coretest.RandomName()), ImageId: aws.String("ami-123"), Architecture: aws.String("newnew"), CreationDate: aws.String("2022-01-01T12:00:00Z")}}}) nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} ExpectApplied(ctx, env.Client, nodeClass, nodePool) + ExpectReconcileFailed(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) @@ -1936,6 +2022,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }, }}) ExpectApplied(ctx, env.Client, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) ExpectApplied(ctx, env.Client, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1951,6 +2038,7 @@ var _ = Describe("LaunchTemplate Provider", func() { {Tags: map[string]string{"Name": "test-subnet-3"}}, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1962,6 +2050,7 @@ var _ = Describe("LaunchTemplate Provider", func() { {Tags: map[string]string{"Name": "test-subnet-2"}}, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1973,6 +2062,7 @@ var _ = Describe("LaunchTemplate Provider", func() { func(setValue, expectedValue, isEFA bool) { nodeClass.Spec.AssociatePublicIPAddress = lo.ToPtr(setValue) ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod(lo.Ternary(isEFA, coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{v1beta1.ResourceEFA: resource.MustParse("2")}, @@ -1994,6 +2084,7 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should specify the --dns-cluster-ip flag when clusterDNSIP is set", func() { nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ClusterDNS: []string{"10.0.10.100"}} ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -2011,6 +2102,7 @@ var _ = Describe("LaunchTemplate Provider", func() { Expect(err).To(BeNil()) nodeClass.Spec.UserData = aws.String(string(content)) ExpectApplied(ctx, env.Client, nodeClass, nodePool) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(nodePool), nodePool)).To(Succeed()) pod := coretest.UnschedulablePod(coretest.PodOptions{ NodeSelector: map[string]string{ @@ -2026,6 +2118,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should bootstrap when custom user data is empty", func() { ExpectApplied(ctx, env.Client, nodeClass, nodePool) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(nodePool), nodePool)).To(Succeed()) pod := coretest.UnschedulablePod(coretest.PodOptions{ NodeSelector: map[string]string{ @@ -2045,6 +2138,7 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should default detailed monitoring to off", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -2057,6 +2151,7 @@ var _ = Describe("LaunchTemplate Provider", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 nodeClass.Spec.DetailedMonitoring = aws.Bool(true) ExpectApplied(ctx, env.Client, nodePool, nodeClass) + ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod)