From 9bf86c1dd2184b32533ccfb0e5cb3e6f086d2f0b Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Thu, 21 Sep 2023 12:23:34 -0700 Subject: [PATCH] test: Add `v1beta1/EC2NodeClass` instance testing for cloudprovider (#4670) --- pkg/providers/instance/nodeclass_test.go | 82 +++++++++++++++++ pkg/providers/instance/nodetemplate_test.go | 97 +++++++++++++++++++++ pkg/providers/instance/suite_test.go | 72 --------------- 3 files changed, 179 insertions(+), 72 deletions(-) create mode 100644 pkg/providers/instance/nodeclass_test.go create mode 100644 pkg/providers/instance/nodetemplate_test.go diff --git a/pkg/providers/instance/nodeclass_test.go b/pkg/providers/instance/nodeclass_test.go new file mode 100644 index 000000000000..08330f42439f --- /dev/null +++ b/pkg/providers/instance/nodeclass_test.go @@ -0,0 +1,82 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instance_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" + coretest "github.com/aws/karpenter-core/pkg/test" + . "github.com/aws/karpenter-core/pkg/test/expectations" + "github.com/aws/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/test" +) + +var _ = Describe("NodeClass/InstanceProvider", func() { + var nodeClass *v1beta1.EC2NodeClass + var nodePool *corev1beta1.NodePool + var nodeClaim *corev1beta1.NodeClaim + BeforeEach(func() { + nodeClass = test.EC2NodeClass() + nodePool = coretest.NodePool(corev1beta1.NodePool{ + Spec: corev1beta1.NodePoolSpec{ + Template: corev1beta1.NodeClaimTemplate{ + Spec: corev1beta1.NodeClaimSpec{ + NodeClass: &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, + }, + }, + }, + }, + }) + nodeClaim = coretest.NodeClaim(corev1beta1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + corev1beta1.NodePoolLabelKey: nodePool.Name, + }, + }, + Spec: corev1beta1.NodeClaimSpec{ + NodeClass: &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, + }, + }, + }) + }) + It("should return an ICE error when all attempted instance types return an ICE error", func() { + ExpectApplied(ctx, env.Client, nodeClaim, nodePool, nodeClass) + awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{ + {CapacityType: v1alpha5.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, + {CapacityType: v1alpha5.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1b"}, + {CapacityType: v1alpha5.CapacityTypeSpot, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, + {CapacityType: v1alpha5.CapacityTypeSpot, InstanceType: "m5.xlarge", Zone: "test-zone-1b"}, + }) + instanceTypes, err := cloudProvider.GetInstanceTypes(ctx, nodePool) + Expect(err).ToNot(HaveOccurred()) + + // Filter down to a single instance type + instanceTypes = lo.Filter(instanceTypes, func(i *corecloudprovider.InstanceType, _ int) bool { return i.Name == "m5.xlarge" }) + + // Since all the capacity pools are ICEd. This should return back an ICE error + instance, err := awsEnv.InstanceProvider.Create(ctx, nodeClass, nodeClaim, instanceTypes) + Expect(corecloudprovider.IsInsufficientCapacityError(err)).To(BeTrue()) + Expect(instance).To(BeNil()) + }) +}) diff --git a/pkg/providers/instance/nodetemplate_test.go b/pkg/providers/instance/nodetemplate_test.go new file mode 100644 index 000000000000..baa4a2751a92 --- /dev/null +++ b/pkg/providers/instance/nodetemplate_test.go @@ -0,0 +1,97 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instance_test + +import ( + "github.com/aws/aws-sdk-go/aws" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" + coretest "github.com/aws/karpenter-core/pkg/test" + . "github.com/aws/karpenter-core/pkg/test/expectations" + nodeclaimutil "github.com/aws/karpenter-core/pkg/utils/nodeclaim" + nodepoolutil "github.com/aws/karpenter-core/pkg/utils/nodepool" + "github.com/aws/karpenter/pkg/apis/v1alpha1" + "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/test" + nodeclassutil "github.com/aws/karpenter/pkg/utils/nodeclass" +) + +var _ = Describe("NodeTemplate/InstanceProvider", func() { + var nodeTemplate *v1alpha1.AWSNodeTemplate + var provisioner *v1alpha5.Provisioner + var machine *v1alpha5.Machine + BeforeEach(func() { + nodeTemplate = &v1alpha1.AWSNodeTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: coretest.RandomName(), + }, + Spec: v1alpha1.AWSNodeTemplateSpec{ + AWS: v1alpha1.AWS{ + AMIFamily: aws.String(v1alpha1.AMIFamilyAL2), + SubnetSelector: map[string]string{"*": "*"}, + SecurityGroupSelector: map[string]string{"*": "*"}, + }, + }, + } + provisioner = test.Provisioner(coretest.ProvisionerOptions{ + Requirements: []v1.NodeSelectorRequirement{{ + Key: v1alpha1.LabelInstanceCategory, + Operator: v1.NodeSelectorOpExists, + }}, + ProviderRef: &v1alpha5.MachineTemplateRef{ + APIVersion: nodeTemplate.APIVersion, + Kind: nodeTemplate.Kind, + Name: nodeTemplate.Name, + }, + }) + machine = coretest.Machine(v1alpha5.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + v1alpha5.ProvisionerNameLabelKey: provisioner.Name, + }, + }, + Spec: v1alpha5.MachineSpec{ + MachineTemplateRef: &v1alpha5.MachineTemplateRef{ + Name: nodeTemplate.Name, + }, + }, + }) + }) + It("should return an ICE error when all attempted instance types return an ICE error", func() { + ExpectApplied(ctx, env.Client, machine, provisioner, nodeTemplate) + awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{ + {CapacityType: v1alpha5.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, + {CapacityType: v1alpha5.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1b"}, + {CapacityType: v1alpha5.CapacityTypeSpot, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, + {CapacityType: v1alpha5.CapacityTypeSpot, InstanceType: "m5.xlarge", Zone: "test-zone-1b"}, + }) + instanceTypes, err := cloudProvider.GetInstanceTypes(ctx, nodepoolutil.New(provisioner)) + Expect(err).ToNot(HaveOccurred()) + + // Filter down to a single instance type + instanceTypes = lo.Filter(instanceTypes, func(i *corecloudprovider.InstanceType, _ int) bool { return i.Name == "m5.xlarge" }) + + // Since all the capacity pools are ICEd. This should return back an ICE error + instance, err := awsEnv.InstanceProvider.Create(ctx, nodeclassutil.New(nodeTemplate), nodeclaimutil.New(machine), instanceTypes) + Expect(corecloudprovider.IsInsufficientCapacityError(err)).To(BeTrue()) + Expect(instance).To(BeNil()) + }) +}) diff --git a/pkg/providers/instance/suite_test.go b/pkg/providers/instance/suite_test.go index 626c49b2b808..b27ca56a7cae 100644 --- a/pkg/providers/instance/suite_test.go +++ b/pkg/providers/instance/suite_test.go @@ -18,42 +18,27 @@ import ( "context" "testing" - "github.com/aws/aws-sdk-go/aws" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/samber/lo" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" . "knative.dev/pkg/logging/testing" coresettings "github.com/aws/karpenter-core/pkg/apis/settings" - "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" "github.com/aws/karpenter-core/pkg/events" "github.com/aws/karpenter-core/pkg/operator/injection" "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" - . "github.com/aws/karpenter-core/pkg/test/expectations" - nodeclaimutil "github.com/aws/karpenter-core/pkg/utils/nodeclaim" - nodepoolutil "github.com/aws/karpenter-core/pkg/utils/nodepool" "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" - "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/cloudprovider" - "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/test" - nodeclassutil "github.com/aws/karpenter/pkg/utils/nodeclass" ) var ctx context.Context var opts options.Options var env *coretest.Environment var awsEnv *test.Environment -var provisioner *v1alpha5.Provisioner -var nodeTemplate *v1alpha1.AWSNodeTemplate -var machine *v1alpha5.Machine var cloudProvider *cloudprovider.CloudProvider func TestAWS(t *testing.T) { @@ -79,61 +64,4 @@ var _ = BeforeEach(func() { ctx = injection.WithOptions(ctx, opts) ctx = coresettings.ToContext(ctx, coretest.Settings()) ctx = settings.ToContext(ctx, test.Settings()) - nodeTemplate = &v1alpha1.AWSNodeTemplate{ - ObjectMeta: metav1.ObjectMeta{ - Name: coretest.RandomName(), - }, - Spec: v1alpha1.AWSNodeTemplateSpec{ - AWS: v1alpha1.AWS{ - AMIFamily: aws.String(v1alpha1.AMIFamilyAL2), - SubnetSelector: map[string]string{"*": "*"}, - SecurityGroupSelector: map[string]string{"*": "*"}, - }, - }, - } - provisioner = test.Provisioner(coretest.ProvisionerOptions{ - Requirements: []v1.NodeSelectorRequirement{{ - Key: v1alpha1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpExists, - }}, - ProviderRef: &v1alpha5.MachineTemplateRef{ - APIVersion: nodeTemplate.APIVersion, - Kind: nodeTemplate.Kind, - Name: nodeTemplate.Name, - }, - }) - machine = coretest.Machine(v1alpha5.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - v1alpha5.ProvisionerNameLabelKey: provisioner.Name, - }, - }, - Spec: v1alpha5.MachineSpec{ - MachineTemplateRef: &v1alpha5.MachineTemplateRef{ - Name: nodeTemplate.Name, - }, - }, - }) -}) - -var _ = Describe("InstanceProvider", func() { - It("should return an ICE error when all attempted instance types return an ICE error", func() { - ExpectApplied(ctx, env.Client, machine, provisioner, nodeTemplate) - awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{ - {CapacityType: v1alpha5.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, - {CapacityType: v1alpha5.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1b"}, - {CapacityType: v1alpha5.CapacityTypeSpot, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, - {CapacityType: v1alpha5.CapacityTypeSpot, InstanceType: "m5.xlarge", Zone: "test-zone-1b"}, - }) - instanceTypes, err := cloudProvider.GetInstanceTypes(ctx, nodepoolutil.New(provisioner)) - Expect(err).ToNot(HaveOccurred()) - - // Filter down to a single instance type - instanceTypes = lo.Filter(instanceTypes, func(i *corecloudprovider.InstanceType, _ int) bool { return i.Name == "m5.xlarge" }) - - // Since all the capacity pools are ICEd. This should return back an ICE error - instance, err := awsEnv.InstanceProvider.Create(ctx, nodeclassutil.New(nodeTemplate), nodeclaimutil.New(machine), instanceTypes) - Expect(corecloudprovider.IsInsufficientCapacityError(err)).To(BeTrue()) - Expect(instance).To(BeNil()) - }) })