From 5112b5d2cea1e5f1a9ea665386edebea4583f237 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda <74629455+engedaam@users.noreply.github.com> Date: Thu, 4 Apr 2024 16:56:43 -0700 Subject: [PATCH] chore: Explicitly define all components that will be used for Instance type discovery (#5854) Co-authored-by: Chris Negus Co-authored-by: Jonathan Innis --- pkg/providers/instancetype/instancetype.go | 19 +- pkg/providers/instancetype/suite_test.go | 471 +++++++++++++++++++-- pkg/providers/instancetype/types.go | 80 ++-- pkg/providers/launchtemplate/suite_test.go | 52 ++- 4 files changed, 544 insertions(+), 78 deletions(-) diff --git a/pkg/providers/instancetype/instancetype.go b/pkg/providers/instancetype/instancetype.go index 8f7e939b0025..7d11ed22678f 100644 --- a/pkg/providers/instancetype/instancetype.go +++ b/pkg/providers/instancetype/instancetype.go @@ -42,6 +42,8 @@ import ( "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/utils/pretty" + + "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" ) const ( @@ -104,6 +106,13 @@ func (p *Provider) List(ctx context.Context, kc *corev1beta1.KubeletConfiguratio return aws.StringValue(s.AvailabilityZone) })...) + if kc == nil { + kc = &corev1beta1.KubeletConfiguration{} + } + if nodeClass == nil { + nodeClass = &v1beta1.EC2NodeClass{} + } + // Compute fully initialized instance types hash key subnetZonesHash, _ := hashstructure.Hash(subnetZones, hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true}) kcHash, _ := hashstructure.Hash(kc, hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true}) @@ -133,6 +142,7 @@ func (p *Provider) List(ctx context.Context, kc *corev1beta1.KubeletConfiguratio if p.cm.HasChanged("zones", allZones) { logging.FromContext(ctx).With("zones", allZones.UnsortedList()).Debugf("discovered zones") } + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) result := lo.Map(instanceTypes, func(i *ec2.InstanceTypeInfo, _ int) *cloudprovider.InstanceType { instanceTypeVCPU.With(prometheus.Labels{ instanceTypeLabel: *i.InstanceType, @@ -141,7 +151,14 @@ func (p *Provider) List(ctx context.Context, kc *corev1beta1.KubeletConfiguratio instanceTypeLabel: *i.InstanceType, }).Set(float64(aws.Int64Value(i.MemoryInfo.SizeInMiB) * 1024 * 1024)) - return NewInstanceType(ctx, i, kc, p.region, nodeClass, p.createOfferings(ctx, i, instanceTypeOfferings[aws.StringValue(i.InstanceType)], allZones, subnetZones)) + // !!! Important !!! + // Any changes to the values passed into the NewInstanceType method will require making updates to the cache key + // so that Karpenter is able to cache the set of InstanceTypes based on values that alter the set of instance types + // !!! Important !!! + return NewInstanceType(ctx, i, p.region, + nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, + kc.MaxPods, kc.PodsPerCore, kc.KubeReserved, kc.SystemReserved, kc.EvictionHard, kc.EvictionSoft, + amiFamily, p.createOfferings(ctx, i, instanceTypeOfferings[aws.StringValue(i.InstanceType)], allZones, subnetZones)) }) p.cache.SetDefault(key, result) return result, nil diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index 586fff9fa3d1..16b6f7263adf 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -54,6 +54,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" + "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" "github.com/aws/karpenter-provider-aws/pkg/providers/instancetype" "github.com/aws/karpenter-provider-aws/pkg/test" ) @@ -119,6 +120,7 @@ var _ = Describe("InstanceTypes", func() { }, }, }, + Kubelet: &corev1beta1.KubeletConfiguration{}, NodeClassRef: &corev1beta1.NodeClassReference{ Name: nodeClass.Name, }, @@ -731,7 +733,21 @@ var _ = Describe("InstanceTypes", func() { instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) for _, info := range instanceInfo { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).ToNot(BeNumerically("==", 110)) } }) @@ -740,7 +756,21 @@ var _ = Describe("InstanceTypes", func() { Expect(err).To(BeNil()) for _, info := range instanceInfo { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, windowsNodeClass, nil) + amiFamily := amifamily.GetAMIFamily(windowsNodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + windowsNodeClass.Spec.BlockDeviceMappings, + windowsNodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 110)) } }) @@ -841,7 +871,21 @@ var _ = Describe("InstanceTypes", func() { }) Context("System Reserved Resources", func() { It("should use defaults when no kubelet is specified", func() { - it := instancetype.NewInstanceType(ctx, info, &corev1beta1.KubeletConfiguration{}, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.SystemReserved.Cpu().String()).To(Equal("0")) Expect(it.Overhead.SystemReserved.Memory().String()).To(Equal("0")) Expect(it.Overhead.SystemReserved.StorageEphemeral().String()).To(Equal("0")) @@ -854,7 +898,21 @@ var _ = Describe("InstanceTypes", func() { string(v1.ResourceEphemeralStorage): "10Gi", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.SystemReserved.Cpu().String()).To(Equal("2")) Expect(it.Overhead.SystemReserved.Memory().String()).To(Equal("20Gi")) Expect(it.Overhead.SystemReserved.StorageEphemeral().String()).To(Equal("10Gi")) @@ -862,13 +920,28 @@ var _ = Describe("InstanceTypes", func() { }) Context("Kube Reserved Resources", func() { It("should use defaults when no kubelet is specified", func() { - it := instancetype.NewInstanceType(ctx, info, &corev1beta1.KubeletConfiguration{}, fake.DefaultRegion, nodeClass, nil) + nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{} + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.KubeReserved.Cpu().String()).To(Equal("80m")) Expect(it.Overhead.KubeReserved.Memory().String()).To(Equal("893Mi")) Expect(it.Overhead.KubeReserved.StorageEphemeral().String()).To(Equal("1Gi")) }) It("should override kube reserved when specified", func() { - it := instancetype.NewInstanceType(ctx, info, &corev1beta1.KubeletConfiguration{ + nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ SystemReserved: map[string]string{ string(v1.ResourceCPU): "1", string(v1.ResourceMemory): "20Gi", @@ -879,7 +952,22 @@ var _ = Describe("InstanceTypes", func() { string(v1.ResourceMemory): "10Gi", string(v1.ResourceEphemeralStorage): "2Gi", }, - }, fake.DefaultRegion, nodeClass, nil) + } + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.KubeReserved.Cpu().String()).To(Equal("2")) Expect(it.Overhead.KubeReserved.Memory().String()).To(Equal("10Gi")) Expect(it.Overhead.KubeReserved.StorageEphemeral().String()).To(Equal("2Gi")) @@ -904,7 +992,21 @@ var _ = Describe("InstanceTypes", func() { instancetype.MemoryAvailable: "500Mi", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("500Mi")) }) It("should override eviction threshold when specified as a percentage value", func() { @@ -919,7 +1021,21 @@ var _ = Describe("InstanceTypes", func() { instancetype.MemoryAvailable: "10%", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) It("should consider the eviction threshold disabled when specified as 100%", func() { @@ -934,7 +1050,21 @@ var _ = Describe("InstanceTypes", func() { instancetype.MemoryAvailable: "100%", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("0")) }) It("should used default eviction threshold for memory when evictionHard not specified", func() { @@ -949,7 +1079,21 @@ var _ = Describe("InstanceTypes", func() { instancetype.MemoryAvailable: "50Mi", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("50Mi")) }) }) @@ -966,7 +1110,21 @@ var _ = Describe("InstanceTypes", func() { instancetype.MemoryAvailable: "500Mi", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("500Mi")) }) It("should override eviction threshold when specified as a percentage value", func() { @@ -984,7 +1142,21 @@ var _ = Describe("InstanceTypes", func() { instancetype.MemoryAvailable: "10%", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) It("should consider the eviction threshold disabled when specified as 100%", func() { @@ -999,7 +1171,21 @@ var _ = Describe("InstanceTypes", func() { instancetype.MemoryAvailable: "100%", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("0")) }) It("should ignore eviction threshold when using Bottlerocket AMI", func() { @@ -1018,12 +1204,41 @@ var _ = Describe("InstanceTypes", func() { instancetype.MemoryAvailable: "10Gi", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("1Gi")) }) }) It("should take the default eviction threshold when none is specified", func() { - it := instancetype.NewInstanceType(ctx, info, &corev1beta1.KubeletConfiguration{}, fake.DefaultRegion, nodeClass, nil) + nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{} + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Cpu().String()).To(Equal("0")) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("100Mi")) Expect(it.Overhead.EvictionThreshold.StorageEphemeral().AsApproximateFloat64()).To(BeNumerically("~", resources.Quantity("2Gi").AsApproximateFloat64())) @@ -1043,7 +1258,21 @@ var _ = Describe("InstanceTypes", func() { instancetype.MemoryAvailable: "1Gi", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("3Gi")) }) It("should take the greater of evictionHard and evictionSoft for overhead as a value", func() { @@ -1061,7 +1290,21 @@ var _ = Describe("InstanceTypes", func() { instancetype.MemoryAvailable: "5%", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.05, 10)) }) It("should take the greater of evictionHard and evictionSoft for overhead with mixed percentage/value", func() { @@ -1079,7 +1322,21 @@ var _ = Describe("InstanceTypes", func() { instancetype.MemoryAvailable: "1Gi", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) }) @@ -1088,11 +1345,39 @@ var _ = Describe("InstanceTypes", func() { Expect(err).To(BeNil()) for _, info := range instanceInfo { if *info.InstanceType == "t3.large" { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 35)) } if *info.InstanceType == "m6idn.32xlarge" { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 345)) } } @@ -1104,7 +1389,21 @@ var _ = Describe("InstanceTypes", func() { MaxPods: ptr.Int32(10), } for _, info := range instanceInfo { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 10)) } }) @@ -1115,7 +1414,21 @@ var _ = Describe("InstanceTypes", func() { MaxPods: ptr.Int32(10), } for _, info := range instanceInfo { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 10)) } }) @@ -1130,7 +1443,21 @@ var _ = Describe("InstanceTypes", func() { return *info.InstanceType == "t3.large" }) Expect(ok).To(Equal(true)) - it := instancetype.NewInstanceType(ctx, t3Large, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + t3Large, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) // t3.large // maxInterfaces = 3 // maxIPv4PerInterface = 12 @@ -1150,7 +1477,21 @@ var _ = Describe("InstanceTypes", func() { return *info.InstanceType == "t3.large" }) Expect(ok).To(Equal(true)) - it := instancetype.NewInstanceType(ctx, t3Large, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + t3Large, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) // t3.large // maxInterfaces = 3 // maxIPv4PerInterface = 12 @@ -1167,7 +1508,21 @@ var _ = Describe("InstanceTypes", func() { PodsPerCore: ptr.Int32(1), } for _, info := range instanceInfo { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", ptr.Int64Value(info.VCpuInfo.DefaultVCpus))) } }) @@ -1179,7 +1534,21 @@ var _ = Describe("InstanceTypes", func() { MaxPods: ptr.Int32(20), } for _, info := range instanceInfo { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", lo.Min([]int64{20, ptr.Int64Value(info.VCpuInfo.DefaultVCpus) * 4}))) } }) @@ -1191,7 +1560,21 @@ var _ = Describe("InstanceTypes", func() { PodsPerCore: ptr.Int32(1), } for _, info := range instanceInfo { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) limitedPods := instancetype.ENILimitedPods(ctx, info) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", limitedPods.Value())) } @@ -1204,11 +1587,39 @@ var _ = Describe("InstanceTypes", func() { } for _, info := range instanceInfo { if *info.InstanceType == "t3.large" { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 35)) } if *info.InstanceType == "m6idn.32xlarge" { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 345)) } } diff --git a/pkg/providers/instancetype/types.go b/pkg/providers/instancetype/types.go index 32cb71e3daa4..6cebec2f523b 100644 --- a/pkg/providers/instancetype/types.go +++ b/pkg/providers/instancetype/types.go @@ -49,19 +49,20 @@ var ( instanceTypeScheme = regexp.MustCompile(`(^[a-z]+)(\-[0-9]+tb)?([0-9]+).*\.`) ) -func NewInstanceType(ctx context.Context, info *ec2.InstanceTypeInfo, kc *corev1beta1.KubeletConfiguration, - region string, nodeClass *v1beta1.EC2NodeClass, offerings cloudprovider.Offerings) *cloudprovider.InstanceType { +func NewInstanceType(ctx context.Context, info *ec2.InstanceTypeInfo, region string, + blockDeviceMappings []*v1beta1.BlockDeviceMapping, instanceStorePolicy *v1beta1.InstanceStorePolicy, maxPods *int32, podsPerCore *int32, + kubeReserved map[string]string, systemReserved map[string]string, evictionHard map[string]string, evictionSoft map[string]string, + amiFamily amifamily.AMIFamily, offerings cloudprovider.Offerings) *cloudprovider.InstanceType { - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) it := &cloudprovider.InstanceType{ Name: aws.StringValue(info.InstanceType), Requirements: computeRequirements(info, offerings, region, amiFamily), Offerings: offerings, - Capacity: computeCapacity(ctx, info, amiFamily, nodeClass, kc), + Capacity: computeCapacity(ctx, info, amiFamily, blockDeviceMappings, instanceStorePolicy, maxPods, podsPerCore), Overhead: &cloudprovider.InstanceTypeOverhead{ - KubeReserved: kubeReservedResources(cpu(info), pods(ctx, info, amiFamily, kc), ENILimitedPods(ctx, info), amiFamily, kc), - SystemReserved: systemReservedResources(kc), - EvictionThreshold: evictionThreshold(memory(ctx, info), ephemeralStorage(info, amiFamily, nodeClass), amiFamily, kc), + KubeReserved: kubeReservedResources(cpu(info), pods(ctx, info, amiFamily, maxPods, podsPerCore), ENILimitedPods(ctx, info), amiFamily, kubeReserved), + SystemReserved: systemReservedResources(systemReserved), + EvictionThreshold: evictionThreshold(memory(ctx, info), ephemeralStorage(info, amiFamily, blockDeviceMappings, instanceStorePolicy), amiFamily, evictionHard, evictionSoft), }, } if it.Requirements.Compatible(scheduling.NewRequirements(scheduling.NewRequirement(v1.LabelOSStable, v1.NodeSelectorOpIn, string(v1.Windows)))) == nil { @@ -174,13 +175,14 @@ func getArchitecture(info *ec2.InstanceTypeInfo) string { } func computeCapacity(ctx context.Context, info *ec2.InstanceTypeInfo, amiFamily amifamily.AMIFamily, - nodeClass *v1beta1.EC2NodeClass, kc *corev1beta1.KubeletConfiguration) v1.ResourceList { + blockDeviceMapping []*v1beta1.BlockDeviceMapping, instanceStorePolicy *v1beta1.InstanceStorePolicy, + maxPods *int32, podsPerCore *int32) v1.ResourceList { resourceList := v1.ResourceList{ v1.ResourceCPU: *cpu(info), v1.ResourceMemory: *memory(ctx, info), - v1.ResourceEphemeralStorage: *ephemeralStorage(info, amiFamily, nodeClass), - v1.ResourcePods: *pods(ctx, info, amiFamily, kc), + v1.ResourceEphemeralStorage: *ephemeralStorage(info, amiFamily, blockDeviceMapping, instanceStorePolicy), + v1.ResourcePods: *pods(ctx, info, amiFamily, maxPods, podsPerCore), v1beta1.ResourceAWSPodENI: *awsPodENI(aws.StringValue(info.InstanceType)), v1beta1.ResourceNVIDIAGPU: *nvidiaGPUs(info), v1beta1.ResourceAMDGPU: *amdGPUs(info), @@ -208,16 +210,16 @@ func memory(ctx context.Context, info *ec2.InstanceTypeInfo) *resource.Quantity } // Setting ephemeral-storage to be either the default value, what is defined in blockDeviceMappings, or the combined size of local store volumes. -func ephemeralStorage(info *ec2.InstanceTypeInfo, amiFamily amifamily.AMIFamily, nodeClass *v1beta1.EC2NodeClass) *resource.Quantity { +func ephemeralStorage(info *ec2.InstanceTypeInfo, amiFamily amifamily.AMIFamily, blockDeviceMappings []*v1beta1.BlockDeviceMapping, instanceStorePolicy *v1beta1.InstanceStorePolicy) *resource.Quantity { // If local store disks have been configured for node ephemeral-storage, use the total size of the disks. - if lo.FromPtr(nodeClass.Spec.InstanceStorePolicy) == v1beta1.InstanceStorePolicyRAID0 { + if lo.FromPtr(instanceStorePolicy) == v1beta1.InstanceStorePolicyRAID0 { if info.InstanceStorageInfo != nil && info.InstanceStorageInfo.TotalSizeInGB != nil { return resources.Quantity(fmt.Sprintf("%dG", *info.InstanceStorageInfo.TotalSizeInGB)) } } - if len(nodeClass.Spec.BlockDeviceMappings) != 0 { + if len(blockDeviceMappings) != 0 { // First check if there's a root volume configured in blockDeviceMappings. - if blockDeviceMapping, ok := lo.Find(nodeClass.Spec.BlockDeviceMappings, func(bdm *v1beta1.BlockDeviceMapping) bool { + if blockDeviceMapping, ok := lo.Find(blockDeviceMappings, func(bdm *v1beta1.BlockDeviceMapping) bool { return bdm.RootVolume }); ok && blockDeviceMapping.EBS.VolumeSize != nil { return blockDeviceMapping.EBS.VolumeSize @@ -225,11 +227,11 @@ func ephemeralStorage(info *ec2.InstanceTypeInfo, amiFamily amifamily.AMIFamily, switch amiFamily.(type) { case *amifamily.Custom: // We can't know if a custom AMI is going to have a volume size. - volumeSize := nodeClass.Spec.BlockDeviceMappings[len(nodeClass.Spec.BlockDeviceMappings)-1].EBS.VolumeSize + volumeSize := blockDeviceMappings[len(blockDeviceMappings)-1].EBS.VolumeSize return lo.Ternary(volumeSize != nil, volumeSize, amifamily.DefaultEBS.VolumeSize) default: // If a block device mapping exists in the provider for the root volume, use the volume size specified in the provider. If not, use the default - if blockDeviceMapping, ok := lo.Find(nodeClass.Spec.BlockDeviceMappings, func(bdm *v1beta1.BlockDeviceMapping) bool { + if blockDeviceMapping, ok := lo.Find(blockDeviceMappings, func(bdm *v1beta1.BlockDeviceMapping) bool { return *bdm.DeviceName == *amiFamily.EphemeralBlockDevice() }); ok && blockDeviceMapping.EBS.VolumeSize != nil { return blockDeviceMapping.EBS.VolumeSize @@ -338,16 +340,13 @@ func privateIPv4Address(info *ec2.InstanceTypeInfo) *resource.Quantity { return resources.Quantity(fmt.Sprint(capacity)) } -func systemReservedResources(kc *corev1beta1.KubeletConfiguration) v1.ResourceList { - if kc != nil && kc.SystemReserved != nil { - return lo.MapEntries(kc.SystemReserved, func(k string, v string) (v1.ResourceName, resource.Quantity) { - return v1.ResourceName(k), resource.MustParse(v) - }) - } - return v1.ResourceList{} +func systemReservedResources(systemReserved map[string]string) v1.ResourceList { + return lo.MapEntries(systemReserved, func(k string, v string) (v1.ResourceName, resource.Quantity) { + return v1.ResourceName(k), resource.MustParse(v) + }) } -func kubeReservedResources(cpus, pods, eniLimitedPods *resource.Quantity, amiFamily amifamily.AMIFamily, kc *corev1beta1.KubeletConfiguration) v1.ResourceList { +func kubeReservedResources(cpus, pods, eniLimitedPods *resource.Quantity, amiFamily amifamily.AMIFamily, kubeReserved map[string]string) v1.ResourceList { if amiFamily.FeatureFlags().UsesENILimitedMemoryOverhead { pods = eniLimitedPods } @@ -377,31 +376,24 @@ func kubeReservedResources(cpus, pods, eniLimitedPods *resource.Quantity, amiFam resources[v1.ResourceCPU] = *cpuOverhead } } - if kc != nil && kc.KubeReserved != nil { - kubeReserved := lo.MapEntries(kc.KubeReserved, func(k string, v string) (v1.ResourceName, resource.Quantity) { - return v1.ResourceName(k), resource.MustParse(v) - }) - return lo.Assign(resources, kubeReserved) - } - return resources + return lo.Assign(resources, lo.MapEntries(kubeReserved, func(k string, v string) (v1.ResourceName, resource.Quantity) { + return v1.ResourceName(k), resource.MustParse(v) + })) } -func evictionThreshold(memory *resource.Quantity, storage *resource.Quantity, amiFamily amifamily.AMIFamily, kc *corev1beta1.KubeletConfiguration) v1.ResourceList { +func evictionThreshold(memory *resource.Quantity, storage *resource.Quantity, amiFamily amifamily.AMIFamily, evictionHard map[string]string, evictionSoft map[string]string) v1.ResourceList { overhead := v1.ResourceList{ v1.ResourceMemory: resource.MustParse("100Mi"), v1.ResourceEphemeralStorage: resource.MustParse(fmt.Sprint(math.Ceil(float64(storage.Value()) / 100 * 10))), } - if kc == nil { - return overhead - } override := v1.ResourceList{} var evictionSignals []map[string]string - if kc.EvictionHard != nil { - evictionSignals = append(evictionSignals, kc.EvictionHard) + if evictionHard != nil { + evictionSignals = append(evictionSignals, evictionHard) } - if kc.EvictionSoft != nil && amiFamily.FeatureFlags().EvictionSoftEnabled { - evictionSignals = append(evictionSignals, kc.EvictionSoft) + if evictionSoft != nil && amiFamily.FeatureFlags().EvictionSoftEnabled { + evictionSignals = append(evictionSignals, evictionSoft) } for _, m := range evictionSignals { @@ -418,19 +410,19 @@ func evictionThreshold(memory *resource.Quantity, storage *resource.Quantity, am return lo.Assign(overhead, override) } -func pods(ctx context.Context, info *ec2.InstanceTypeInfo, amiFamily amifamily.AMIFamily, kc *corev1beta1.KubeletConfiguration) *resource.Quantity { +func pods(ctx context.Context, info *ec2.InstanceTypeInfo, amiFamily amifamily.AMIFamily, maxPods *int32, podsPerCore *int32) *resource.Quantity { var count int64 switch { - case kc != nil && kc.MaxPods != nil: - count = int64(ptr.Int32Value(kc.MaxPods)) + case maxPods != nil: + count = int64(ptr.Int32Value(maxPods)) case amiFamily.FeatureFlags().SupportsENILimitedPodDensity: count = ENILimitedPods(ctx, info).Value() default: count = 110 } - if kc != nil && ptr.Int32Value(kc.PodsPerCore) > 0 && amiFamily.FeatureFlags().PodsPerCoreEnabled { - count = lo.Min([]int64{int64(ptr.Int32Value(kc.PodsPerCore)) * ptr.Int64Value(info.VCpuInfo.DefaultVCpus), count}) + if ptr.Int32Value(podsPerCore) > 0 && amiFamily.FeatureFlags().PodsPerCoreEnabled { + count = lo.Min([]int64{int64(ptr.Int32Value(podsPerCore)) * ptr.Int64Value(info.VCpuInfo.DefaultVCpus), count}) } return resources.Quantity(fmt.Sprint(count)) } diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index 7e956d0115c3..cbe23d5bd586 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -58,6 +58,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" + "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap/mime" "github.com/aws/karpenter-provider-aws/pkg/providers/instancetype" @@ -135,6 +136,7 @@ var _ = Describe("LaunchTemplates", func() { }, }, }, + Kubelet: &corev1beta1.KubeletConfiguration{}, NodeClassRef: &corev1beta1.NodeClassReference{ Name: nodeClass.Name, }, @@ -822,7 +824,22 @@ var _ = Describe("LaunchTemplates", func() { })) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, "", nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + "", + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) + overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("993Mi")) }) @@ -860,7 +877,22 @@ var _ = Describe("LaunchTemplates", func() { })) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, "", nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + "", + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) + overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("993Mi")) }) @@ -871,7 +903,21 @@ var _ = Describe("LaunchTemplates", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, "", nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + "", + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1565Mi")) })