From 96e8c1f13373cc6d18621cb5d133cc393645f1de Mon Sep 17 00:00:00 2001 From: Cameron McAvoy Date: Mon, 4 Mar 2024 10:33:48 -0600 Subject: [PATCH] Add option to configure extra pod capacity for alternate cnis --- pkg/operator/options/options.go | 2 ++ pkg/operator/options/options_validation.go | 8 ++++++++ pkg/operator/options/suite_test.go | 11 ++++++++++- pkg/providers/instancetype/types.go | 2 +- pkg/test/options.go | 2 ++ 5 files changed, 23 insertions(+), 2 deletions(-) diff --git a/pkg/operator/options/options.go b/pkg/operator/options/options.go index 47121c75e9da..59138dfcddd1 100644 --- a/pkg/operator/options/options.go +++ b/pkg/operator/options/options.go @@ -44,6 +44,7 @@ type Options struct { VMMemoryOverheadPercent float64 InterruptionQueue string ReservedENIs int + MaxPodsExtraCapacity int } func (o *Options) AddFlags(fs *coreoptions.FlagSet) { @@ -56,6 +57,7 @@ func (o *Options) AddFlags(fs *coreoptions.FlagSet) { fs.Float64Var(&o.VMMemoryOverheadPercent, "vm-memory-overhead-percent", utils.WithDefaultFloat64("VM_MEMORY_OVERHEAD_PERCENT", 0.075), "The VM memory overhead as a percent that will be subtracted from the total memory for all instance types.") fs.StringVar(&o.InterruptionQueue, "interruption-queue", env.WithDefaultString("INTERRUPTION_QUEUE", ""), "Interruption queue is the name of the SQS queue used for processing interruption events from EC2. Interruption handling is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.") fs.IntVar(&o.ReservedENIs, "reserved-enis", env.WithDefaultInt("RESERVED_ENIS", 0), "Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html.") + fs.IntVar(&o.MaxPodsExtraCapacity, "max-pods-extra-capacity", env.WithDefaultInt("MAX_PODS_EXTRA_CAPACITY", 2), "Extra pod capacity allocatable on nodes due to host networked pods that do not consume ENIs. The default is for aws-cni and kube-proxy pods, which are not included in the calculations for max-pods or kube-reserved.") } func (o *Options) Parse(fs *coreoptions.FlagSet, args ...string) error { diff --git a/pkg/operator/options/options_validation.go b/pkg/operator/options/options_validation.go index 1c4a733700ec..55893722b247 100644 --- a/pkg/operator/options/options_validation.go +++ b/pkg/operator/options/options_validation.go @@ -28,6 +28,7 @@ func (o Options) Validate() error { o.validateVMMemoryOverheadPercent(), o.validateAssumeRoleDuration(), o.validateReservedENIs(), + o.validateMaxPodsExtraCapacity(), o.validateRequiredFields(), ) } @@ -66,6 +67,13 @@ func (o Options) validateReservedENIs() error { return nil } +func (o Options) validateMaxPodsExtraCapacity() error { + if o.MaxPodsExtraCapacity < 0 { + return fmt.Errorf("max-pods-extra-capacity cannot be negative") + } + return nil +} + func (o Options) validateRequiredFields() error { if o.ClusterName == "" { return fmt.Errorf("missing field, cluster-name") diff --git a/pkg/operator/options/suite_test.go b/pkg/operator/options/suite_test.go index 04281d8dacdf..792b2bd85fd8 100644 --- a/pkg/operator/options/suite_test.go +++ b/pkg/operator/options/suite_test.go @@ -65,7 +65,8 @@ var _ = Describe("Options", func() { "--isolated-vpc", "--vm-memory-overhead-percent", "0.1", "--interruption-queue", "env-cluster", - "--reserved-enis", "10") + "--reserved-enis", "10", + "--max-pods-extra-capacity", "1") Expect(err).ToNot(HaveOccurred()) expectOptionsEqual(opts, test.Options(test.OptionsFields{ AssumeRoleARN: lo.ToPtr("env-role"), @@ -77,6 +78,7 @@ var _ = Describe("Options", func() { VMMemoryOverheadPercent: lo.ToPtr[float64](0.1), InterruptionQueue: lo.ToPtr("env-cluster"), ReservedENIs: lo.ToPtr(10), + MaxPodsExtraCapacity: lo.ToPtr(1), })) }) It("should correctly fallback to env vars when CLI flags aren't set", func() { @@ -89,6 +91,7 @@ var _ = Describe("Options", func() { os.Setenv("VM_MEMORY_OVERHEAD_PERCENT", "0.1") os.Setenv("INTERRUPTION_QUEUE", "env-cluster") os.Setenv("RESERVED_ENIS", "10") + os.Setenv("MAX_PODS_EXTRA_CAPACITY", "1") // Add flags after we set the environment variables so that the parsing logic correctly refers // to the new environment variable values @@ -105,6 +108,7 @@ var _ = Describe("Options", func() { VMMemoryOverheadPercent: lo.ToPtr[float64](0.1), InterruptionQueue: lo.ToPtr("env-cluster"), ReservedENIs: lo.ToPtr(10), + MaxPodsExtraCapacity: lo.ToPtr(1), })) }) @@ -132,6 +136,10 @@ var _ = Describe("Options", func() { err := opts.Parse(fs, "--cluster-name", "test-cluster", "--reserved-enis", "-1") Expect(err).To(HaveOccurred()) }) + It("should fail when maxPodsExtraCapacity is negative", func() { + err := opts.Parse(fs, "--cluster-name", "test-cluster", "--max-pods-extra-capacity", "-1") + Expect(err).To(HaveOccurred()) + }) }) }) @@ -146,4 +154,5 @@ func expectOptionsEqual(optsA *options.Options, optsB *options.Options) { Expect(optsA.VMMemoryOverheadPercent).To(Equal(optsB.VMMemoryOverheadPercent)) Expect(optsA.InterruptionQueue).To(Equal(optsB.InterruptionQueue)) Expect(optsA.ReservedENIs).To(Equal(optsB.ReservedENIs)) + Expect(optsA.MaxPodsExtraCapacity).To(Equal(optsB.MaxPodsExtraCapacity)) } diff --git a/pkg/providers/instancetype/types.go b/pkg/providers/instancetype/types.go index bce08d2be7ed..9b4ad233f300 100644 --- a/pkg/providers/instancetype/types.go +++ b/pkg/providers/instancetype/types.go @@ -348,7 +348,7 @@ func ENILimitedPods(ctx context.Context, info *ec2.InstanceTypeInfo) *resource.Q return resource.NewQuantity(0, resource.DecimalSI) } addressesPerInterface := *info.NetworkInfo.Ipv4AddressesPerInterface - return resources.Quantity(fmt.Sprint(usableNetworkInterfaces*(addressesPerInterface-1) + 2)) + return resources.Quantity(fmt.Sprint(usableNetworkInterfaces*(addressesPerInterface-1) + int64(options.FromContext(ctx).MaxPodsExtraCapacity))) } func privateIPv4Address(instanceTypeName string) *resource.Quantity { diff --git a/pkg/test/options.go b/pkg/test/options.go index 4657f28e9c53..585241496c2d 100644 --- a/pkg/test/options.go +++ b/pkg/test/options.go @@ -34,6 +34,7 @@ type OptionsFields struct { VMMemoryOverheadPercent *float64 InterruptionQueue *string ReservedENIs *int + MaxPodsExtraCapacity *int } func Options(overrides ...OptionsFields) *options.Options { @@ -53,5 +54,6 @@ func Options(overrides ...OptionsFields) *options.Options { VMMemoryOverheadPercent: lo.FromPtrOr(opts.VMMemoryOverheadPercent, 0.075), InterruptionQueue: lo.FromPtrOr(opts.InterruptionQueue, ""), ReservedENIs: lo.FromPtrOr(opts.ReservedENIs, 0), + MaxPodsExtraCapacity: lo.FromPtrOr(opts.MaxPodsExtraCapacity, 2), } }