diff --git a/pkg/operator/options/options.go b/pkg/operator/options/options.go index 84d9c38c5268..25b33a50c16a 100644 --- a/pkg/operator/options/options.go +++ b/pkg/operator/options/options.go @@ -42,6 +42,7 @@ type Options struct { VMMemoryOverheadPercent float64 InterruptionQueue string ReservedENIs int + MaxPodsExtraCapacity int } func (o *Options) AddFlags(fs *coreoptions.FlagSet) { @@ -54,6 +55,7 @@ func (o *Options) AddFlags(fs *coreoptions.FlagSet) { fs.Float64Var(&o.VMMemoryOverheadPercent, "vm-memory-overhead-percent", env.WithDefaultFloat64("VM_MEMORY_OVERHEAD_PERCENT", 0.075), "The VM memory overhead as a percent that will be subtracted from the total memory for all instance types.") fs.StringVar(&o.InterruptionQueue, "interruption-queue", env.WithDefaultString("INTERRUPTION_QUEUE", ""), "Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.") fs.IntVar(&o.ReservedENIs, "reserved-enis", env.WithDefaultInt("RESERVED_ENIS", 0), "Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html.") + fs.IntVar(&o.MaxPodsExtraCapacity, "max-pods-extra-capacity", env.WithDefaultInt("MAX_PODS_EXTRA_CAPACITY", 2), "Extra pod capacity allocatable on nodes due to host networked pods that do not consume ENIs. The default is for aws-cni and kube-proxy pods, which are not included in the calculations for max-pods or kube-reserved.") } func (o *Options) Parse(fs *coreoptions.FlagSet, args ...string) error { diff --git a/pkg/operator/options/options_validation.go b/pkg/operator/options/options_validation.go index 1c4a733700ec..55893722b247 100644 --- a/pkg/operator/options/options_validation.go +++ b/pkg/operator/options/options_validation.go @@ -28,6 +28,7 @@ func (o Options) Validate() error { o.validateVMMemoryOverheadPercent(), o.validateAssumeRoleDuration(), o.validateReservedENIs(), + o.validateMaxPodsExtraCapacity(), o.validateRequiredFields(), ) } @@ -66,6 +67,13 @@ func (o Options) validateReservedENIs() error { return nil } +func (o Options) validateMaxPodsExtraCapacity() error { + if o.MaxPodsExtraCapacity < 0 { + return fmt.Errorf("max-pods-extra-capacity cannot be negative") + } + return nil +} + func (o Options) validateRequiredFields() error { if o.ClusterName == "" { return fmt.Errorf("missing field, cluster-name") diff --git a/pkg/operator/options/suite_test.go b/pkg/operator/options/suite_test.go index 6b7e39715149..266110114497 100644 --- a/pkg/operator/options/suite_test.go +++ b/pkg/operator/options/suite_test.go @@ -65,7 +65,8 @@ var _ = Describe("Options", func() { "--isolated-vpc", "--vm-memory-overhead-percent", "0.1", "--interruption-queue", "env-cluster", - "--reserved-enis", "10") + "--reserved-enis", "10", + "--max-pods-extra-capacity", "1") Expect(err).ToNot(HaveOccurred()) expectOptionsEqual(opts, test.Options(test.OptionsFields{ AssumeRoleARN: lo.ToPtr("env-role"), @@ -77,6 +78,7 @@ var _ = Describe("Options", func() { VMMemoryOverheadPercent: lo.ToPtr[float64](0.1), InterruptionQueue: lo.ToPtr("env-cluster"), ReservedENIs: lo.ToPtr(10), + MaxPodsExtraCapacity: lo.ToPtr(1), })) }) It("should correctly fallback to env vars when CLI flags aren't set", func() { @@ -89,6 +91,7 @@ var _ = Describe("Options", func() { os.Setenv("VM_MEMORY_OVERHEAD_PERCENT", "0.1") os.Setenv("INTERRUPTION_QUEUE", "env-cluster") os.Setenv("RESERVED_ENIS", "10") + os.Setenv("MAX_PODS_EXTRA_CAPACITY", "1") // Add flags after we set the environment variables so that the parsing logic correctly refers // to the new environment variable values @@ -105,6 +108,7 @@ var _ = Describe("Options", func() { VMMemoryOverheadPercent: lo.ToPtr[float64](0.1), InterruptionQueue: lo.ToPtr("env-cluster"), ReservedENIs: lo.ToPtr(10), + MaxPodsExtraCapacity: lo.ToPtr(1), })) }) @@ -132,6 +136,10 @@ var _ = Describe("Options", func() { err := opts.Parse(fs, "--cluster-name", "test-cluster", "--reserved-enis", "-1") Expect(err).To(HaveOccurred()) }) + It("should fail when maxPodsExtraCapacity is negative", func() { + err := opts.Parse(fs, "--cluster-name", "test-cluster", "--max-pods-extra-capacity", "-1") + Expect(err).To(HaveOccurred()) + }) }) }) @@ -146,4 +154,5 @@ func expectOptionsEqual(optsA *options.Options, optsB *options.Options) { Expect(optsA.VMMemoryOverheadPercent).To(Equal(optsB.VMMemoryOverheadPercent)) Expect(optsA.InterruptionQueue).To(Equal(optsB.InterruptionQueue)) Expect(optsA.ReservedENIs).To(Equal(optsB.ReservedENIs)) + Expect(optsA.MaxPodsExtraCapacity).To(Equal(optsB.MaxPodsExtraCapacity)) } diff --git a/pkg/providers/instancetype/types.go b/pkg/providers/instancetype/types.go index 6cebec2f523b..fbbff2e71aa2 100644 --- a/pkg/providers/instancetype/types.go +++ b/pkg/providers/instancetype/types.go @@ -331,7 +331,7 @@ func ENILimitedPods(ctx context.Context, info *ec2.InstanceTypeInfo) *resource.Q return resource.NewQuantity(0, resource.DecimalSI) } addressesPerInterface := *info.NetworkInfo.Ipv4AddressesPerInterface - return resources.Quantity(fmt.Sprint(usableNetworkInterfaces*(addressesPerInterface-1) + 2)) + return resources.Quantity(fmt.Sprint(usableNetworkInterfaces*(addressesPerInterface-1) + int64(options.FromContext(ctx).MaxPodsExtraCapacity))) } func privateIPv4Address(info *ec2.InstanceTypeInfo) *resource.Quantity { diff --git a/pkg/test/options.go b/pkg/test/options.go index 4657f28e9c53..585241496c2d 100644 --- a/pkg/test/options.go +++ b/pkg/test/options.go @@ -34,6 +34,7 @@ type OptionsFields struct { VMMemoryOverheadPercent *float64 InterruptionQueue *string ReservedENIs *int + MaxPodsExtraCapacity *int } func Options(overrides ...OptionsFields) *options.Options { @@ -53,5 +54,6 @@ func Options(overrides ...OptionsFields) *options.Options { VMMemoryOverheadPercent: lo.FromPtrOr(opts.VMMemoryOverheadPercent, 0.075), InterruptionQueue: lo.FromPtrOr(opts.InterruptionQueue, ""), ReservedENIs: lo.FromPtrOr(opts.ReservedENIs, 0), + MaxPodsExtraCapacity: lo.FromPtrOr(opts.MaxPodsExtraCapacity, 2), } }