From 96960f7e7ba138773810ab2c5187969b1133e40b Mon Sep 17 00:00:00 2001
From: Jason Deal <jmdeal@amazon.com>
Date: Fri, 10 Nov 2023 11:32:07 -0800
Subject: [PATCH 1/3] support EFA resource requests

---
 hack/code/instancetype_testdata_gen/main.go   |   5 +
 pkg/apis/v1beta1/labels.go                    |   1 +
 pkg/cloudprovider/cloudprovider.go            |  10 +-
 pkg/cloudprovider/suite_test.go               |  30 +++++
 .../zz_generated.describe_instance_types.go   |   9 ++
 pkg/providers/amifamily/resolver.go           |  21 +++-
 pkg/providers/instance/instance.go            |   3 +-
 pkg/providers/instance/types.go               |   7 +-
 pkg/providers/instancetype/suite_test.go      |  32 +++++
 pkg/providers/instancetype/types.go           |   9 ++
 .../launchtemplate/launchtemplate.go          |  24 ++--
 .../integration/extended_resources_test.go    | 119 ++++++++++++++++++
 12 files changed, 254 insertions(+), 16 deletions(-)

diff --git a/hack/code/instancetype_testdata_gen/main.go b/hack/code/instancetype_testdata_gen/main.go
index 881faa700372..35b684036226 100644
--- a/hack/code/instancetype_testdata_gen/main.go
+++ b/hack/code/instancetype_testdata_gen/main.go
@@ -154,6 +154,11 @@ func getInstanceTypeInfo(info *ec2.InstanceTypeInfo) string {
 		fmt.Fprintf(src, "},\n")
 	}
 	fmt.Fprintf(src, "NetworkInfo: &ec2.NetworkInfo{\n")
+	if info.NetworkInfo.EfaInfo != nil {
+		fmt.Fprintf(src, "EfaInfo: &ec2.EfaInfo{\n")
+		fmt.Fprintf(src, "MaximumEfaInterfaces: aws.Int64(%d),\n", lo.FromPtr(info.NetworkInfo.EfaInfo.MaximumEfaInterfaces))
+		fmt.Fprintf(src, "},\n")
+	}
 	fmt.Fprintf(src, "MaximumNetworkInterfaces: aws.Int64(%d),\n", lo.FromPtr(info.NetworkInfo.MaximumNetworkInterfaces))
 	fmt.Fprintf(src, "Ipv4AddressesPerInterface: aws.Int64(%d),\n", lo.FromPtr(info.NetworkInfo.Ipv4AddressesPerInterface))
 	fmt.Fprintf(src, "EncryptionInTransitSupported: aws.Bool(%t),\n", lo.FromPtr(info.NetworkInfo.EncryptionInTransitSupported))
diff --git a/pkg/apis/v1beta1/labels.go b/pkg/apis/v1beta1/labels.go
index db637ea393bd..1b86eb70726f 100644
--- a/pkg/apis/v1beta1/labels.go
+++ b/pkg/apis/v1beta1/labels.go
@@ -88,6 +88,7 @@ var (
 	ResourceHabanaGaudi        v1.ResourceName = "habana.ai/gaudi"
 	ResourceAWSPodENI          v1.ResourceName = "vpc.amazonaws.com/pod-eni"
 	ResourcePrivateIPv4Address v1.ResourceName = "vpc.amazonaws.com/PrivateIPv4Address"
+	ResourceEFA                v1.ResourceName = "vpc.amazonaws.com/efa"
 
 	LabelNodeClass = Group + "/ec2nodeclass"
 
diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go
index cfc3265ba35a..634078ad03b7 100644
--- a/pkg/cloudprovider/cloudprovider.go
+++ b/pkg/cloudprovider/cloudprovider.go
@@ -311,8 +311,14 @@ func (c *CloudProvider) instanceToNodeClaim(i *instance.Instance, instanceType *
 				labels[key] = req.Values()[0]
 			}
 		}
-		nodeClaim.Status.Capacity = functional.FilterMap(instanceType.Capacity, func(_ v1.ResourceName, v resource.Quantity) bool { return !resources.IsZero(v) })
-		nodeClaim.Status.Allocatable = functional.FilterMap(instanceType.Allocatable(), func(_ v1.ResourceName, v resource.Quantity) bool { return !resources.IsZero(v) })
+		resourceFilter := func(n v1.ResourceName, v resource.Quantity) bool {
+			if !i.EFAEnabled && n == v1beta1.ResourceEFA {
+				return false
+			}
+			return !resources.IsZero(v)
+		}
+		nodeClaim.Status.Capacity = functional.FilterMap(instanceType.Capacity, resourceFilter)
+		nodeClaim.Status.Allocatable = functional.FilterMap(instanceType.Allocatable(), resourceFilter)
 	}
 	labels[v1.LabelTopologyZone] = i.Zone
 	labels[corev1beta1.CapacityTypeLabelKey] = i.CapacityType
diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go
index 4a845bce9d83..872819708a7a 100644
--- a/pkg/cloudprovider/suite_test.go
+++ b/pkg/cloudprovider/suite_test.go
@@ -22,6 +22,7 @@ import (
 	"time"
 
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/tools/record"
 
@@ -600,4 +601,33 @@ var _ = Describe("CloudProvider", func() {
 			ExpectScheduled(ctx, env.Client, pod)
 		})
 	})
+	Context("EFA", func() {
+		It("should include vpc.amazonaws.com/efa on a nodeclaim if it requests it", func() {
+			nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{
+				{
+					Key:      v1.LabelInstanceTypeStable,
+					Operator: v1.NodeSelectorOpIn,
+					Values:   []string{"dl1.24xlarge"},
+				},
+			}
+			nodeClaim.Spec.Resources.Requests = v1.ResourceList{v1beta1.ResourceEFA: resource.MustParse("1")}
+			ExpectApplied(ctx, env.Client, nodePool, nodeClass, nodeClaim)
+			cloudProviderNodeClaim, err := cloudProvider.Create(ctx, nodeClaim)
+			Expect(err).To(BeNil())
+			Expect(lo.Keys(cloudProviderNodeClaim.Status.Allocatable)).To(ContainElement(v1beta1.ResourceEFA))
+		})
+		It("shouldn't include vpc.amazonaws.com/efa on a nodeclaim if it doesn't request it", func() {
+			nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{
+				{
+					Key:      v1.LabelInstanceTypeStable,
+					Operator: v1.NodeSelectorOpIn,
+					Values:   []string{"dl1.24xlarge"},
+				},
+			}
+			ExpectApplied(ctx, env.Client, nodePool, nodeClass, nodeClaim)
+			cloudProviderNodeClaim, err := cloudProvider.Create(ctx, nodeClaim)
+			Expect(err).To(BeNil())
+			Expect(lo.Keys(cloudProviderNodeClaim.Status.Allocatable)).ToNot(ContainElement(v1beta1.ResourceEFA))
+		})
+	})
 })
diff --git a/pkg/fake/zz_generated.describe_instance_types.go b/pkg/fake/zz_generated.describe_instance_types.go
index 0458a976133b..0d9aa45d555b 100644
--- a/pkg/fake/zz_generated.describe_instance_types.go
+++ b/pkg/fake/zz_generated.describe_instance_types.go
@@ -90,6 +90,9 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{
 				TotalSizeInGB: aws.Int64(4000),
 			},
 			NetworkInfo: &ec2.NetworkInfo{
+				EfaInfo: &ec2.EfaInfo{
+					MaximumEfaInterfaces: aws.Int64(4),
+				},
 				MaximumNetworkInterfaces:     aws.Int64(60),
 				Ipv4AddressesPerInterface:    aws.Int64(50),
 				EncryptionInTransitSupported: aws.Bool(true),
@@ -147,6 +150,9 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{
 				TotalSizeInGB: aws.Int64(900),
 			},
 			NetworkInfo: &ec2.NetworkInfo{
+				EfaInfo: &ec2.EfaInfo{
+					MaximumEfaInterfaces: aws.Int64(1),
+				},
 				MaximumNetworkInterfaces:     aws.Int64(4),
 				Ipv4AddressesPerInterface:    aws.Int64(15),
 				EncryptionInTransitSupported: aws.Bool(true),
@@ -348,6 +354,9 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{
 				TotalSizeInGB: aws.Int64(7600),
 			},
 			NetworkInfo: &ec2.NetworkInfo{
+				EfaInfo: &ec2.EfaInfo{
+					MaximumEfaInterfaces: aws.Int64(2),
+				},
 				MaximumNetworkInterfaces:     aws.Int64(14),
 				Ipv4AddressesPerInterface:    aws.Int64(50),
 				EncryptionInTransitSupported: aws.Bool(true),
diff --git a/pkg/providers/amifamily/resolver.go b/pkg/providers/amifamily/resolver.go
index ab5754dbdf5a..221fe103f745 100644
--- a/pkg/providers/amifamily/resolver.go
+++ b/pkg/providers/amifamily/resolver.go
@@ -69,6 +69,7 @@ type LaunchTemplate struct {
 	AMIID               string
 	InstanceTypes       []*cloudprovider.InstanceType `hash:"ignore"`
 	DetailedMonitoring  bool
+	EFACount            int
 }
 
 // AMIFamily can be implemented to override the default logic for generating dynamic launch template parameters
@@ -130,13 +131,24 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1beta1.EC2NodeClass,
 	}
 	var resolvedTemplates []*LaunchTemplate
 	for amiID, instanceTypes := range mappedAMIs {
-		maxPodsToInstanceTypes := lo.GroupBy(instanceTypes, func(instanceType *cloudprovider.InstanceType) int {
-			return int(instanceType.Capacity.Pods().Value())
+		type launchTemplateParams struct {
+			efaCount int
+			maxPods  int
+		}
+		paramsToInstanceTypes := lo.GroupBy(instanceTypes, func(instanceType *cloudprovider.InstanceType) launchTemplateParams {
+			return launchTemplateParams{
+				efaCount: lo.Ternary(
+					lo.Contains(lo.Keys(nodeClaim.Spec.Resources.Requests), v1beta1.ResourceEFA),
+					int(lo.ToPtr(instanceType.Capacity[v1beta1.ResourceEFA]).Value()),
+					0,
+				),
+				maxPods: int(instanceType.Capacity.Pods().Value()),
+			}
 		})
 		// In order to support reserved ENIs for CNI custom networking setups,
 		// we need to pass down the max-pods calculation to the kubelet.
 		// This requires that we resolve a unique launch template per max-pods value.
-		for maxPods, instanceTypes := range maxPodsToInstanceTypes {
+		for params, instanceTypes := range paramsToInstanceTypes {
 			kubeletConfig := &corev1beta1.KubeletConfiguration{}
 			if nodeClaim.Spec.Kubelet != nil {
 				if err := mergo.Merge(kubeletConfig, nodeClaim.Spec.Kubelet); err != nil {
@@ -144,7 +156,7 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1beta1.EC2NodeClass,
 				}
 			}
 			if kubeletConfig.MaxPods == nil {
-				kubeletConfig.MaxPods = lo.ToPtr(int32(maxPods))
+				kubeletConfig.MaxPods = lo.ToPtr(int32(params.maxPods))
 			}
 			resolved := &LaunchTemplate{
 				Options: options,
@@ -161,6 +173,7 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1beta1.EC2NodeClass,
 				DetailedMonitoring:  aws.BoolValue(nodeClass.Spec.DetailedMonitoring),
 				AMIID:               amiID,
 				InstanceTypes:       instanceTypes,
+				EFACount:            params.efaCount,
 			}
 			if len(resolved.BlockDeviceMappings) == 0 {
 				resolved.BlockDeviceMappings = amiFamily.DefaultBlockDeviceMappings()
diff --git a/pkg/providers/instance/instance.go b/pkg/providers/instance/instance.go
index cfd102ef9838..47d4b6fba0b0 100644
--- a/pkg/providers/instance/instance.go
+++ b/pkg/providers/instance/instance.go
@@ -100,7 +100,8 @@ func (p *Provider) Create(ctx context.Context, nodeClass *v1beta1.EC2NodeClass,
 	if err != nil {
 		return nil, err
 	}
-	return NewInstanceFromFleet(fleetInstance, tags), nil
+	efaEnabled := lo.Contains(lo.Keys(nodeClaim.Spec.Resources.Requests), v1beta1.ResourceEFA)
+	return NewInstanceFromFleet(fleetInstance, tags, efaEnabled), nil
 }
 
 func (p *Provider) Link(ctx context.Context, id, provisionerName string) error {
diff --git a/pkg/providers/instance/types.go b/pkg/providers/instance/types.go
index e44deeb408dd..b12a7f9a96a6 100644
--- a/pkg/providers/instance/types.go
+++ b/pkg/providers/instance/types.go
@@ -37,6 +37,7 @@ type Instance struct {
 	SecurityGroupIDs []string
 	SubnetID         string
 	Tags             map[string]string
+	EFAEnabled       bool
 }
 
 func NewInstance(out *ec2.Instance) *Instance {
@@ -53,11 +54,14 @@ func NewInstance(out *ec2.Instance) *Instance {
 		}),
 		SubnetID: aws.StringValue(out.SubnetId),
 		Tags:     lo.SliceToMap(out.Tags, func(t *ec2.Tag) (string, string) { return aws.StringValue(t.Key), aws.StringValue(t.Value) }),
+		EFAEnabled: lo.ContainsBy(out.NetworkInterfaces, func(ni *ec2.InstanceNetworkInterface) bool {
+			return ni != nil && lo.FromPtr(ni.InterfaceType) == "efa"
+		}),
 	}
 
 }
 
-func NewInstanceFromFleet(out *ec2.CreateFleetInstance, tags map[string]string) *Instance {
+func NewInstanceFromFleet(out *ec2.CreateFleetInstance, tags map[string]string, efaEnabled bool) *Instance {
 	return &Instance{
 		LaunchTime:   time.Now(), // estimate the launch time since we just launched
 		State:        ec2.StatePending,
@@ -68,5 +72,6 @@ func NewInstanceFromFleet(out *ec2.CreateFleetInstance, tags map[string]string)
 		CapacityType: aws.StringValue(out.Lifecycle),
 		SubnetID:     aws.StringValue(out.LaunchTemplateAndOverrides.Overrides.SubnetId),
 		Tags:         tags,
+		EFAEnabled:   efaEnabled,
 	}
 }
diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go
index 12b199b111e3..7a83ccc53ee4 100644
--- a/pkg/providers/instancetype/suite_test.go
+++ b/pkg/providers/instancetype/suite_test.go
@@ -623,6 +623,38 @@ var _ = Describe("InstanceTypes", func() {
 		}
 		Expect(nodeNames.Len()).To(Equal(1))
 	})
+	It("should launch instances for vpc.amazonaws.com/efa resource requests", func() {
+		nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{
+			{
+				Key:      v1.LabelInstanceTypeStable,
+				Operator: v1.NodeSelectorOpIn,
+				Values:   []string{"dl1.24xlarge"},
+			},
+		}
+		ExpectApplied(ctx, env.Client, nodePool, nodeClass)
+		pods := []*v1.Pod{
+			coretest.UnschedulablePod(coretest.PodOptions{
+				ResourceRequirements: v1.ResourceRequirements{
+					Requests: v1.ResourceList{v1beta1.ResourceEFA: resource.MustParse("1")},
+					Limits:   v1.ResourceList{v1beta1.ResourceEFA: resource.MustParse("1")},
+				},
+			}),
+			coretest.UnschedulablePod(coretest.PodOptions{
+				ResourceRequirements: v1.ResourceRequirements{
+					Requests: v1.ResourceList{v1beta1.ResourceEFA: resource.MustParse("2")},
+					Limits:   v1.ResourceList{v1beta1.ResourceEFA: resource.MustParse("2")},
+				},
+			}),
+		}
+		ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pods...)
+		nodes := sets.NewString()
+		for _, pod := range pods {
+			node := ExpectScheduled(ctx, env.Client, pod)
+			Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "dl1.24xlarge"))
+			nodes.Insert(node.Name)
+		}
+		Expect(nodes.Len()).To(Equal(1))
+	})
 	It("should not set pods to 110 if using ENI-based pod density", func() {
 		instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx)
 		Expect(err).To(BeNil())
diff --git a/pkg/providers/instancetype/types.go b/pkg/providers/instancetype/types.go
index 00458f30942c..36af3cd1900f 100644
--- a/pkg/providers/instancetype/types.go
+++ b/pkg/providers/instancetype/types.go
@@ -181,6 +181,7 @@ func computeCapacity(ctx context.Context, info *ec2.InstanceTypeInfo, amiFamily
 		v1beta1.ResourceAMDGPU:      *amdGPUs(info),
 		v1beta1.ResourceAWSNeuron:   *awsNeurons(info),
 		v1beta1.ResourceHabanaGaudi: *habanaGaudis(info),
+		v1beta1.ResourceEFA:         *efas(info),
 	}
 	return resourceList
 }
@@ -296,6 +297,14 @@ func habanaGaudis(info *ec2.InstanceTypeInfo) *resource.Quantity {
 	return resources.Quantity(fmt.Sprint(count))
 }
 
+func efas(info *ec2.InstanceTypeInfo) *resource.Quantity {
+	count := int64(0)
+	if info.NetworkInfo != nil && info.NetworkInfo.EfaInfo != nil {
+		count = lo.FromPtr(info.NetworkInfo.EfaInfo.MaximumEfaInterfaces)
+	}
+	return resources.Quantity(fmt.Sprint(count))
+}
+
 func ENILimitedPods(ctx context.Context, info *ec2.InstanceTypeInfo) *resource.Quantity {
 	// The number of pods per node is calculated using the formula:
 	// max number of ENIs * (IPv4 Addresses per ENI -1) + 2
diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go
index 0a3af6cff0d2..953e3006ecb3 100644
--- a/pkg/providers/launchtemplate/launchtemplate.go
+++ b/pkg/providers/launchtemplate/launchtemplate.go
@@ -243,6 +243,7 @@ func (p *Provider) createLaunchTemplate(ctx context.Context, capacityType string
 		launchTemplateDataTags = append(launchTemplateDataTags, &ec2.LaunchTemplateTagSpecificationRequest{ResourceType: aws.String(ec2.ResourceTypeSpotInstancesRequest), Tags: utils.MergeTags(options.Tags)})
 	}
 	networkInterface := p.generateNetworkInterface(options)
+	logging.FromContext(ctx).Debugf("Network Interfaces: %d", len(networkInterface))
 	output, err := p.ec2api.CreateLaunchTemplateWithContext(ctx, &ec2.CreateLaunchTemplateInput{
 		LaunchTemplateName: aws.String(launchTemplateName(options)),
 		LaunchTemplateData: &ec2.RequestLaunchTemplateData{
@@ -286,16 +287,23 @@ func (p *Provider) createLaunchTemplate(ctx context.Context, capacityType string
 // This is done to help comply with AWS account policies that require explicitly setting that field to 'false'.
 // https://github.com/aws/karpenter/issues/3815
 func (p *Provider) generateNetworkInterface(options *amifamily.LaunchTemplate) []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest {
-	if options.AssociatePublicIPAddress != nil {
-		return []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{
-			{
-				AssociatePublicIpAddress: options.AssociatePublicIPAddress,
-				DeviceIndex:              aws.Int64(0),
-				Groups:                   lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) }),
-			},
+	interfaces := lo.Times(options.EFACount, func(i int) *ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest {
+		return &ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{
+			AssociatePublicIpAddress: options.AssociatePublicIPAddress,
+			NetworkCardIndex:         lo.ToPtr(int64(i)),
+			DeviceIndex:              lo.ToPtr(lo.Ternary[int64](i == 0, 0, 1)),
+			InterfaceType:            lo.ToPtr("efa"),
+			Groups:                   lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) }),
 		}
+	})
+	if len(interfaces) == 0 && options.AssociatePublicIPAddress != nil {
+		interfaces = append(interfaces, &ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{
+			AssociatePublicIpAddress: options.AssociatePublicIPAddress,
+			DeviceIndex:              aws.Int64(0),
+			Groups:                   lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) }),
+		})
 	}
-	return nil
+	return interfaces
 }
 
 func (p *Provider) blockDeviceMappings(blockDeviceMappings []*v1beta1.BlockDeviceMapping) []*ec2.LaunchTemplateBlockDeviceMappingRequest {
diff --git a/test/suites/integration/extended_resources_test.go b/test/suites/integration/extended_resources_test.go
index c5b883e326bb..07eba151ac49 100644
--- a/test/suites/integration/extended_resources_test.go
+++ b/test/suites/integration/extended_resources_test.go
@@ -209,6 +209,48 @@ var _ = Describe("Extended Resources", func() {
 		env.ExpectCreatedNodeCount("==", 1)
 		env.EventuallyExpectInitializedNodeCount("==", 1)
 	})
+
+	It("should provision nodes for a deployment that requests vpc.amazonaws.com/efa", func() {
+		ExpectEFADevicePluginCreated()
+
+		nodePool.Spec.Template.Labels = map[string]string{
+			"aws.amazon.com/efa": "true",
+		}
+		nodePool.Spec.Template.Spec.Taints = []v1.Taint{
+			{
+				Key:    "aws.amazon.com/efa",
+				Effect: v1.TaintEffectNoSchedule,
+			},
+		}
+		numPods := 1
+		dep := test.Deployment(test.DeploymentOptions{
+			Replicas: int32(numPods),
+			PodOptions: test.PodOptions{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{"app": "efa-app"},
+				},
+				Tolerations: []v1.Toleration{
+					{
+						Key:      "aws.amazon.com/efa",
+						Operator: v1.TolerationOpExists,
+					},
+				},
+				ResourceRequirements: v1.ResourceRequirements{
+					Requests: v1.ResourceList{
+						"vpc.amazonaws.com/efa": resource.MustParse("1"),
+					},
+					Limits: v1.ResourceList{
+						"vpc.amazonaws.com/efa": resource.MustParse("1"),
+					},
+				},
+			},
+		})
+		selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
+		env.ExpectCreated(nodeClass, nodePool, dep)
+		env.EventuallyExpectHealthyPodCount(selector, numPods)
+		env.ExpectCreatedNodeCount("==", 1)
+		env.EventuallyExpectInitializedNodeCount("==", 1)
+	})
 })
 
 func ExpectNvidiaDevicePluginCreated() {
@@ -425,3 +467,80 @@ func ExpectHabanaDevicePluginCreated() {
 		},
 	})
 }
+
+func ExpectEFADevicePluginCreated() {
+	GinkgoHelper()
+	env.ExpectCreated(&appsv1.DaemonSet{
+		ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{
+			Name:      "aws-efa-k8s-device-plugin-daemonset",
+			Namespace: "kube-system",
+		}),
+		Spec: appsv1.DaemonSetSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"name": "aws-efa-k8s-device-plugin",
+				},
+			},
+			UpdateStrategy: appsv1.DaemonSetUpdateStrategy{
+				Type: appsv1.RollingUpdateDaemonSetStrategyType,
+			},
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{
+					Annotations: map[string]string{
+						"scheduler.alpha.kubernetes.io/critical-pod": "",
+					},
+					Labels: map[string]string{
+						"name": "aws-efa-k8s-device-plugin",
+					},
+				}),
+				Spec: v1.PodSpec{
+					NodeSelector: map[string]string{
+						"aws.amazon.com/efa": "true",
+					},
+					Tolerations: []v1.Toleration{
+						{
+							Key:      "CriticalAddonsOnly",
+							Operator: v1.TolerationOpExists,
+						},
+						{
+							Key:      "aws.amazon.com/efa",
+							Operator: v1.TolerationOpExists,
+							Effect:   v1.TaintEffectNoSchedule,
+						},
+					},
+					PriorityClassName: "system-node-critical",
+					HostNetwork:       true,
+					Containers: []v1.Container{
+						{
+							Name:  "aws-efea-k8s-device-plugin",
+							Image: "602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/aws-efa-k8s-device-plugin:v0.3.3",
+							SecurityContext: &v1.SecurityContext{
+								AllowPrivilegeEscalation: lo.ToPtr(false),
+								Capabilities: &v1.Capabilities{
+									Drop: []v1.Capability{"ALL"},
+								},
+								RunAsNonRoot: lo.ToPtr(false),
+							},
+							VolumeMounts: []v1.VolumeMount{
+								{
+									Name:      "device-plugin",
+									MountPath: "/var/lib/kubelet/device-plugins",
+								},
+							},
+						},
+					},
+					Volumes: []v1.Volume{
+						{
+							Name: "device-plugin",
+							VolumeSource: v1.VolumeSource{
+								HostPath: &v1.HostPathVolumeSource{
+									Path: "/var/lib/kubelet/device-plugins",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	})
+}

From 2cdfda91fbf74f765969445155d79ff486ca38e5 Mon Sep 17 00:00:00 2001
From: Jason Deal <jmdeal@amazon.com>
Date: Tue, 21 Nov 2023 15:32:10 -0500
Subject: [PATCH 2/3] fix nil vs empty interfaces

---
 .../launchtemplate/launchtemplate.go          | 42 ++++++++++---------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go
index 953e3006ecb3..6b9abc976c8d 100644
--- a/pkg/providers/launchtemplate/launchtemplate.go
+++ b/pkg/providers/launchtemplate/launchtemplate.go
@@ -242,8 +242,7 @@ func (p *Provider) createLaunchTemplate(ctx context.Context, capacityType string
 	if capacityType == corev1beta1.CapacityTypeSpot {
 		launchTemplateDataTags = append(launchTemplateDataTags, &ec2.LaunchTemplateTagSpecificationRequest{ResourceType: aws.String(ec2.ResourceTypeSpotInstancesRequest), Tags: utils.MergeTags(options.Tags)})
 	}
-	networkInterface := p.generateNetworkInterface(options)
-	logging.FromContext(ctx).Debugf("Network Interfaces: %d", len(networkInterface))
+	networkInterfaces := p.generateNetworkInterface(options)
 	output, err := p.ec2api.CreateLaunchTemplateWithContext(ctx, &ec2.CreateLaunchTemplateInput{
 		LaunchTemplateName: aws.String(launchTemplateName(options)),
 		LaunchTemplateData: &ec2.RequestLaunchTemplateData{
@@ -255,7 +254,7 @@ func (p *Provider) createLaunchTemplate(ctx context.Context, capacityType string
 				Enabled: aws.Bool(options.DetailedMonitoring),
 			},
 			// If the network interface is defined, the security groups are defined within it
-			SecurityGroupIds: lo.Ternary(networkInterface != nil, nil, lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) })),
+			SecurityGroupIds: lo.Ternary(networkInterfaces != nil, nil, lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) })),
 			UserData:         aws.String(userData),
 			ImageId:          aws.String(options.AMIID),
 			MetadataOptions: &ec2.LaunchTemplateInstanceMetadataOptionsRequest{
@@ -264,7 +263,7 @@ func (p *Provider) createLaunchTemplate(ctx context.Context, capacityType string
 				HttpPutResponseHopLimit: options.MetadataOptions.HTTPPutResponseHopLimit,
 				HttpTokens:              options.MetadataOptions.HTTPTokens,
 			},
-			NetworkInterfaces: networkInterface,
+			NetworkInterfaces: networkInterfaces,
 			TagSpecifications: launchTemplateDataTags,
 		},
 		TagSpecifications: []*ec2.TagSpecification{
@@ -287,23 +286,28 @@ func (p *Provider) createLaunchTemplate(ctx context.Context, capacityType string
 // This is done to help comply with AWS account policies that require explicitly setting that field to 'false'.
 // https://github.com/aws/karpenter/issues/3815
 func (p *Provider) generateNetworkInterface(options *amifamily.LaunchTemplate) []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest {
-	interfaces := lo.Times(options.EFACount, func(i int) *ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest {
-		return &ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{
-			AssociatePublicIpAddress: options.AssociatePublicIPAddress,
-			NetworkCardIndex:         lo.ToPtr(int64(i)),
-			DeviceIndex:              lo.ToPtr(lo.Ternary[int64](i == 0, 0, 1)),
-			InterfaceType:            lo.ToPtr("efa"),
-			Groups:                   lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) }),
-		}
-	})
-	if len(interfaces) == 0 && options.AssociatePublicIPAddress != nil {
-		interfaces = append(interfaces, &ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{
-			AssociatePublicIpAddress: options.AssociatePublicIPAddress,
-			DeviceIndex:              aws.Int64(0),
-			Groups:                   lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) }),
+	if options.EFACount != 0 {
+		return lo.Times(options.EFACount, func(i int) *ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest {
+			return &ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{
+				// You may only associate a public IP if there is a single network interface.
+				AssociatePublicIpAddress: lo.Ternary(options.EFACount == 1, options.AssociatePublicIPAddress, nil),
+				NetworkCardIndex:         lo.ToPtr(int64(i)),
+				DeviceIndex:              lo.ToPtr(lo.Ternary[int64](i == 0, 0, 1)),
+				InterfaceType:            lo.ToPtr("efa"),
+				Groups:                   lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) }),
+			}
 		})
 	}
-	return interfaces
+	if options.AssociatePublicIPAddress != nil {
+		return []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{
+			{
+				AssociatePublicIpAddress: options.AssociatePublicIPAddress,
+				DeviceIndex:              aws.Int64(0),
+				Groups:                   lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) }),
+			},
+		}
+	}
+	return nil
 }
 
 func (p *Provider) blockDeviceMappings(blockDeviceMappings []*v1beta1.BlockDeviceMapping) []*ec2.LaunchTemplateBlockDeviceMappingRequest {

From af5a4031a6afd5b4807893b2267b1355b2163b51 Mon Sep 17 00:00:00 2001
From: Jason Deal <jmdeal@amazon.com>
Date: Wed, 29 Nov 2023 00:52:43 -0500
Subject: [PATCH 3/3] gh comments

---
 pkg/cloudprovider/cloudprovider.go            |  9 +-
 pkg/providers/amifamily/resolver.go           | 79 ++++++++-------
 pkg/providers/instance/types.go               |  2 +-
 .../launchtemplate/launchtemplate.go          | 27 ++---
 .../integration/extended_resources_test.go    |  7 +-
 .../en/preview/reference/instance-types.md    | 99 +++++++++++++++++++
 .../content/en/preview/reference/settings.md  |  2 +-
 7 files changed, 172 insertions(+), 53 deletions(-)

diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go
index 634078ad03b7..701cf8a132cc 100644
--- a/pkg/cloudprovider/cloudprovider.go
+++ b/pkg/cloudprovider/cloudprovider.go
@@ -312,10 +312,15 @@ func (c *CloudProvider) instanceToNodeClaim(i *instance.Instance, instanceType *
 			}
 		}
 		resourceFilter := func(n v1.ResourceName, v resource.Quantity) bool {
-			if !i.EFAEnabled && n == v1beta1.ResourceEFA {
+			if resources.IsZero(v) {
 				return false
 			}
-			return !resources.IsZero(v)
+			// The nodeclaim should only advertise an EFA resource if it was requested. EFA network interfaces are only
+			// added to the launch template if they're requested, otherwise the instance is launched with a normal ENI.
+			if n == v1beta1.ResourceEFA {
+				return i.EFAEnabled
+			}
+			return true
 		}
 		nodeClaim.Status.Capacity = functional.FilterMap(instanceType.Capacity, resourceFilter)
 		nodeClaim.Status.Allocatable = functional.FilterMap(instanceType.Allocatable(), resourceFilter)
diff --git a/pkg/providers/amifamily/resolver.go b/pkg/providers/amifamily/resolver.go
index 221fe103f745..5b48613b34dd 100644
--- a/pkg/providers/amifamily/resolver.go
+++ b/pkg/providers/amifamily/resolver.go
@@ -131,6 +131,11 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1beta1.EC2NodeClass,
 	}
 	var resolvedTemplates []*LaunchTemplate
 	for amiID, instanceTypes := range mappedAMIs {
+		// In order to support reserved ENIs for CNI custom networking setups,
+		// we need to pass down the max-pods calculation to the kubelet.
+		// This requires that we resolve a unique launch template per max-pods value.
+		// Similarly, instance types configured with EfAs require unique launch templates depending on the number of
+		// EFAs they support.
 		type launchTemplateParams struct {
 			efaCount int
 			maxPods  int
@@ -145,41 +150,10 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1beta1.EC2NodeClass,
 				maxPods: int(instanceType.Capacity.Pods().Value()),
 			}
 		})
-		// In order to support reserved ENIs for CNI custom networking setups,
-		// we need to pass down the max-pods calculation to the kubelet.
-		// This requires that we resolve a unique launch template per max-pods value.
 		for params, instanceTypes := range paramsToInstanceTypes {
-			kubeletConfig := &corev1beta1.KubeletConfiguration{}
-			if nodeClaim.Spec.Kubelet != nil {
-				if err := mergo.Merge(kubeletConfig, nodeClaim.Spec.Kubelet); err != nil {
-					return nil, err
-				}
-			}
-			if kubeletConfig.MaxPods == nil {
-				kubeletConfig.MaxPods = lo.ToPtr(int32(params.maxPods))
-			}
-			resolved := &LaunchTemplate{
-				Options: options,
-				UserData: amiFamily.UserData(
-					r.defaultClusterDNS(options, kubeletConfig),
-					append(nodeClaim.Spec.Taints, nodeClaim.Spec.StartupTaints...),
-					options.Labels,
-					options.CABundle,
-					instanceTypes,
-					nodeClass.Spec.UserData,
-				),
-				BlockDeviceMappings: nodeClass.Spec.BlockDeviceMappings,
-				MetadataOptions:     nodeClass.Spec.MetadataOptions,
-				DetailedMonitoring:  aws.BoolValue(nodeClass.Spec.DetailedMonitoring),
-				AMIID:               amiID,
-				InstanceTypes:       instanceTypes,
-				EFACount:            params.efaCount,
-			}
-			if len(resolved.BlockDeviceMappings) == 0 {
-				resolved.BlockDeviceMappings = amiFamily.DefaultBlockDeviceMappings()
-			}
-			if resolved.MetadataOptions == nil {
-				resolved.MetadataOptions = amiFamily.DefaultMetadataOptions()
+			resolved, err := r.resolveLaunchTemplate(nodeClass, nodeClaim, instanceTypes, amiFamily, amiID, params.maxPods, params.efaCount, options)
+			if err != nil {
+				return nil, err
 			}
 			resolvedTemplates = append(resolvedTemplates, resolved)
 		}
@@ -229,3 +203,40 @@ func (r Resolver) defaultClusterDNS(opts *Options, kubeletConfig *corev1beta1.Ku
 	newKubeletConfig.ClusterDNS = []string{opts.KubeDNSIP.String()}
 	return newKubeletConfig
 }
+
+func (r Resolver) resolveLaunchTemplate(nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType,
+	amiFamily AMIFamily, amiID string, maxPods int, efaCount int, options *Options) (*LaunchTemplate, error) {
+	kubeletConfig := &corev1beta1.KubeletConfiguration{}
+	if nodeClaim.Spec.Kubelet != nil {
+		if err := mergo.Merge(kubeletConfig, nodeClaim.Spec.Kubelet); err != nil {
+			return nil, err
+		}
+	}
+	if kubeletConfig.MaxPods == nil {
+		kubeletConfig.MaxPods = lo.ToPtr(int32(maxPods))
+	}
+	resolved := &LaunchTemplate{
+		Options: options,
+		UserData: amiFamily.UserData(
+			r.defaultClusterDNS(options, kubeletConfig),
+			append(nodeClaim.Spec.Taints, nodeClaim.Spec.StartupTaints...),
+			options.Labels,
+			options.CABundle,
+			instanceTypes,
+			nodeClass.Spec.UserData,
+		),
+		BlockDeviceMappings: nodeClass.Spec.BlockDeviceMappings,
+		MetadataOptions:     nodeClass.Spec.MetadataOptions,
+		DetailedMonitoring:  aws.BoolValue(nodeClass.Spec.DetailedMonitoring),
+		AMIID:               amiID,
+		InstanceTypes:       instanceTypes,
+		EFACount:            efaCount,
+	}
+	if len(resolved.BlockDeviceMappings) == 0 {
+		resolved.BlockDeviceMappings = amiFamily.DefaultBlockDeviceMappings()
+	}
+	if resolved.MetadataOptions == nil {
+		resolved.MetadataOptions = amiFamily.DefaultMetadataOptions()
+	}
+	return resolved, nil
+}
diff --git a/pkg/providers/instance/types.go b/pkg/providers/instance/types.go
index b12a7f9a96a6..5f3804f2d004 100644
--- a/pkg/providers/instance/types.go
+++ b/pkg/providers/instance/types.go
@@ -55,7 +55,7 @@ func NewInstance(out *ec2.Instance) *Instance {
 		SubnetID: aws.StringValue(out.SubnetId),
 		Tags:     lo.SliceToMap(out.Tags, func(t *ec2.Tag) (string, string) { return aws.StringValue(t.Key), aws.StringValue(t.Value) }),
 		EFAEnabled: lo.ContainsBy(out.NetworkInterfaces, func(ni *ec2.InstanceNetworkInterface) bool {
-			return ni != nil && lo.FromPtr(ni.InterfaceType) == "efa"
+			return ni != nil && lo.FromPtr(ni.InterfaceType) == ec2.NetworkInterfaceTypeEfa
 		}),
 	}
 
diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go
index 6b9abc976c8d..56a0a4978881 100644
--- a/pkg/providers/launchtemplate/launchtemplate.go
+++ b/pkg/providers/launchtemplate/launchtemplate.go
@@ -242,7 +242,7 @@ func (p *Provider) createLaunchTemplate(ctx context.Context, capacityType string
 	if capacityType == corev1beta1.CapacityTypeSpot {
 		launchTemplateDataTags = append(launchTemplateDataTags, &ec2.LaunchTemplateTagSpecificationRequest{ResourceType: aws.String(ec2.ResourceTypeSpotInstancesRequest), Tags: utils.MergeTags(options.Tags)})
 	}
-	networkInterfaces := p.generateNetworkInterface(options)
+	networkInterfaces := p.generateNetworkInterfaces(options)
 	output, err := p.ec2api.CreateLaunchTemplateWithContext(ctx, &ec2.CreateLaunchTemplateInput{
 		LaunchTemplateName: aws.String(launchTemplateName(options)),
 		LaunchTemplateData: &ec2.RequestLaunchTemplateData{
@@ -280,24 +280,25 @@ func (p *Provider) createLaunchTemplate(ctx context.Context, capacityType string
 	return output.LaunchTemplate, nil
 }
 
-// generateNetworkInterface generates a network interface for the launch template.
-// If all referenced subnets do not assign public IPv4 addresses to EC2 instances therein, we explicitly set
-// AssociatePublicIpAddress to 'false' in the Launch Template, generated based on this configuration struct.
-// This is done to help comply with AWS account policies that require explicitly setting that field to 'false'.
-// https://github.com/aws/karpenter/issues/3815
-func (p *Provider) generateNetworkInterface(options *amifamily.LaunchTemplate) []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest {
+// generateNetworkInterfaces generates network interfaces for the launch template.
+func (p *Provider) generateNetworkInterfaces(options *amifamily.LaunchTemplate) []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest {
 	if options.EFACount != 0 {
 		return lo.Times(options.EFACount, func(i int) *ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest {
 			return &ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{
-				// You may only associate a public IP if there is a single network interface.
-				AssociatePublicIpAddress: lo.Ternary(options.EFACount == 1, options.AssociatePublicIPAddress, nil),
-				NetworkCardIndex:         lo.ToPtr(int64(i)),
-				DeviceIndex:              lo.ToPtr(lo.Ternary[int64](i == 0, 0, 1)),
-				InterfaceType:            lo.ToPtr("efa"),
-				Groups:                   lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) }),
+				NetworkCardIndex: lo.ToPtr(int64(i)),
+				// Some networking magic to ensure that one network card has higher priority than all the others (important if an instance needs a public IP w/o adding an EIP to every network card)
+				DeviceIndex:   lo.ToPtr(lo.Ternary[int64](i == 0, 0, 1)),
+				InterfaceType: lo.ToPtr(ec2.NetworkInterfaceTypeEfa),
+				Groups:        lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) }),
 			}
 		})
 	}
+
+	// If all referenced subnets do not assign public IPv4 addresses to EC2 instances therein, we explicitly set
+	// AssociatePublicIpAddress to 'false' in the Launch Template, generated based on this configuration struct.
+	// This is done to help comply with AWS account policies that require explicitly setting that field to 'false'.
+	// This is ignored for EFA instances since it can't be specified if you launch with multiple network interfaces.
+	// https://github.com/aws/karpenter/issues/3815
 	if options.AssociatePublicIPAddress != nil {
 		return []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{
 			{
diff --git a/test/suites/integration/extended_resources_test.go b/test/suites/integration/extended_resources_test.go
index 07eba151ac49..771d1c65df08 100644
--- a/test/suites/integration/extended_resources_test.go
+++ b/test/suites/integration/extended_resources_test.go
@@ -222,6 +222,9 @@ var _ = Describe("Extended Resources", func() {
 				Effect: v1.TaintEffectNoSchedule,
 			},
 		}
+		// Only select private subnets since instances with multiple network instances at launch won't get a public IP.
+		nodeClass.Spec.SubnetSelectorTerms[0].Tags["Name"] = "*Private*"
+
 		numPods := 1
 		dep := test.Deployment(test.DeploymentOptions{
 			Replicas: int32(numPods),
@@ -237,10 +240,10 @@ var _ = Describe("Extended Resources", func() {
 				},
 				ResourceRequirements: v1.ResourceRequirements{
 					Requests: v1.ResourceList{
-						"vpc.amazonaws.com/efa": resource.MustParse("1"),
+						"vpc.amazonaws.com/efa": resource.MustParse("2"),
 					},
 					Limits: v1.ResourceList{
-						"vpc.amazonaws.com/efa": resource.MustParse("1"),
+						"vpc.amazonaws.com/efa": resource.MustParse("2"),
 					},
 				},
 			},
diff --git a/website/content/en/preview/reference/instance-types.md b/website/content/en/preview/reference/instance-types.md
index eaccc5712af7..44bc7b15b0f6 100644
--- a/website/content/en/preview/reference/instance-types.md
+++ b/website/content/en/preview/reference/instance-types.md
@@ -1383,6 +1383,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|88002Mi|
  |pods|234|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|54|
 ### `c5n.18xlarge`
 #### Labels
@@ -1407,6 +1408,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|173400Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `c5n.metal`
 #### Labels
@@ -1431,6 +1433,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|173400Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## c6a Family
 ### `c6a.large`
@@ -1672,6 +1675,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|355262Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `c6a.metal`
 #### Labels
@@ -1696,6 +1700,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|355262Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## c6g Family
 ### `c6g.medium`
@@ -2332,6 +2337,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|112720Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## c6i Family
 ### `c6i.large`
@@ -2549,6 +2555,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|234021Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `c6i.metal`
 #### Labels
@@ -2573,6 +2580,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|234021Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## c6id Family
 ### `c6id.large`
@@ -2799,6 +2807,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|234021Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `c6id.metal`
 #### Labels
@@ -2824,6 +2833,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|234021Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## c6in Family
 ### `c6in.large`
@@ -3041,6 +3051,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|238333Mi|
  |pods|345|
+ |vpc.amazonaws.com/efa|2|
  |vpc.amazonaws.com/pod-eni|108|
 ### `c6in.metal`
 #### Labels
@@ -3065,6 +3076,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|238333Mi|
  |pods|345|
+ |vpc.amazonaws.com/efa|2|
  |vpc.amazonaws.com/pod-eni|108|
 ## c7a Family
 ### `c7a.medium`
@@ -3309,6 +3321,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|355262Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
 ### `c7a.metal-48xl`
 #### Labels
  | Label | Value |
@@ -3331,6 +3344,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|355262Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
 ## c7g Family
 ### `c7g.medium`
 #### Labels
@@ -3523,6 +3537,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|112720Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `c7g.metal`
 #### Labels
@@ -3547,6 +3562,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|112720Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## c7gd Family
 ### `c7gd.medium`
@@ -3748,6 +3764,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|112720Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## c7gn Family
 ### `c7gn.medium`
@@ -3941,6 +3958,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|112720Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## c7i Family
 ### `c7i.large`
@@ -4172,6 +4190,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|355262Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
 ### `c7i.metal-48xl`
 #### Labels
  | Label | Value |
@@ -4194,6 +4213,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|355262Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
 ## d2 Family
 ### `d2.xlarge`
 #### Labels
@@ -4566,6 +4586,7 @@ below are the resources available with some assumptions and after the instance o
  |habana.ai/gaudi|8|
  |memory|718987Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|4|
  |vpc.amazonaws.com/pod-eni|62|
 ## f1 Family
 ### `f1.2xlarge`
@@ -5024,6 +5045,7 @@ below are the resources available with some assumptions and after the instance o
  |memory|120248Mi|
  |nvidia.com/gpu|1|
  |pods|58|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|58|
 ### `g4dn.12xlarge`
 #### Labels
@@ -5054,6 +5076,7 @@ below are the resources available with some assumptions and after the instance o
  |memory|178933Mi|
  |nvidia.com/gpu|4|
  |pods|234|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|54|
 ### `g4dn.16xlarge`
 #### Labels
@@ -5084,6 +5107,7 @@ below are the resources available with some assumptions and after the instance o
  |memory|241490Mi|
  |nvidia.com/gpu|1|
  |pods|58|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|118|
 ### `g4dn.metal`
 #### Labels
@@ -5114,6 +5138,7 @@ below are the resources available with some assumptions and after the instance o
  |memory|355262Mi|
  |nvidia.com/gpu|8|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## g5 Family
 ### `g5.xlarge`
@@ -5235,6 +5260,7 @@ below are the resources available with some assumptions and after the instance o
  |memory|118312Mi|
  |nvidia.com/gpu|1|
  |pods|234|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|84|
 ### `g5.12xlarge`
 #### Labels
@@ -5265,6 +5291,7 @@ below are the resources available with some assumptions and after the instance o
  |memory|173400Mi|
  |nvidia.com/gpu|4|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `g5.16xlarge`
 #### Labels
@@ -5295,6 +5322,7 @@ below are the resources available with some assumptions and after the instance o
  |memory|239554Mi|
  |nvidia.com/gpu|1|
  |pods|234|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|114|
 ### `g5.24xlarge`
 #### Labels
@@ -5325,6 +5353,7 @@ below are the resources available with some assumptions and after the instance o
  |memory|355262Mi|
  |nvidia.com/gpu|4|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `g5.48xlarge`
 #### Labels
@@ -5355,6 +5384,7 @@ below are the resources available with some assumptions and after the instance o
  |memory|723299Mi|
  |nvidia.com/gpu|8|
  |pods|345|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|115|
 ## g5g Family
 ### `g5g.xlarge`
@@ -5648,6 +5678,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|118649Mi|
  |pods|198|
+ |vpc.amazonaws.com/efa|1|
 ### `hpc7g.8xlarge`
 #### Labels
  | Label | Value |
@@ -5671,6 +5702,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|118649Mi|
  |pods|198|
+ |vpc.amazonaws.com/efa|1|
 ### `hpc7g.16xlarge`
 #### Labels
  | Label | Value |
@@ -5694,6 +5726,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|118649Mi|
  |pods|198|
+ |vpc.amazonaws.com/efa|1|
 ## i2 Family
 ### `i2.xlarge`
 #### Labels
@@ -6104,6 +6137,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|360795Mi|
  |pods|234|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|54|
 ### `i3en.24xlarge`
 #### Labels
@@ -6129,6 +6163,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|718987Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `i3en.metal`
 #### Labels
@@ -6154,6 +6189,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|718987Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## i4g Family
 ### `i4g.large`
@@ -6305,6 +6341,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|476445Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## i4i Family
 ### `i4i.large`
@@ -6526,6 +6563,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|961470Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|120|
 ### `i4i.metal`
 #### Labels
@@ -6551,6 +6589,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|961470Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|120|
 ## im4gn Family
 ### `im4gn.large`
@@ -6702,6 +6741,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|233962Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## inf1 Family
 ### `inf1.xlarge`
@@ -6815,6 +6855,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|177976Mi|
  |pods|321|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|111|
 ## inf2 Family
 ### `inf2.xlarge`
@@ -8497,6 +8538,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|355262Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `m5dn.metal`
 #### Labels
@@ -8522,6 +8564,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|355262Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## m5n Family
 ### `m5n.large`
@@ -8715,6 +8758,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|355262Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `m5n.metal`
 #### Labels
@@ -8739,6 +8783,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|355262Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## m5zn Family
 ### `m5zn.large`
@@ -8884,6 +8929,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|173400Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `m5zn.metal`
 #### Labels
@@ -8908,6 +8954,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|173400Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## m6a Family
 ### `m6a.large`
@@ -9149,6 +9196,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|718987Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `m6a.metal`
 #### Labels
@@ -9173,6 +9221,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|718987Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## m6g Family
 ### `m6g.medium`
@@ -9833,6 +9882,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|476504Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `m6i.metal`
 #### Labels
@@ -9857,6 +9907,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|476504Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## m6id Family
 ### `m6id.large`
@@ -10083,6 +10134,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|476504Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `m6id.metal`
 #### Labels
@@ -10108,6 +10160,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|476504Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## m6idn Family
 ### `m6idn.large`
@@ -10334,6 +10387,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|480816Mi|
  |pods|345|
+ |vpc.amazonaws.com/efa|2|
  |vpc.amazonaws.com/pod-eni|108|
 ### `m6idn.metal`
 #### Labels
@@ -10359,6 +10413,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|480816Mi|
  |pods|345|
+ |vpc.amazonaws.com/efa|2|
  |vpc.amazonaws.com/pod-eni|108|
 ## m6in Family
 ### `m6in.large`
@@ -10576,6 +10631,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|480816Mi|
  |pods|345|
+ |vpc.amazonaws.com/efa|2|
  |vpc.amazonaws.com/pod-eni|108|
 ### `m6in.metal`
 #### Labels
@@ -10600,6 +10656,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|480816Mi|
  |pods|345|
+ |vpc.amazonaws.com/efa|2|
  |vpc.amazonaws.com/pod-eni|108|
 ## m7a Family
 ### `m7a.medium`
@@ -10865,6 +10922,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|718987Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `m7a.metal-48xl`
 #### Labels
@@ -10889,6 +10947,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|718987Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## m7g Family
 ### `m7g.medium`
@@ -11082,6 +11141,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|233962Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `m7g.metal`
 #### Labels
@@ -11106,6 +11166,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|233962Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## m7gd Family
 ### `m7gd.medium`
@@ -11307,6 +11368,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|233962Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## m7i Family
 ### `m7i.large`
@@ -11546,6 +11608,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|718987Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `m7i.metal-48xl`
 #### Labels
@@ -11569,6 +11632,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|718987Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
 ## m7i-flex Family
 ### `m7i-flex.large`
 #### Labels
@@ -11891,6 +11955,7 @@ below are the resources available with some assumptions and after the instance o
  |memory|718987Mi|
  |nvidia.com/gpu|8|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## p4d Family
 ### `p4d.24xlarge`
@@ -11922,6 +11987,7 @@ below are the resources available with some assumptions and after the instance o
  |memory|1082712Mi|
  |nvidia.com/gpu|8|
  |pods|737|
+ |vpc.amazonaws.com/efa|4|
  |vpc.amazonaws.com/pod-eni|62|
 ## p5 Family
 ### `p5.48xlarge`
@@ -11953,6 +12019,7 @@ below are the resources available with some assumptions and after the instance o
  |memory|1938410Mi|
  |nvidia.com/gpu|8|
  |pods|100|
+ |vpc.amazonaws.com/efa|32|
  |vpc.amazonaws.com/pod-eni|120|
 ## r3 Family
 ### `r3.large`
@@ -13459,6 +13526,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|718987Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `r5dn.metal`
 #### Labels
@@ -13484,6 +13552,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|718987Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## r5n Family
 ### `r5n.large`
@@ -13677,6 +13746,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|718987Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `r5n.metal`
 #### Labels
@@ -13701,6 +13771,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|718987Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## r6a Family
 ### `r6a.large`
@@ -13942,6 +14013,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|1446437Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `r6a.metal`
 #### Labels
@@ -13966,6 +14038,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|1446437Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## r6g Family
 ### `r6g.medium`
@@ -14626,6 +14699,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|961470Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `r6i.metal`
 #### Labels
@@ -14650,6 +14724,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|961470Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## r6id Family
 ### `r6id.large`
@@ -14876,6 +14951,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|961470Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `r6id.metal`
 #### Labels
@@ -14901,6 +14977,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|961470Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## r6idn Family
 ### `r6idn.large`
@@ -15127,6 +15204,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|965782Mi|
  |pods|345|
+ |vpc.amazonaws.com/efa|2|
  |vpc.amazonaws.com/pod-eni|108|
 ### `r6idn.metal`
 #### Labels
@@ -15152,6 +15230,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|965782Mi|
  |pods|345|
+ |vpc.amazonaws.com/efa|2|
  |vpc.amazonaws.com/pod-eni|108|
 ## r6in Family
 ### `r6in.large`
@@ -15369,6 +15448,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|965782Mi|
  |pods|345|
+ |vpc.amazonaws.com/efa|2|
  |vpc.amazonaws.com/pod-eni|108|
 ### `r6in.metal`
 #### Labels
@@ -15393,6 +15473,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|965782Mi|
  |pods|345|
+ |vpc.amazonaws.com/efa|2|
  |vpc.amazonaws.com/pod-eni|108|
 ## r7a Family
 ### `r7a.medium`
@@ -15648,6 +15729,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|1446437Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
 ### `r7a.metal-48xl`
 #### Labels
  | Label | Value |
@@ -15670,6 +15752,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|1446437Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
 ## r7g Family
 ### `r7g.medium`
 #### Labels
@@ -15862,6 +15945,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|476445Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `r7g.metal`
 #### Labels
@@ -15886,6 +15970,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|476445Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## r7gd Family
 ### `r7gd.medium`
@@ -16087,6 +16172,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|476445Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## r7i Family
 ### `r7i.large`
@@ -16309,6 +16395,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|1446437Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
 ### `r7i.metal-48xl`
 #### Labels
  | Label | Value |
@@ -16331,6 +16418,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|1446437Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
 ## r7iz Family
 ### `r7iz.large`
 #### Labels
@@ -16538,6 +16626,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|961470Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
 ### `r7iz.metal-32xl`
 #### Labels
  | Label | Value |
@@ -16560,6 +16649,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|961470Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
 ## t1 Family
 ### `t1.micro`
 #### Labels
@@ -17282,6 +17372,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|481894Mi|
  |pods|247|
+ |vpc.amazonaws.com/efa|8|
  |vpc.amazonaws.com/pod-eni|82|
 ## trn1n Family
 ### `trn1n.32xlarge`
@@ -17312,6 +17403,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|481894Mi|
  |pods|247|
+ |vpc.amazonaws.com/efa|16|
  |vpc.amazonaws.com/pod-eni|120|
 ## u-12tb1 Family
 ### `u-12tb1.112xlarge`
@@ -17553,6 +17645,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|173400Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## x1 Family
 ### `x1.16xlarge`
@@ -18041,6 +18134,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|1931403Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `x2idn.metal`
 #### Labels
@@ -18066,6 +18160,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|1931403Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## x2iedn Family
 ### `x2iedn.xlarge`
@@ -18242,6 +18337,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|3871269Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `x2iedn.metal`
 #### Labels
@@ -18267,6 +18363,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|3871269Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## x2iezn Family
 ### `x2iezn.2xlarge`
@@ -18388,6 +18485,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|1446437Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ### `x2iezn.metal`
 #### Labels
@@ -18412,6 +18510,7 @@ below are the resources available with some assumptions and after the instance o
  |ephemeral-storage|17Gi|
  |memory|1446437Mi|
  |pods|737|
+ |vpc.amazonaws.com/efa|1|
  |vpc.amazonaws.com/pod-eni|107|
 ## z1d Family
 ### `z1d.large`
diff --git a/website/content/en/preview/reference/settings.md b/website/content/en/preview/reference/settings.md
index dbbf59d44d2a..438c24244d26 100644
--- a/website/content/en/preview/reference/settings.md
+++ b/website/content/en/preview/reference/settings.md
@@ -64,4 +64,4 @@ This value is expressed as a string value like `10s`, `1m` or `2h45m`. The valid
 
 The batch max duration is the maximum period of time a batching window can be extended to. Increasing this value will allow the maximum batch window size to increase to collect more pending pods into a single batch at the expense of a longer delay from when the first pending pod was created.
 
-This value is expressed as a string value like `10s`, `1m` or `2h45m`. The valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`.
\ No newline at end of file
+This value is expressed as a string value like `10s`, `1m` or `2h45m`. The valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`.