feat: add networkInterfaces configuration to launchTemplate

Signed-off-by: Mahmoud Gaballah <[email protected]>

pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml

@@ -192,6 +192,22 @@ spec:
                       credentials are not available."
                     type: string
                 type: object
+              networkInterfaces:
+                items:
+                  properties:
+                    associatePublicIPAddress:
+                      description: Associates a public IPv4 address with eth0 for
+                        a new network interface.
+                      type: boolean
+                    description:
+                      description: A description for the network interface.
+                      type: string
+                    deviceIndex:
+                      description: The device index for the network interface attachment.
+                      format: int64
+                      type: integer
+                  type: object
+                type: array
               securityGroupSelector:
                 additionalProperties:
                   type: string

pkg/apis/v1alpha1/provider.go

@@ -73,6 +73,8 @@ type LaunchTemplate struct {
 	// BlockDeviceMappings to be applied to provisioned nodes.
 	// +optionals
 	BlockDeviceMappings []*BlockDeviceMapping `json:"blockDeviceMappings,omitempty"`
+
+	NetworkInterfaces []*NetworkInterface `json:"networkInterfaces,omitempty"`
 }
 
 // MetadataOptions contains parameters for specifying the exposure of the
@@ -196,3 +198,15 @@ func DeserializeProvider(raw []byte) (*AWS, error) {
 	}
 	return a, nil
 }
+
+type NetworkInterface struct {
+
+	// Associates a public IPv4 address with eth0 for a new network interface.
+	AssociatePublicIPAddress *bool `json:"associatePublicIPAddress,omitempty"`
+
+	// A description for the network interface.
+	Description *string `json:"description,omitempty"`
+
+	// The device index for the network interface attachment.
+	DeviceIndex *int64 `json:"deviceIndex,omitempty"`
+}

pkg/providers/amifamily/resolver.go

@@ -69,6 +69,7 @@ type LaunchTemplate struct {
 	AMIID               string
 	InstanceTypes       []*cloudprovider.InstanceType `hash:"ignore"`
 	DetailedMonitoring  bool
+	NetworkInterfaces   []*v1alpha1.NetworkInterface
 }
 
 // AMIFamily can be implemented to override the default logic for generating dynamic launch template parameters
@@ -161,6 +162,7 @@ func (r Resolver) Resolve(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTem
 				DetailedMonitoring:  aws.BoolValue(nodeTemplate.Spec.DetailedMonitoring),
 				AMIID:               amiID,
 				InstanceTypes:       instanceTypes,
+				NetworkInterfaces:   nodeTemplate.Spec.NetworkInterfaces,
 			}
 			if resolved.BlockDeviceMappings == nil {
 				resolved.BlockDeviceMappings = amiFamily.DefaultBlockDeviceMappings()

pkg/providers/launchtemplate/launchtemplate.go

@@ -214,7 +214,7 @@ func (p *Provider) createLaunchTemplate(ctx context.Context, options *amifamily.
 	if err != nil {
 		return nil, err
 	}
-	networkInterface := p.generateNetworkInterface(options)
+	networkInterface := p.generateNetworkInterfaces(options)
 	output, err := p.ec2api.CreateLaunchTemplateWithContext(ctx, &ec2.CreateLaunchTemplateInput{
 		LaunchTemplateName: aws.String(launchTemplateName(options)),
 		LaunchTemplateData: &ec2.RequestLaunchTemplateData{
@@ -254,22 +254,34 @@ func (p *Provider) createLaunchTemplate(ctx context.Context, options *amifamily.
 	return output.LaunchTemplate, nil
 }
 
-// generateNetworkInterface generates a network interface for the launch template.
+// generateNetworkInterfaces generates a network interface for the launch template.
 // If all referenced subnets do not assign public IPv4 addresses to EC2 instances therein, we explicitly set
 // AssociatePublicIpAddress to 'false' in the Launch Template, generated based on this configuration struct.
 // This is done to help comply with AWS account policies that require explicitly setting that field to 'false'.
 // https://github.com/aws/karpenter/issues/3815
-func (p *Provider) generateNetworkInterface(options *amifamily.LaunchTemplate) []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest {
-	if options.AssociatePublicIPAddress != nil {
-		return []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{
-			{
-				AssociatePublicIpAddress: options.AssociatePublicIPAddress,
-				DeviceIndex:              aws.Int64(0),
-				Groups:                   lo.Map(options.SecurityGroups, func(s v1alpha1.SecurityGroup, _ int) *string { return aws.String(s.ID) }),
-			},
+func (p *Provider) generateNetworkInterfaces(options *amifamily.LaunchTemplate) []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest {
+	if len(options.NetworkInterfaces) == 0 {
+		if options.AssociatePublicIPAddress != nil {
+			return []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{
+				{
+					AssociatePublicIpAddress: options.AssociatePublicIPAddress,
+					DeviceIndex:              aws.Int64(0),
+					Groups:                   lo.Map(options.SecurityGroups, func(s v1alpha1.SecurityGroup, _ int) *string { return aws.String(s.ID) }),
+				},
+			}
 		}
+		return nil
 	}
-	return nil
+
+	var networkInterfacesRequest []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest
+	for _, networkInterface := range options.NetworkInterfaces {
+		networkInterfacesRequest = append(networkInterfacesRequest, &ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{
+			AssociatePublicIpAddress: networkInterface.AssociatePublicIPAddress,
+			Groups:                   lo.Map(options.SecurityGroups, func(s v1alpha1.SecurityGroup, _ int) *string { return aws.String(s.ID) }),
+		})
+	}
+	return networkInterfacesRequest
+
 }
 
 func (p *Provider) blockDeviceMappings(blockDeviceMappings []*v1alpha1.BlockDeviceMapping) []*ec2.LaunchTemplateBlockDeviceMappingRequest {

pkg/providers/launchtemplate/suite_test.go

@@ -1642,25 +1642,95 @@ var _ = Describe("LaunchTemplates", func() {
 				Expect(*input.LaunchTemplateData.ImageId).To(ContainSubstring("test-ami"))
 			})
 		})
-		Context("Subnet-based Launch Template Configration", func() {
-			It("should explicitly set 'AssignPublicIPv4' to false in the Launch Template", func() {
+		Context("NetworkInterfaces Configuration of launch template", func() {
+			It("should explicitly set 'AssignPublicIPv4' to false when all subnets are private", func() {
 				nodeTemplate.Spec.SubnetSelector = map[string]string{"Name": "test-subnet-1,test-subnet-3"}
 				ExpectApplied(ctx, env.Client, provisioner, nodeTemplate)
 				pod := coretest.UnschedulablePod()
 				ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod)
 				ExpectScheduled(ctx, env.Client, pod)
 				input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop()
 				Expect(*input.LaunchTemplateData.NetworkInterfaces[0].AssociatePublicIpAddress).To(BeFalse())
+				Expect(len(input.LaunchTemplateData.SecurityGroupIds)).To(BeNumerically("==", 0))
+			})
+			It("should overwrite 'AssignPublicIPv4' to true when specified by user in the AWSNodeTemplate.NetworkInterfaces", func() {
+				nodeTemplate.Spec.SubnetSelector = map[string]string{"Name": "test-subnet-1,test-subnet-3"}
+				nodeTemplate.Spec.NetworkInterfaces = []*v1alpha1.NetworkInterface{
+					{
+						AssociatePublicIPAddress: aws.Bool(true),
+						DeviceIndex:              aws.Int64(0),
+					},
+				}
+				ExpectApplied(ctx, env.Client, provisioner, nodeTemplate)
+				pod := coretest.UnschedulablePod()
+				ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod)
+				ExpectScheduled(ctx, env.Client, pod)
+				input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop()
+				Expect(*input.LaunchTemplateData.NetworkInterfaces[0].AssociatePublicIpAddress).To(BeTrue())
+				Expect(len(input.LaunchTemplateData.SecurityGroupIds)).To(BeNumerically("==", 0))
 			})
 
-			It("should not explicitly set 'AssignPublicIPv4' when the subnets are configured to assign public IPv4 addresses", func() {
+			It("should not define networkInterfaces when the subnets are configured to assign public IPv4 addresses and user did not specify otherwise", func() {
 				nodeTemplate.Spec.SubnetSelector = map[string]string{"Name": "test-subnet-2"}
 				ExpectApplied(ctx, env.Client, provisioner, nodeTemplate)
 				pod := coretest.UnschedulablePod()
 				ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod)
 				ExpectScheduled(ctx, env.Client, pod)
 				input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop()
 				Expect(len(input.LaunchTemplateData.NetworkInterfaces)).To(BeNumerically("==", 0))
+				Expect(len(input.LaunchTemplateData.SecurityGroupIds)).To(BeNumerically(">", 0))
+			})
+
+			It("should use the same securityGroup for all networkInterfaces ", func() {
+				nodeTemplate.Spec.SubnetSelector = map[string]string{"Name": "test-subnet-2"}
+				nodeTemplate.Spec.NetworkInterfaces = []*v1alpha1.NetworkInterface{
+					{
+						DeviceIndex: aws.Int64(0),
+					},
+					{
+						DeviceIndex: aws.Int64(1),
+					},
+				}
+				ExpectApplied(ctx, env.Client, provisioner, nodeTemplate)
+				pod := coretest.UnschedulablePod()
+				ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod)
+				ExpectScheduled(ctx, env.Client, pod)
+				input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop()
+				Expect(len(input.LaunchTemplateData.SecurityGroupIds)).To(BeNumerically("==", 0))
+				Expect(len(input.LaunchTemplateData.NetworkInterfaces)).To(BeNumerically("==", 2))
+				Expect(input.LaunchTemplateData.NetworkInterfaces[0].Groups).To(Equal(input.LaunchTemplateData.NetworkInterfaces[1].Groups))
+			})
+
+			It("should match the values of AWSNodeTemplate.NetworkInterfaces", func() {
+				nodeTemplate.Spec.SubnetSelector = map[string]string{"Name": "test-subnet-2"}
+				nodeTemplate.Spec.NetworkInterfaces = []*v1alpha1.NetworkInterface{
+					{
+						AssociatePublicIPAddress: aws.Bool(true),
+						Description:              aws.String("example"),
+						DeviceIndex:              aws.Int64(1),
+					},
+				}
+				ExpectApplied(ctx, env.Client, provisioner, nodeTemplate)
+				pod := coretest.UnschedulablePod()
+				ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod)
+				ExpectScheduled(ctx, env.Client, pod)
+				input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop()
+				Expect(len(input.LaunchTemplateData.SecurityGroupIds)).To(BeNumerically("==", 0))
+				Expect(len(input.LaunchTemplateData.NetworkInterfaces)).To(BeNumerically("==", 1))
+				Expect(input.LaunchTemplateData.NetworkInterfaces[0].AssociateCarrierIpAddress).To(HaveValue(BeTrue()))
+				Expect(input.LaunchTemplateData.NetworkInterfaces[0].AssociatePublicIpAddress).To(HaveValue(BeTrue()))
+				Expect(input.LaunchTemplateData.NetworkInterfaces[0].DeleteOnTermination).To(HaveValue(BeTrue()))
+
+				Expect(input.LaunchTemplateData.NetworkInterfaces[0].Description).To(HaveValue(Equal("example")))
+				Expect(input.LaunchTemplateData.NetworkInterfaces[0].InterfaceType).To(HaveValue(Equal("example")))
+
+				Expect(input.LaunchTemplateData.NetworkInterfaces[0].DeviceIndex).To(HaveValue(BeNumerically("==", 1)))
+				Expect(input.LaunchTemplateData.NetworkInterfaces[0].NetworkCardIndex).To(HaveValue(BeNumerically("==", 1)))
+
+				Expect(input.LaunchTemplateData.NetworkInterfaces[0].Ipv4PrefixCount).To(HaveValue(BeNumerically("==", 1)))
+				Expect(input.LaunchTemplateData.NetworkInterfaces[0].Ipv6PrefixCount).To(HaveValue(BeNumerically("==", 1)))
+
+				Expect(len(input.LaunchTemplateData.NetworkInterfaces[0].Groups)).To(BeNumerically(">", 0))
 			})
 		})
 		Context("Kubelet Args", func() {