diff --git a/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml b/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml index 0d5d77257e76..fe84dfed89ce 100644 --- a/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml +++ b/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml @@ -192,6 +192,22 @@ spec: credentials are not available." type: string type: object + networkInterfaces: + items: + properties: + associatePublicIPAddress: + description: Associates a public IPv4 address with eth0 for + a new network interface. + type: boolean + description: + description: A description for the network interface. + type: string + deviceIndex: + description: The device index for the network interface attachment. + format: int64 + type: integer + type: object + type: array securityGroupSelector: additionalProperties: type: string diff --git a/pkg/apis/v1alpha1/provider.go b/pkg/apis/v1alpha1/provider.go index 0fe2f8071b06..9a46dc1645ef 100644 --- a/pkg/apis/v1alpha1/provider.go +++ b/pkg/apis/v1alpha1/provider.go @@ -73,6 +73,8 @@ type LaunchTemplate struct { // BlockDeviceMappings to be applied to provisioned nodes. // +optionals BlockDeviceMappings []*BlockDeviceMapping `json:"blockDeviceMappings,omitempty"` + + NetworkInterfaces []*NetworkInterface `json:"networkInterfaces,omitempty"` } // MetadataOptions contains parameters for specifying the exposure of the @@ -196,3 +198,15 @@ func DeserializeProvider(raw []byte) (*AWS, error) { } return a, nil } + +type NetworkInterface struct { + + // Associates a public IPv4 address with eth0 for a new network interface. + AssociatePublicIPAddress *bool `json:"associatePublicIPAddress,omitempty"` + + // A description for the network interface. + Description *string `json:"description,omitempty"` + + // The device index for the network interface attachment. + DeviceIndex *int64 `json:"deviceIndex,omitempty"` +} diff --git a/pkg/apis/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/v1alpha1/zz_generated.deepcopy.go index 4576b8d376de..a502803f7024 100644 --- a/pkg/apis/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/v1alpha1/zz_generated.deepcopy.go @@ -335,6 +335,17 @@ func (in *LaunchTemplate) DeepCopyInto(out *LaunchTemplate) { } } } + if in.NetworkInterfaces != nil { + in, out := &in.NetworkInterfaces, &out.NetworkInterfaces + *out = make([]*NetworkInterface, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(NetworkInterface) + (*in).DeepCopyInto(*out) + } + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LaunchTemplate. @@ -382,6 +393,36 @@ func (in *MetadataOptions) DeepCopy() *MetadataOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterface) DeepCopyInto(out *NetworkInterface) { + *out = *in + if in.AssociatePublicIPAddress != nil { + in, out := &in.AssociatePublicIPAddress, &out.AssociatePublicIPAddress + *out = new(bool) + **out = **in + } + if in.Description != nil { + in, out := &in.Description, &out.Description + *out = new(string) + **out = **in + } + if in.DeviceIndex != nil { + in, out := &in.DeviceIndex, &out.DeviceIndex + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterface. +func (in *NetworkInterface) DeepCopy() *NetworkInterface { + if in == nil { + return nil + } + out := new(NetworkInterface) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecurityGroup) DeepCopyInto(out *SecurityGroup) { *out = *in diff --git a/pkg/providers/amifamily/resolver.go b/pkg/providers/amifamily/resolver.go index 6cd2f8e77e58..36761ae39448 100644 --- a/pkg/providers/amifamily/resolver.go +++ b/pkg/providers/amifamily/resolver.go @@ -69,6 +69,7 @@ type LaunchTemplate struct { AMIID string InstanceTypes []*cloudprovider.InstanceType `hash:"ignore"` DetailedMonitoring bool + NetworkInterfaces []*v1alpha1.NetworkInterface } // AMIFamily can be implemented to override the default logic for generating dynamic launch template parameters @@ -161,6 +162,7 @@ func (r Resolver) Resolve(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTem DetailedMonitoring: aws.BoolValue(nodeTemplate.Spec.DetailedMonitoring), AMIID: amiID, InstanceTypes: instanceTypes, + NetworkInterfaces: nodeTemplate.Spec.NetworkInterfaces, } if resolved.BlockDeviceMappings == nil { resolved.BlockDeviceMappings = amiFamily.DefaultBlockDeviceMappings() diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index 60a252cabf32..2c6b7f8e71ed 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -214,7 +214,7 @@ func (p *Provider) createLaunchTemplate(ctx context.Context, options *amifamily. if err != nil { return nil, err } - networkInterface := p.generateNetworkInterface(options) + networkInterface := p.generateNetworkInterfaces(options) output, err := p.ec2api.CreateLaunchTemplateWithContext(ctx, &ec2.CreateLaunchTemplateInput{ LaunchTemplateName: aws.String(launchTemplateName(options)), LaunchTemplateData: &ec2.RequestLaunchTemplateData{ @@ -254,22 +254,34 @@ func (p *Provider) createLaunchTemplate(ctx context.Context, options *amifamily. return output.LaunchTemplate, nil } -// generateNetworkInterface generates a network interface for the launch template. +// generateNetworkInterfaces generates a network interface for the launch template. // If all referenced subnets do not assign public IPv4 addresses to EC2 instances therein, we explicitly set // AssociatePublicIpAddress to 'false' in the Launch Template, generated based on this configuration struct. // This is done to help comply with AWS account policies that require explicitly setting that field to 'false'. // https://github.com/aws/karpenter/issues/3815 -func (p *Provider) generateNetworkInterface(options *amifamily.LaunchTemplate) []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest { - if options.AssociatePublicIPAddress != nil { - return []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ - { - AssociatePublicIpAddress: options.AssociatePublicIPAddress, - DeviceIndex: aws.Int64(0), - Groups: lo.Map(options.SecurityGroups, func(s v1alpha1.SecurityGroup, _ int) *string { return aws.String(s.ID) }), - }, +func (p *Provider) generateNetworkInterfaces(options *amifamily.LaunchTemplate) []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest { + if len(options.NetworkInterfaces) == 0 { + if options.AssociatePublicIPAddress != nil { + return []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ + { + AssociatePublicIpAddress: options.AssociatePublicIPAddress, + DeviceIndex: aws.Int64(0), + Groups: lo.Map(options.SecurityGroups, func(s v1alpha1.SecurityGroup, _ int) *string { return aws.String(s.ID) }), + }, + } } + return nil } - return nil + + var networkInterfacesRequest []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest + for _, networkInterface := range options.NetworkInterfaces { + networkInterfacesRequest = append(networkInterfacesRequest, &ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ + AssociatePublicIpAddress: networkInterface.AssociatePublicIPAddress, + Groups: lo.Map(options.SecurityGroups, func(s v1alpha1.SecurityGroup, _ int) *string { return aws.String(s.ID) }), + }) + } + return networkInterfacesRequest + } func (p *Provider) blockDeviceMappings(blockDeviceMappings []*v1alpha1.BlockDeviceMapping) []*ec2.LaunchTemplateBlockDeviceMappingRequest { diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index 617d12ebcbc0..b8afc0b0ccb9 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -1642,8 +1642,8 @@ var _ = Describe("LaunchTemplates", func() { Expect(*input.LaunchTemplateData.ImageId).To(ContainSubstring("test-ami")) }) }) - Context("Subnet-based Launch Template Configration", func() { - It("should explicitly set 'AssignPublicIPv4' to false in the Launch Template", func() { + Context("NetworkInterfaces Configuration of launch template", func() { + It("should explicitly set 'AssignPublicIPv4' to false when all subnets are private", func() { nodeTemplate.Spec.SubnetSelector = map[string]string{"Name": "test-subnet-1,test-subnet-3"} ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() @@ -1651,9 +1651,26 @@ var _ = Describe("LaunchTemplates", func() { ExpectScheduled(ctx, env.Client, pod) input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(*input.LaunchTemplateData.NetworkInterfaces[0].AssociatePublicIpAddress).To(BeFalse()) + Expect(len(input.LaunchTemplateData.SecurityGroupIds)).To(BeNumerically("==", 0)) + }) + It("should overwrite 'AssignPublicIPv4' to true when specified by user in the AWSNodeTemplate.NetworkInterfaces", func() { + nodeTemplate.Spec.SubnetSelector = map[string]string{"Name": "test-subnet-1,test-subnet-3"} + nodeTemplate.Spec.NetworkInterfaces = []*v1alpha1.NetworkInterface{ + { + AssociatePublicIPAddress: aws.Bool(true), + DeviceIndex: aws.Int64(0), + }, + } + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() + Expect(*input.LaunchTemplateData.NetworkInterfaces[0].AssociatePublicIpAddress).To(BeTrue()) + Expect(len(input.LaunchTemplateData.SecurityGroupIds)).To(BeNumerically("==", 0)) }) - It("should not explicitly set 'AssignPublicIPv4' when the subnets are configured to assign public IPv4 addresses", func() { + It("should not define networkInterfaces when the subnets are configured to assign public IPv4 addresses and user did not specify otherwise", func() { nodeTemplate.Spec.SubnetSelector = map[string]string{"Name": "test-subnet-2"} ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() @@ -1661,6 +1678,59 @@ var _ = Describe("LaunchTemplates", func() { ExpectScheduled(ctx, env.Client, pod) input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(len(input.LaunchTemplateData.NetworkInterfaces)).To(BeNumerically("==", 0)) + Expect(len(input.LaunchTemplateData.SecurityGroupIds)).To(BeNumerically(">", 0)) + }) + + It("should use the same securityGroup for all networkInterfaces ", func() { + nodeTemplate.Spec.SubnetSelector = map[string]string{"Name": "test-subnet-2"} + nodeTemplate.Spec.NetworkInterfaces = []*v1alpha1.NetworkInterface{ + { + DeviceIndex: aws.Int64(0), + }, + { + DeviceIndex: aws.Int64(1), + }, + } + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() + Expect(len(input.LaunchTemplateData.SecurityGroupIds)).To(BeNumerically("==", 0)) + Expect(len(input.LaunchTemplateData.NetworkInterfaces)).To(BeNumerically("==", 2)) + Expect(input.LaunchTemplateData.NetworkInterfaces[0].Groups).To(Equal(input.LaunchTemplateData.NetworkInterfaces[1].Groups)) + }) + + It("should match the values of AWSNodeTemplate.NetworkInterfaces", func() { + nodeTemplate.Spec.SubnetSelector = map[string]string{"Name": "test-subnet-2"} + nodeTemplate.Spec.NetworkInterfaces = []*v1alpha1.NetworkInterface{ + { + AssociatePublicIPAddress: aws.Bool(true), + Description: aws.String("example"), + DeviceIndex: aws.Int64(1), + }, + } + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() + Expect(len(input.LaunchTemplateData.SecurityGroupIds)).To(BeNumerically("==", 0)) + Expect(len(input.LaunchTemplateData.NetworkInterfaces)).To(BeNumerically("==", 1)) + Expect(input.LaunchTemplateData.NetworkInterfaces[0].AssociateCarrierIpAddress).To(HaveValue(BeTrue())) + Expect(input.LaunchTemplateData.NetworkInterfaces[0].AssociatePublicIpAddress).To(HaveValue(BeTrue())) + Expect(input.LaunchTemplateData.NetworkInterfaces[0].DeleteOnTermination).To(HaveValue(BeTrue())) + + Expect(input.LaunchTemplateData.NetworkInterfaces[0].Description).To(HaveValue(Equal("example"))) + Expect(input.LaunchTemplateData.NetworkInterfaces[0].InterfaceType).To(HaveValue(Equal("example"))) + + Expect(input.LaunchTemplateData.NetworkInterfaces[0].DeviceIndex).To(HaveValue(BeNumerically("==", 1))) + Expect(input.LaunchTemplateData.NetworkInterfaces[0].NetworkCardIndex).To(HaveValue(BeNumerically("==", 1))) + + Expect(input.LaunchTemplateData.NetworkInterfaces[0].Ipv4PrefixCount).To(HaveValue(BeNumerically("==", 1))) + Expect(input.LaunchTemplateData.NetworkInterfaces[0].Ipv6PrefixCount).To(HaveValue(BeNumerically("==", 1))) + + Expect(len(input.LaunchTemplateData.NetworkInterfaces[0].Groups)).To(BeNumerically(">", 0)) }) }) Context("Kubelet Args", func() { diff --git a/test/suites/integration/networkinterfaces_test.go b/test/suites/integration/networkinterfaces_test.go new file mode 100644 index 000000000000..f4c9a13cd6df --- /dev/null +++ b/test/suites/integration/networkinterfaces_test.go @@ -0,0 +1,182 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration_test + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" + + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + "github.com/aws/karpenter-core/pkg/test" + "github.com/aws/karpenter/pkg/apis/settings" + "github.com/aws/karpenter/pkg/apis/v1alpha1" + awstest "github.com/aws/karpenter/pkg/test" +) + +var _ = Describe("NetworkInterfaces", func() { + It("should create a default NetworkInterface if none specified, with no public IP auto assignment", func() { + subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + Expect(len(subnets)).ToNot(Equal(0)) + allSubnets := lo.Flatten(lo.Values(subnets)) + ExpectAssociatePublicIPAddressToBe(false, allSubnets...) + + provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ + AWS: v1alpha1.AWS{ + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + LaunchTemplate: v1alpha1.LaunchTemplate{}, + }, + }) + provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) + pod := test.Pod() + env.ExpectCreated(pod, provider, provisioner) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + instance := env.GetInstance(pod.Spec.NodeName) + Expect(instance.NetworkInterfaces).To(HaveLen(1)) + Expect(instance.NetworkInterfaces[0]).ToNot(BeNil()) + Expect(instance.NetworkInterfaces[0].Attachment).To(HaveField("DeviceIndex", HaveValue(Equal(int64(0))))) + Expect(instance.NetworkInterfaces[0].Attachment).To(HaveField("NetworkCardIndex", HaveValue(Equal(int64(0))))) + Expect(instance.PublicIpAddress).To(BeNil()) + }) + It("should create a default NetworkInterface if none specified, with public IP auto assignment", func() { + subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + Expect(len(subnets)).ToNot(Equal(0)) + allSubnets := lo.Flatten(lo.Values(subnets)) + ExpectAssociatePublicIPAddressToBe(true, allSubnets...) + + provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ + AWS: v1alpha1.AWS{ + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + LaunchTemplate: v1alpha1.LaunchTemplate{}, + }, + }) + provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) + pod := test.Pod() + env.ExpectCreated(pod, provider, provisioner) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + instance := env.GetInstance(pod.Spec.NodeName) + Expect(instance.NetworkInterfaces).To(HaveLen(1)) + Expect(instance.NetworkInterfaces[0]).ToNot(BeNil()) + Expect(instance.NetworkInterfaces[0].Attachment).To(HaveField("DeviceIndex", HaveValue(Equal(int64(0))))) + Expect(instance.NetworkInterfaces[0].Attachment).To(HaveField("NetworkCardIndex", HaveValue(Equal(int64(0))))) + Expect(instance.PublicIpAddress).ToNot(BeNil()) + }) + It("should use the specified NetworkInterface", func() { + subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + Expect(len(subnets)).ToNot(Equal(0)) + allSubnets := lo.Flatten(lo.Values(subnets)) + ExpectAssociatePublicIPAddressToBe(false, allSubnets...) + + desc := "a test network interface" + provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ + AWS: v1alpha1.AWS{ + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + LaunchTemplate: v1alpha1.LaunchTemplate{ + NetworkInterfaces: []*v1alpha1.NetworkInterface{ + { + AssociatePublicIPAddress: aws.Bool(true), + Description: aws.String(desc), + DeviceIndex: aws.Int64(0), + }, + }, + }, + }, + }) + provisioner := test.Provisioner(test.ProvisionerOptions{ + ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, + Requirements: []v1.NodeSelectorRequirement{ + { + Key: "karpenter.k8s.aws/instance-hypervisor", + Operator: "In", + Values: []string{"nitro"}, + }, + }, + }) + pod := test.Pod() + env.ExpectCreated(pod, provider, provisioner) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + instance := env.GetInstance(pod.Spec.NodeName) + Expect(instance.NetworkInterfaces).To(HaveLen(1)) + Expect(instance.NetworkInterfaces[0]).ToNot(BeNil()) + Expect(instance.NetworkInterfaces[0].Attachment).To(HaveField("DeviceIndex", HaveValue(Equal(int64(0))))) + Expect(instance.NetworkInterfaces[0].Attachment).To(HaveField("NetworkCardIndex", HaveValue(Equal(int64(0))))) + Expect(instance.NetworkInterfaces[0].Ipv4Prefixes).To(HaveLen(2)) + Expect(instance.NetworkInterfaces[0].Ipv6Prefixes).To(HaveLen(2)) + Expect(instance.NetworkInterfaces[0].Description).To(Equal(desc)) + Expect(instance.PublicIpAddress).ToNot(BeNil()) + }) + It("should create a node with more than one NetworkInterface", func() { + desc1 := "a test network interface" + desc2 := "another test network interface" + provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ + AWS: v1alpha1.AWS{ + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + LaunchTemplate: v1alpha1.LaunchTemplate{ + NetworkInterfaces: []*v1alpha1.NetworkInterface{ + { + Description: aws.String(desc1), + DeviceIndex: aws.Int64(0), + }, + { + Description: aws.String(desc2), + DeviceIndex: aws.Int64(1), + }, + }, + }, + }, + }) + provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) + pod := test.Pod() + env.ExpectCreated(pod, provider, provisioner) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + instance := env.GetInstance(pod.Spec.NodeName) + Expect(instance.NetworkInterfaces).To(HaveLen(2)) + + Expect(instance.NetworkInterfaces[0]).ToNot(BeNil()) + Expect(instance.NetworkInterfaces[0].Attachment).To(HaveField("DeviceIndex", HaveValue(Equal(int64(0))))) + Expect(instance.NetworkInterfaces[0].Attachment).To(HaveField("NetworkCardIndex", HaveValue(Equal(int64(0))))) + Expect(instance.NetworkInterfaces[0].Description).To(Equal(desc1)) + Expect(instance.NetworkInterfaces[0].InterfaceType).To(Equal("interface")) + + Expect(instance.NetworkInterfaces[1]).ToNot(BeNil()) + Expect(instance.NetworkInterfaces[1].Attachment).To(HaveField("DeviceIndex", HaveValue(Equal(int64(1))))) + Expect(instance.NetworkInterfaces[1].Attachment).To(HaveField("NetworkCardIndex", HaveValue(Equal(int64(1))))) + Expect(instance.NetworkInterfaces[1].Description).To(Equal(desc2)) + Expect(instance.NetworkInterfaces[1].InterfaceType).To(Equal("efa")) + }) +}) + +func ExpectAssociatePublicIPAddressToBe(enabled bool, subnetIDs ...string) { + for subnetID := range subnetIDs { + _, err := env.EC2API.ModifySubnetAttribute(&ec2.ModifySubnetAttributeInput{ + MapPublicIpOnLaunch: &ec2.AttributeBooleanValue{ + Value: aws.Bool(enabled), + }, + SubnetId: aws.String(subnetIDs[subnetID]), + }) + Expect(err).To(BeNil()) + } +} diff --git a/website/content/en/preview/concepts/node-templates.md b/website/content/en/preview/concepts/node-templates.md index 94568f34f09c..93b429510cf6 100644 --- a/website/content/en/preview/concepts/node-templates.md +++ b/website/content/en/preview/concepts/node-templates.md @@ -33,6 +33,7 @@ spec: tags: { ... } # optional, propagates tags to underlying EC2 resources metadataOptions: { ... } # optional, configures IMDS for the instance blockDeviceMappings: [ ... ] # optional, configures storage devices for the instance + networkInterfaces: { ... } # optional, configures the network interfaces for the instance detailedMonitoring: "..." # optional, configures detailed monitoring for the instance status: subnets: { ... } # resolved subnets @@ -388,6 +389,34 @@ spec: ``` {{% /alert %}} +## spec.networkInterfaces +the `networkInterfaces` field on the AWSNodeTemplate is mapped to [AWS EC2 LaunchTemplate NetworkInterfaces](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-launchtemplate-networkinterface.html) +and can be used to configure the network Interfaces that AWS EC2 will attach to the provisioned nodes + +when not specified, a single network interface will be added to the node. In case all referenced subnets had `MapPublicIpOnLaunch` set to `false`, the generated network interface will have `AssociatePublicIpAddress` equals `false` + +currently these are the supported fields +```yaml +networkInterfaces: # optional, configures network interfaces for the instance + associatePublicIPAddress: false # optional, Indicates whether to assign a public IPv4 address to eth0 for a new network interface. + description: "..." # optional, A description for the network interface. + deviceIndex: 0 # optional, The device index for the network interface attachment. +``` +Learn more about [Network Interfaces](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html). +### Examples + +```yaml +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +spec: + networkInterfaces: + - associatePublicIPAddress: true + description: "main network interface" + deviceIndex: 0 + - description: "secondary network interface" + deviceIndex: 1 +``` + ## spec.userData You can control the UserData that is applied to your worker nodes via this field.