diff --git a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml index 8045ddb48973..586b653ff06e 100644 --- a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -263,6 +263,23 @@ spec: - optional type: string type: object + networkInterfaces: + description: NetworkInterfaces to be applied to provisioned nodes. + items: + properties: + associatePublicIPAddress: + description: Associates a public IPv4 address with eth0 for + a new network interface. + type: boolean + description: + description: A description for the network interface. + type: string + deviceIndex: + description: The device index for the network interface attachment. + format: int64 + type: integer + type: object + type: array role: description: Role is the AWS identity that nodes use. This field is immutable. Marking this field as immutable avoids concerns around diff --git a/pkg/apis/v1beta1/ec2nodeclass.go b/pkg/apis/v1beta1/ec2nodeclass.go index fd3dbebd0094..4bee857c1d24 100644 --- a/pkg/apis/v1beta1/ec2nodeclass.go +++ b/pkg/apis/v1beta1/ec2nodeclass.go @@ -97,6 +97,9 @@ type EC2NodeClassSpec struct { // +kubebuilder:default={"httpEndpoint":"enabled","httpProtocolIPv6":"disabled","httpPutResponseHopLimit":2,"httpTokens":"required"} // +optional MetadataOptions *MetadataOptions `json:"metadataOptions,omitempty"` + // NetworkInterfaces to be applied to provisioned nodes. + // +optional + NetworkInterfaces []*NetworkInterface `json:"networkInterfaces,omitempty"` // Context is a Reserved field in EC2 APIs // https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html // +optional @@ -312,6 +315,18 @@ type BlockDevice struct { VolumeType *string `json:"volumeType,omitempty"` } +type NetworkInterface struct { + + // Associates a public IPv4 address with eth0 for a new network interface. + AssociatePublicIPAddress *bool `json:"associatePublicIPAddress,omitempty"` + + // A description for the network interface. + Description *string `json:"description,omitempty"` + + // The device index for the network interface attachment. + DeviceIndex *int64 `json:"deviceIndex,omitempty"` +} + // EC2NodeClass is the Schema for the EC2NodeClass API // +kubebuilder:object:root=true // +kubebuilder:resource:path=ec2nodeclasses,scope=Cluster,categories=karpenter,shortName={ec2nc,ec2ncs} diff --git a/pkg/apis/v1beta1/zz_generated.deepcopy.go b/pkg/apis/v1beta1/zz_generated.deepcopy.go index bdc0e52d0092..283953a96b61 100644 --- a/pkg/apis/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/v1beta1/zz_generated.deepcopy.go @@ -268,6 +268,17 @@ func (in *EC2NodeClassSpec) DeepCopyInto(out *EC2NodeClassSpec) { *out = new(MetadataOptions) (*in).DeepCopyInto(*out) } + if in.NetworkInterfaces != nil { + in, out := &in.NetworkInterfaces, &out.NetworkInterfaces + *out = make([]*NetworkInterface, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(NetworkInterface) + (*in).DeepCopyInto(*out) + } + } + } if in.Context != nil { in, out := &in.Context, &out.Context *out = new(string) @@ -383,6 +394,36 @@ func (in *MetadataOptions) DeepCopy() *MetadataOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterface) DeepCopyInto(out *NetworkInterface) { + *out = *in + if in.AssociatePublicIPAddress != nil { + in, out := &in.AssociatePublicIPAddress, &out.AssociatePublicIPAddress + *out = new(bool) + **out = **in + } + if in.Description != nil { + in, out := &in.Description, &out.Description + *out = new(string) + **out = **in + } + if in.DeviceIndex != nil { + in, out := &in.DeviceIndex, &out.DeviceIndex + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterface. +func (in *NetworkInterface) DeepCopy() *NetworkInterface { + if in == nil { + return nil + } + out := new(NetworkInterface) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecurityGroup) DeepCopyInto(out *SecurityGroup) { *out = *in diff --git a/pkg/providers/amifamily/resolver.go b/pkg/providers/amifamily/resolver.go index 135ae96fa1fc..8bb94753d87a 100644 --- a/pkg/providers/amifamily/resolver.go +++ b/pkg/providers/amifamily/resolver.go @@ -69,6 +69,7 @@ type LaunchTemplate struct { AMIID string InstanceTypes []*cloudprovider.InstanceType `hash:"ignore"` DetailedMonitoring bool + NetworkInterfaces []*v1beta1.NetworkInterface } // AMIFamily can be implemented to override the default logic for generating dynamic launch template parameters @@ -161,6 +162,7 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, DetailedMonitoring: aws.BoolValue(nodeClass.Spec.DetailedMonitoring), AMIID: amiID, InstanceTypes: instanceTypes, + NetworkInterfaces: nodeClass.Spec.NetworkInterfaces, } if len(resolved.BlockDeviceMappings) == 0 { resolved.BlockDeviceMappings = amiFamily.DefaultBlockDeviceMappings() diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index 40a84ee734ce..22808d61811d 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -235,7 +235,7 @@ func (p *Provider) createLaunchTemplate(ctx context.Context, options *amifamily. if err != nil { return nil, err } - networkInterface := p.generateNetworkInterface(options) + networkInterface := p.generateNetworkInterfaces(options) output, err := p.ec2api.CreateLaunchTemplateWithContext(ctx, &ec2.CreateLaunchTemplateInput{ LaunchTemplateName: aws.String(launchTemplateName(options)), LaunchTemplateData: &ec2.RequestLaunchTemplateData{ @@ -275,22 +275,36 @@ func (p *Provider) createLaunchTemplate(ctx context.Context, options *amifamily. return output.LaunchTemplate, nil } -// generateNetworkInterface generates a network interface for the launch template. +// generateNetworkInterfaces generates a network interface for the launch template. // If all referenced subnets do not assign public IPv4 addresses to EC2 instances therein, we explicitly set // AssociatePublicIpAddress to 'false' in the Launch Template, generated based on this configuration struct. // This is done to help comply with AWS account policies that require explicitly setting that field to 'false'. // https://github.com/aws/karpenter/issues/3815 -func (p *Provider) generateNetworkInterface(options *amifamily.LaunchTemplate) []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest { - if options.AssociatePublicIPAddress != nil { - return []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ - { - AssociatePublicIpAddress: options.AssociatePublicIPAddress, - DeviceIndex: aws.Int64(0), - Groups: lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) }), - }, +func (p *Provider) generateNetworkInterfaces(options *amifamily.LaunchTemplate) []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest { + if len(options.NetworkInterfaces) == 0 { + if options.AssociatePublicIPAddress != nil { + return []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ + { + AssociatePublicIpAddress: options.AssociatePublicIPAddress, + DeviceIndex: aws.Int64(0), + Groups: lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) }), + }, + } } + return nil } - return nil + + var networkInterfacesRequest []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest + for _, networkInterface := range options.NetworkInterfaces { + networkInterfacesRequest = append(networkInterfacesRequest, &ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ + AssociatePublicIpAddress: networkInterface.AssociatePublicIPAddress, + DeviceIndex: networkInterface.DeviceIndex, + Description: networkInterface.Description, + Groups: lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) }), + }) + } + return networkInterfacesRequest + } func (p *Provider) blockDeviceMappings(blockDeviceMappings []*v1beta1.BlockDeviceMapping) []*ec2.LaunchTemplateBlockDeviceMappingRequest { diff --git a/pkg/providers/launchtemplate/nodeclass_test.go b/pkg/providers/launchtemplate/nodeclass_test.go index 44fb34b0e737..06f16a20ead9 100644 --- a/pkg/providers/launchtemplate/nodeclass_test.go +++ b/pkg/providers/launchtemplate/nodeclass_test.go @@ -1563,7 +1563,7 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() { Expect(*input.LaunchTemplateData.ImageId).To(ContainSubstring("test-ami")) }) }) - Context("Subnet-based Launch Template Configration", func() { + Context("NetworkInterfaces", func() { It("should explicitly set 'AssignPublicIPv4' to false in the Launch Template", func() { nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{{Tags: map[string]string{"Name": "test-subnet-1,test-subnet-3"}}} ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -1573,8 +1573,23 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() { input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(*input.LaunchTemplateData.NetworkInterfaces[0].AssociatePublicIpAddress).To(BeFalse()) }) - - It("should not explicitly set 'AssignPublicIPv4' when the subnets are configured to assign public IPv4 addresses", func() { + It("should overwrite 'AssignPublicIPv4' to true when specified by user in the EC2NodeClass.NetworkInterfaces", func() { + nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{{Tags: map[string]string{"Name": "test-subnet-1,test-subnet-3"}}} + nodeClass.Spec.NetworkInterfaces = []*v1beta1.NetworkInterface{ + { + AssociatePublicIPAddress: aws.Bool(true), + DeviceIndex: aws.Int64(0), + }, + } + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() + Expect(*input.LaunchTemplateData.NetworkInterfaces[0].AssociatePublicIpAddress).To(BeTrue()) + Expect(len(input.LaunchTemplateData.SecurityGroupIds)).To(BeNumerically("==", 0)) + }) + It("should not explicitly set 'AssignPublicIPv4' when the subnets are configured to assign public IPv4 addresses and the user did not specify otherwise", func() { nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{{Tags: map[string]string{"Name": "test-subnet-2"}}} ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() @@ -1583,6 +1598,47 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() { input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(len(input.LaunchTemplateData.NetworkInterfaces)).To(BeNumerically("==", 0)) }) + It("should use the same securityGroup for all networkInterfaces ", func() { + nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{{Tags: map[string]string{"Name": "test-subnet-2"}}} + nodeClass.Spec.NetworkInterfaces = []*v1beta1.NetworkInterface{ + { + DeviceIndex: aws.Int64(0), + }, + { + DeviceIndex: aws.Int64(1), + }, + } + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() + Expect(len(input.LaunchTemplateData.SecurityGroupIds)).To(BeNumerically("==", 0)) + Expect(len(input.LaunchTemplateData.NetworkInterfaces)).To(BeNumerically("==", 2)) + Expect(input.LaunchTemplateData.NetworkInterfaces[0].Groups).To(Equal(input.LaunchTemplateData.NetworkInterfaces[1].Groups)) + }) + It("should match the values of AWSNodeTemplate.NetworkInterfaces", func() { + nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{{Tags: map[string]string{"Name": "test-subnet-2"}}} + nodeClass.Spec.NetworkInterfaces = []*v1beta1.NetworkInterface{ + { + AssociatePublicIPAddress: aws.Bool(true), + Description: aws.String("example"), + DeviceIndex: aws.Int64(1), + }, + } + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() + Expect(len(input.LaunchTemplateData.SecurityGroupIds)).To(BeNumerically("==", 0)) + Expect(len(input.LaunchTemplateData.NetworkInterfaces)).To(BeNumerically("==", 1)) + Expect(input.LaunchTemplateData.NetworkInterfaces[0].AssociatePublicIpAddress).To(HaveValue(BeTrue())) + + Expect(input.LaunchTemplateData.NetworkInterfaces[0].Description).To(HaveValue(Equal("example"))) + Expect(input.LaunchTemplateData.NetworkInterfaces[0].DeviceIndex).To(HaveValue(BeNumerically("==", 1))) + Expect(len(input.LaunchTemplateData.NetworkInterfaces[0].Groups)).To(BeNumerically(">", 0)) + }) }) Context("Kubelet Args", func() { It("should specify the --dns-cluster-ip flag when clusterDNSIP is set", func() { diff --git a/test/suites/beta/integration/networkinterfaces_test.go b/test/suites/beta/integration/networkinterfaces_test.go new file mode 100644 index 000000000000..e163f253e697 --- /dev/null +++ b/test/suites/beta/integration/networkinterfaces_test.go @@ -0,0 +1,96 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration_test + +import ( + "github.com/aws/aws-sdk-go/service/ec2" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + + "github.com/aws/karpenter-core/pkg/test" + "github.com/aws/karpenter/pkg/apis/v1beta1" +) + +var _ = Describe("NetworkInterfaces", func() { + BeforeEach(func() { + // Ensure that nodes schedule to private subnets. If a node without a public IP is assigned to a public subnet, + // and that subnet does not contain any private endpoints to the cluster, the node will be unable to join the + // cluster. + nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "Name": "*Private*", + "karpenter.sh/discovery": env.ClusterName, + }, + }, + } + }) + + DescribeTable( + "should correctly create NetworkInterfaces", + func(interfaces ...*v1beta1.NetworkInterface) { + nodeClass.Spec.NetworkInterfaces = interfaces + pod := test.Pod() + env.ExpectCreated(pod, nodeClass, nodePool) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + instance := env.GetInstance(pod.Spec.NodeName) + for _, interfaceSpec := range interfaces { + ni, ok := lo.Find(instance.NetworkInterfaces, func(ni *ec2.InstanceNetworkInterface) bool { + if ni.Description == nil { + return false + } + return *ni.Description == *interfaceSpec.Description + }) + Expect(ok).To(BeTrue()) + Expect(ni.Attachment).To(HaveField("DeviceIndex", HaveValue(Equal(*interfaceSpec.DeviceIndex)))) + } + + if len(interfaces) == 1 && interfaces[0].AssociatePublicIPAddress != nil { + if *interfaces[0].AssociatePublicIPAddress { + Expect(instance.PublicIpAddress).ToNot(BeNil()) + } else { + Expect(instance.PublicIpAddress).To(BeNil()) + } + } + }, + Entry("when a single interface is specified", &v1beta1.NetworkInterface{ + Description: lo.ToPtr("a test interface"), + DeviceIndex: lo.ToPtr(int64(0)), + }), + Entry("when a single interface is specified with AssociatePublicIPAddress = true", &v1beta1.NetworkInterface{ + AssociatePublicIPAddress: lo.ToPtr(true), + Description: lo.ToPtr("a test interface"), + DeviceIndex: lo.ToPtr(int64(0)), + }), + Entry("when a single interface is specified with AssociatePublicIPAddress = false", &v1beta1.NetworkInterface{ + AssociatePublicIPAddress: lo.ToPtr(false), + Description: lo.ToPtr("a test interface"), + DeviceIndex: lo.ToPtr(int64(0)), + }), + Entry( + "when multiple interfaces are specified", + &v1beta1.NetworkInterface{ + Description: lo.ToPtr("a test interface"), + DeviceIndex: lo.ToPtr(int64(0)), + }, + &v1beta1.NetworkInterface{ + Description: lo.ToPtr("another test interface"), + DeviceIndex: lo.ToPtr(int64(1)), + }, + ), + ) +}) diff --git a/website/content/en/preview/concepts/nodeclasses.md b/website/content/en/preview/concepts/nodeclasses.md index eec8a13b2da1..6cabedb1b8a0 100644 --- a/website/content/en/preview/concepts/nodeclasses.md +++ b/website/content/en/preview/concepts/nodeclasses.md @@ -29,35 +29,35 @@ metadata: name: default spec: # required, resolves a default ami and userdata - amiFamily: AL2 - + amiFamily: AL2 + # required, discovers subnets to attach to instances - subnetSelectorTerms: + subnetSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" - + # required, discovers security groups to attach to instances - securityGroupSelectorTerms: + securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" - + # required, IAM role to use for the node identity role: "KarpenterNodeRole-${CLUSTER_NAME}" # optional, discovers amis to override the amiFamily's default - amiSelectorTerms: + amiSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" - + # optional, overrides autogenerated userdata with a merge semantic - userData: | + userData: | echo "Hello world" # optional, propagates tags to underlying EC2 resources - tags: + tags: team: team-a app: team-a-app - + # optional, configures IMDS for the instance metadataOptions: httpEndpoint: enabled @@ -65,6 +65,11 @@ spec: httpPutResponseHopLimit: 2 httpTokens: required + # optional, configure network interfaces for the instance + - associatePublicIPAddress: true + description: "a network interface" + deviceIndex: 0 + # optional, configures storage devices for the instance blockDeviceMappings: - deviceName: /dev/xvda @@ -95,7 +100,7 @@ status: zone: us-east-2c - id: subnet-03941e7ad6afeaa72 zone: us-east-2a - + # resolved security groups securityGroups: - id: sg-041513b454818610b @@ -103,7 +108,7 @@ status: - id: sg-0286715698b894bca name: ControlPlaneSecurityGroup-1AQ073TSAAPW - # resolved AMIs + # resolved AMIs amis: - id: ami-01234567890123456 name: custom-ami-amd64 @@ -119,7 +124,7 @@ status: operator: In values: - arm64 - + # generated instance profile name instanceProfile: "${CLUSTER_NAME}-0123456778901234567789" ``` @@ -206,7 +211,7 @@ Karpenter will automatically query for the appropriate [EKS optimized AMI](https ### Custom -The `Custom` AMIFamily ships without any default userData to allow you to configure custom bootstrapping for control planes or images that don't support the default methods from the other families. +The `Custom` AMIFamily ships without any default userData to allow you to configure custom bootstrapping for control planes or images that don't support the default methods from the other families. ## spec.subnetSelectorTerms @@ -265,7 +270,7 @@ spec: ## spec.securityGroupSelectorTerms The security group of an instance is comparable to a set of firewall rules. -[EKS creates at least two security groups by default](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html). +[EKS creates at least two security groups by default](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html). {{% alert title="Tip" color="secondary" %}} Security groups may be specified by any tag, including "Name". Selecting tags using wildcards (`*`) is supported. @@ -531,6 +536,38 @@ spec: The `Custom` AMIFamily ships without any default `blockDeviceMappings`. +## spec.networkInterfaces + +The `networkInterfaces` field is mapped to [AWS EC2 LaunchTemplate NetworkInterfaces](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-launchtemplate-networkinterface.html) +and can be used to configure the network interfaces that AWS EC2 will attach to the provisioned nodes at launch. + +Learn more about [Network Interfaces](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html). + +### Examples + +Single network interface (with associated public IP): +```yaml +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +spec: + networkInterfaces: + - associatePublicIPAddress: true + description: "a network interface" + deviceIndex: 0 +``` + +Multiple network interfaces: +```yaml +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +spec: + networkInterfaces: + - description: "main network interface" + deviceIndex: 0 + - description: "secondary network interface" + deviceIndex: 1 +``` + ## spec.userData You can control the UserData that is applied to your worker nodes via this field. This allows you to run custom scripts or pass-through custom configuration to Karpenter instances on start-up. @@ -880,4 +917,4 @@ spec: role: "KarpenterNodeRole-${CLUSTER_NAME}" status: instanceProfile: "${CLUSTER_NAME}-0123456778901234567789" -``` \ No newline at end of file +```