diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index fd0fd5128fbd..cfae066188cb 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -79,6 +79,7 @@ type DefaultProvider struct { CABundle *string ClusterEndpoint string ClusterCIDR atomic.Pointer[string] + ClusterIPFamily corev1.IPFamily } func NewDefaultProvider(ctx context.Context, cache *cache.Cache, ec2api sdk.EC2API, eksapi sdk.EKSAPI, amiFamily amifamily.Resolver, @@ -95,6 +96,7 @@ func NewDefaultProvider(ctx context.Context, cache *cache.Cache, ec2api sdk.EC2A cm: pretty.NewChangeMonitor(), KubeDNSIP: kubeDNSIP, ClusterEndpoint: clusterEndpoint, + ClusterIPFamily: lo.Ternary(kubeDNSIP != nil && kubeDNSIP.To4() == nil, corev1.IPv6Protocol, corev1.IPv4Protocol), } l.cache.OnEvicted(l.cachedEvictedFunc(ctx)) go func() { @@ -284,6 +286,8 @@ func (p *DefaultProvider) generateNetworkInterfaces(options *amifamily.LaunchTem // Instances launched with multiple pre-configured network interfaces cannot set AssociatePublicIPAddress to true. This is an EC2 limitation. However, this does not apply for instances // with a single EFA network interface, and we should support those use cases. Launch failures with multiple enis should be considered user misconfiguration. AssociatePublicIpAddress: options.AssociatePublicIPAddress, + PrimaryIpv6: lo.Ternary(p.ClusterIPFamily == corev1.IPv6Protocol, lo.ToPtr(true), nil), + Ipv6PrefixCount: lo.Ternary(p.ClusterIPFamily == corev1.IPv6Protocol, lo.ToPtr(int32(1)), nil), } }) } @@ -296,6 +300,8 @@ func (p *DefaultProvider) generateNetworkInterfaces(options *amifamily.LaunchTem Groups: lo.Map(options.SecurityGroups, func(s v1.SecurityGroup, _ int) string { return s.ID }), + PrimaryIpv6: lo.Ternary(p.ClusterIPFamily == corev1.IPv6Protocol, lo.ToPtr(true), nil), + Ipv6PrefixCount: lo.Ternary(p.ClusterIPFamily == corev1.IPv6Protocol, lo.ToPtr(int32(1)), nil), }, } } diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index ab62a470ac89..79412d204883 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -2220,6 +2220,68 @@ essential = true }) }) }) + Context("Networking", func() { + Context("launch template respect to DNS ip for ipfamily selection", func() { + DescribeTable( + "should select correct ipFamily based on DNS ip", + func(ipFamily corev1.IPFamily) { + provider := launchtemplate.NewDefaultProvider( + ctx, + awsEnv.LaunchTemplateCache, + awsEnv.EC2API, + awsEnv.EKSAPI, + awsEnv.AMIResolver, + awsEnv.SecurityGroupProvider, + awsEnv.SubnetProvider, + awsEnv.LaunchTemplateProvider.CABundle, + make(chan struct{}), + net.ParseIP(lo.Ternary(ipFamily == corev1.IPv4Protocol, "10.0.100.10", "fd01:99f0:d47b::a")), + "https://test-cluster", + ) + Expect(provider.ClusterIPFamily).To(Equal(ipFamily)) + }, + Entry("DNS has ipv4 address", corev1.IPv4Protocol), + Entry("DNS has ipv6 address", corev1.IPv6Protocol), + ) + }) + Context("should provision a v6 prefix and set v6 primary IP as true when running in an ipv6 cluster", func() { + DescribeTable( + "should set Primary IPv6 as true and provision a prefix", + func(isPublicAddressSet, isPublic, isEFA bool) { + awsEnv.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("fd4b:121b:812b::a") + awsEnv.LaunchTemplateProvider.ClusterIPFamily = corev1.IPv6Protocol + if isPublicAddressSet { + nodeClass.Spec.AssociatePublicIPAddress = lo.ToPtr(isPublic) + } + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod(lo.Ternary(isEFA, coretest.PodOptions{ + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{v1.ResourceEFA: resource.MustParse("2")}, + Limits: corev1.ResourceList{v1.ResourceEFA: resource.MustParse("2")}, + }, + }, coretest.PodOptions{})) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() + if isPublicAddressSet { + Expect(lo.FromPtr(input.LaunchTemplateData.NetworkInterfaces[0].AssociatePublicIpAddress)).To(Equal(isPublic)) + Expect(lo.FromPtr(input.LaunchTemplateData.NetworkInterfaces[0].Ipv6PrefixCount)).To(Equal(int32(1))) + Expect(lo.FromPtr(input.LaunchTemplateData.NetworkInterfaces[0].PrimaryIpv6)).To(BeTrue()) + } else if !isEFA { + Expect(input.LaunchTemplateData.NetworkInterfaces).To(HaveLen(0)) + } else { + Expect(lo.FromPtr(input.LaunchTemplateData.NetworkInterfaces[0].InterfaceType)).To(Equal(string(ec2types.NetworkInterfaceTypeEfa))) + Expect(lo.FromPtr(input.LaunchTemplateData.NetworkInterfaces[0].Ipv6PrefixCount)).To(Equal(int32(1))) + Expect(lo.FromPtr(input.LaunchTemplateData.NetworkInterfaces[0].PrimaryIpv6)).To(BeTrue()) + } + }, + Entry("AssociatePublicIPAddress is not set and EFA is false", false, true, false), + Entry("AssociatePublicIPAddress is not set and EFA is true", false, false, true), + Entry("AssociatePublicIPAddress is set as true and EFA is true", true, true, true), + Entry("AssociatePublicIPAddress is set as false and EFA is false", true, false, false), + ) + }) + }) }) // ExpectTags verifies that the expected tags are a subset of the tags found diff --git a/test/suites/ipv6/suite_test.go b/test/suites/ipv6/suite_test.go index 980bfc8e1b73..bb5df90f82c1 100644 --- a/test/suites/ipv6/suite_test.go +++ b/test/suites/ipv6/suite_test.go @@ -20,10 +20,16 @@ import ( "github.com/samber/lo" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" coretest "sigs.k8s.io/karpenter/pkg/test" + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" @@ -94,4 +100,60 @@ var _ = Describe("IPv6", func() { }) Expect(internalIPv6Addrs).To(HaveLen(1)) }) + It("should provision a static IPv6 prefix with node launch and set IPv6 as primary in the primary network interface", func() { + clusterDNSAddr := env.ExpectIPv6ClusterDNS() + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ClusterDNS: []string{clusterDNSAddr}} + Expect(disableVPCCNIProvisioning(true)).To(Succeed()) + DeferCleanup(func() { + Expect(disableVPCCNIProvisioning(false)).To(Succeed()) + }) + pod := coretest.Pod() + env.ExpectCreated(pod, nodeClass, nodePool) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + output, err := env.EC2API.DescribeInstances(env.Context, &ec2.DescribeInstancesInput{ + InstanceIds: []string{coretest.Pod().Spec.NodeName}, + }) + Expect(err).ShouldNot(HaveOccurred()) + Expect(output.Reservations).To(HaveLen(1)) + Expect(output.Reservations[0].Instances).To(HaveLen(1)) + Expect(output.Reservations[0].Instances[0].NetworkInterfaces).To(HaveLen(1)) + Expect(output.Reservations[0].Instances[0].NetworkInterfaces[0].Ipv6Prefixes).To(HaveLen(1)) + _, hasIPv6Primary := lo.Find(output.Reservations[0].Instances[0].NetworkInterfaces[0].Ipv6Addresses, func(ip types.InstanceIpv6Address) bool { + return lo.FromPtr(ip.IsPrimaryIpv6) + }) + Expect(hasIPv6Primary).To(BeTrue()) + }) }) + +// disable VPC CNI provisioning on network interfaces and IPs +func disableVPCCNIProvisioning(disable bool) error { + dsClient := env.KubeClient.AppsV1().DaemonSets("kube-system") + retryErr := retry.OnError( + retry.DefaultRetry, + func(err error) bool { + return true + }, + func() error { + awsNode, getErr := dsClient.Get(env.Context, "aws-node", metav1.GetOptions{}) + if getErr != nil { + return getErr + } + + for i := range awsNode.Spec.Template.Spec.Containers { + if awsNode.Spec.Template.Spec.Containers[i].Name == "aws-node" { + for j := range awsNode.Spec.Template.Spec.Containers[i].Env { + if awsNode.Spec.Template.Spec.Containers[i].Env[j].Name == "DISABLE_NETWORK_RESOURCE_PROVISIONING" { + awsNode.Spec.Template.Spec.Containers[i].Env[j].Value = lo.Ternary(disable, "true", "false") + } + } + } + } + + _, updateErr := dsClient.Update(env.Context, awsNode, metav1.UpdateOptions{}) + return updateErr + }, + ) + // ignore AWS VPC CNI is not installed + return client.IgnoreNotFound(retryErr) +}