Skip to content

Commit

Permalink
Feat: Provisioning IPv6 prefix to LT if cluster is IPv6 (#7275)
Browse files Browse the repository at this point in the history
  • Loading branch information
@haouc
haouc authored Nov 8, 2024
1 parent 655d473 commit 008b087
Show file tree
Hide file tree
Showing 3 changed files with 130 additions and 0 deletions.
6 changes: 6 additions & 0 deletions pkg/providers/launchtemplate/launchtemplate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ type DefaultProvider struct {
CABundle *string
ClusterEndpoint string
ClusterCIDR atomic.Pointer[string]
ClusterIPFamily corev1.IPFamily
}

func NewDefaultProvider(ctx context.Context, cache *cache.Cache, ec2api sdk.EC2API, eksapi sdk.EKSAPI, amiFamily amifamily.Resolver,
Expand All @@ -95,6 +96,7 @@ func NewDefaultProvider(ctx context.Context, cache *cache.Cache, ec2api sdk.EC2A
cm: pretty.NewChangeMonitor(),
KubeDNSIP: kubeDNSIP,
ClusterEndpoint: clusterEndpoint,
ClusterIPFamily: lo.Ternary(kubeDNSIP != nil && kubeDNSIP.To4() == nil, corev1.IPv6Protocol, corev1.IPv4Protocol),
}
l.cache.OnEvicted(l.cachedEvictedFunc(ctx))
go func() {
Expand Down Expand Up @@ -284,6 +286,8 @@ func (p *DefaultProvider) generateNetworkInterfaces(options *amifamily.LaunchTem
// Instances launched with multiple pre-configured network interfaces cannot set AssociatePublicIPAddress to true. This is an EC2 limitation. However, this does not apply for instances
// with a single EFA network interface, and we should support those use cases. Launch failures with multiple enis should be considered user misconfiguration.
AssociatePublicIpAddress: options.AssociatePublicIPAddress,
PrimaryIpv6: lo.Ternary(p.ClusterIPFamily == corev1.IPv6Protocol, lo.ToPtr(true), nil),
Ipv6PrefixCount: lo.Ternary(p.ClusterIPFamily == corev1.IPv6Protocol, lo.ToPtr(int32(1)), nil),
}
})
}
Expand All @@ -296,6 +300,8 @@ func (p *DefaultProvider) generateNetworkInterfaces(options *amifamily.LaunchTem
Groups: lo.Map(options.SecurityGroups, func(s v1.SecurityGroup, _ int) string {
return s.ID
}),
PrimaryIpv6: lo.Ternary(p.ClusterIPFamily == corev1.IPv6Protocol, lo.ToPtr(true), nil),
Ipv6PrefixCount: lo.Ternary(p.ClusterIPFamily == corev1.IPv6Protocol, lo.ToPtr(int32(1)), nil),
},
}
}
Expand Down
62 changes: 62 additions & 0 deletions pkg/providers/launchtemplate/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2220,6 +2220,68 @@ essential = true
})
})
})
Context("Networking", func() {
Context("launch template respect to DNS ip for ipfamily selection", func() {
DescribeTable(
"should select correct ipFamily based on DNS ip",
func(ipFamily corev1.IPFamily) {
provider := launchtemplate.NewDefaultProvider(
ctx,
awsEnv.LaunchTemplateCache,
awsEnv.EC2API,
awsEnv.EKSAPI,
awsEnv.AMIResolver,
awsEnv.SecurityGroupProvider,
awsEnv.SubnetProvider,
awsEnv.LaunchTemplateProvider.CABundle,
make(chan struct{}),
net.ParseIP(lo.Ternary(ipFamily == corev1.IPv4Protocol, "10.0.100.10", "fd01:99f0:d47b::a")),
"https://test-cluster",
)
Expect(provider.ClusterIPFamily).To(Equal(ipFamily))
},
Entry("DNS has ipv4 address", corev1.IPv4Protocol),
Entry("DNS has ipv6 address", corev1.IPv6Protocol),
)
})
Context("should provision a v6 prefix and set v6 primary IP as true when running in an ipv6 cluster", func() {
DescribeTable(
"should set Primary IPv6 as true and provision a prefix",
func(isPublicAddressSet, isPublic, isEFA bool) {
awsEnv.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("fd4b:121b:812b::a")
awsEnv.LaunchTemplateProvider.ClusterIPFamily = corev1.IPv6Protocol
if isPublicAddressSet {
nodeClass.Spec.AssociatePublicIPAddress = lo.ToPtr(isPublic)
}
ExpectApplied(ctx, env.Client, nodePool, nodeClass)
pod := coretest.UnschedulablePod(lo.Ternary(isEFA, coretest.PodOptions{
ResourceRequirements: corev1.ResourceRequirements{
Requests: corev1.ResourceList{v1.ResourceEFA: resource.MustParse("2")},
Limits: corev1.ResourceList{v1.ResourceEFA: resource.MustParse("2")},
},
}, coretest.PodOptions{}))
ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod)
ExpectScheduled(ctx, env.Client, pod)
input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop()
if isPublicAddressSet {
Expect(lo.FromPtr(input.LaunchTemplateData.NetworkInterfaces[0].AssociatePublicIpAddress)).To(Equal(isPublic))
Expect(lo.FromPtr(input.LaunchTemplateData.NetworkInterfaces[0].Ipv6PrefixCount)).To(Equal(int32(1)))
Expect(lo.FromPtr(input.LaunchTemplateData.NetworkInterfaces[0].PrimaryIpv6)).To(BeTrue())
} else if !isEFA {
Expect(input.LaunchTemplateData.NetworkInterfaces).To(HaveLen(0))
} else {
Expect(lo.FromPtr(input.LaunchTemplateData.NetworkInterfaces[0].InterfaceType)).To(Equal(string(ec2types.NetworkInterfaceTypeEfa)))
Expect(lo.FromPtr(input.LaunchTemplateData.NetworkInterfaces[0].Ipv6PrefixCount)).To(Equal(int32(1)))
Expect(lo.FromPtr(input.LaunchTemplateData.NetworkInterfaces[0].PrimaryIpv6)).To(BeTrue())
}
},
Entry("AssociatePublicIPAddress is not set and EFA is false", false, true, false),
Entry("AssociatePublicIPAddress is not set and EFA is true", false, false, true),
Entry("AssociatePublicIPAddress is set as true and EFA is true", true, true, true),
Entry("AssociatePublicIPAddress is set as false and EFA is false", true, false, false),
)
})
})
})

// ExpectTags verifies that the expected tags are a subset of the tags found
Expand Down
62 changes: 62 additions & 0 deletions test/suites/ipv6/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,16 @@ import (

"github.com/samber/lo"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/util/retry"

"sigs.k8s.io/controller-runtime/pkg/client"
karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
coretest "sigs.k8s.io/karpenter/pkg/test"

"github.com/aws/aws-sdk-go-v2/service/ec2"
"github.com/aws/aws-sdk-go-v2/service/ec2/types"

v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
"github.com/aws/karpenter-provider-aws/test/pkg/environment/aws"

Expand Down Expand Up @@ -94,4 +100,60 @@ var _ = Describe("IPv6", func() {
})
Expect(internalIPv6Addrs).To(HaveLen(1))
})
It("should provision a static IPv6 prefix with node launch and set IPv6 as primary in the primary network interface", func() {
clusterDNSAddr := env.ExpectIPv6ClusterDNS()
nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ClusterDNS: []string{clusterDNSAddr}}
Expect(disableVPCCNIProvisioning(true)).To(Succeed())
DeferCleanup(func() {
Expect(disableVPCCNIProvisioning(false)).To(Succeed())
})
pod := coretest.Pod()
env.ExpectCreated(pod, nodeClass, nodePool)
env.EventuallyExpectHealthy(pod)
env.ExpectCreatedNodeCount("==", 1)
output, err := env.EC2API.DescribeInstances(env.Context, &ec2.DescribeInstancesInput{
InstanceIds: []string{coretest.Pod().Spec.NodeName},
})
Expect(err).ShouldNot(HaveOccurred())
Expect(output.Reservations).To(HaveLen(1))
Expect(output.Reservations[0].Instances).To(HaveLen(1))
Expect(output.Reservations[0].Instances[0].NetworkInterfaces).To(HaveLen(1))
Expect(output.Reservations[0].Instances[0].NetworkInterfaces[0].Ipv6Prefixes).To(HaveLen(1))
_, hasIPv6Primary := lo.Find(output.Reservations[0].Instances[0].NetworkInterfaces[0].Ipv6Addresses, func(ip types.InstanceIpv6Address) bool {
return lo.FromPtr(ip.IsPrimaryIpv6)
})
Expect(hasIPv6Primary).To(BeTrue())
})
})

// disable VPC CNI provisioning on network interfaces and IPs
func disableVPCCNIProvisioning(disable bool) error {
dsClient := env.KubeClient.AppsV1().DaemonSets("kube-system")
retryErr := retry.OnError(
retry.DefaultRetry,
func(err error) bool {
return true
},
func() error {
awsNode, getErr := dsClient.Get(env.Context, "aws-node", metav1.GetOptions{})
if getErr != nil {
return getErr
}

for i := range awsNode.Spec.Template.Spec.Containers {
if awsNode.Spec.Template.Spec.Containers[i].Name == "aws-node" {
for j := range awsNode.Spec.Template.Spec.Containers[i].Env {
if awsNode.Spec.Template.Spec.Containers[i].Env[j].Name == "DISABLE_NETWORK_RESOURCE_PROVISIONING" {
awsNode.Spec.Template.Spec.Containers[i].Env[j].Value = lo.Ternary(disable, "true", "false")
}
}
}
}

_, updateErr := dsClient.Update(env.Context, awsNode, metav1.UpdateOptions{})
return updateErr
},
)
// ignore AWS VPC CNI is not installed
return client.IgnoreNotFound(retryErr)
}

0 comments on commit 008b087

Please sign in to comment.