From 69884dce736703448f5aed39ef3da19138c73b14 Mon Sep 17 00:00:00 2001 From: Jason Deal Date: Sun, 25 Feb 2024 02:13:41 -0500 Subject: [PATCH] chore: dynamic cluster CIDR resolution --- pkg/controllers/nodeclass/controller.go | 3 ++ pkg/fake/eksapi.go | 13 +++++++- pkg/operator/operator.go | 25 +-------------- .../amifamily/bootstrap/bootstrap.go | 2 +- pkg/providers/amifamily/bootstrap/nodeadm.go | 2 +- pkg/providers/amifamily/resolver.go | 2 +- .../launchtemplate/launchtemplate.go | 32 ++++++++++++++++--- pkg/test/environment.go | 3 +- 8 files changed, 49 insertions(+), 33 deletions(-) diff --git a/pkg/controllers/nodeclass/controller.go b/pkg/controllers/nodeclass/controller.go index 7ebd8decfb3d..538a7fae1ac0 100644 --- a/pkg/controllers/nodeclass/controller.go +++ b/pkg/controllers/nodeclass/controller.go @@ -87,6 +87,9 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeCl c.resolveAMIs(ctx, nodeClass), c.resolveInstanceProfile(ctx, nodeClass), ) + if lo.FromPtr(nodeClass.Spec.AMIFamily) == v1beta1.AMIFamilyAL2023 { + err = multierr.Append(err, c.launchTemplateProvider.ResolveClusterCIDR(ctx)) + } if !equality.Semantic.DeepEqual(stored, nodeClass) { statusCopy := nodeClass.DeepCopy() if patchErr := c.kubeClient.Patch(ctx, nodeClass, client.MergeFrom(stored)); err != nil { diff --git a/pkg/fake/eksapi.go b/pkg/fake/eksapi.go index 4c0a48e4fd06..8d39706dd9e5 100644 --- a/pkg/fake/eksapi.go +++ b/pkg/fake/eksapi.go @@ -20,6 +20,7 @@ import ( "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/service/eks" "github.com/aws/aws-sdk-go/service/eks/eksiface" + "github.com/samber/lo" ) const () @@ -35,6 +36,10 @@ type EKSAPI struct { EKSAPIBehavior } +func NewEKSAPI() *EKSAPI { + return &EKSAPI{} +} + // Reset must be called between tests otherwise tests will pollute // each other. func (s *EKSAPI) Reset() { @@ -43,6 +48,12 @@ func (s *EKSAPI) Reset() { func (s *EKSAPI) DescribeClusterWithContext(_ context.Context, input *eks.DescribeClusterInput, _ ...request.Option) (*eks.DescribeClusterOutput, error) { return s.DescribeClusterBehavior.Invoke(input, func(*eks.DescribeClusterInput) (*eks.DescribeClusterOutput, error) { - return nil, nil + return &eks.DescribeClusterOutput{ + Cluster: &eks.Cluster{ + KubernetesNetworkConfig: &eks.KubernetesNetworkConfigResponse{ + ServiceIpv4Cidr: lo.ToPtr("10.100.0.0/16"), + }, + }, + }, nil }) } diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 0566f4134191..41c7e048c09e 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -122,12 +122,6 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont } else { logging.FromContext(ctx).With("cluster-endpoint", clusterEndpoint).Debugf("discovered cluster endpoint") } - clusterCIDR, err := ResolveClusterCIDR(ctx, eks.New(sess)) - if err != nil { - logging.FromContext(ctx).Fatalf("unable to detect the cluster CIDR, %s", err) - } else { - logging.FromContext(ctx).With("cluster-cidr", clusterCIDR).Debugf("discovered cluster CIDR") - } // We perform best-effort on resolving the kube-dns IP kubeDNSIP, err := kubeDNSIP(ctx, operator.KubernetesInterface) if err != nil { @@ -155,6 +149,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont ctx, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), ec2api, + eks.New(sess), amiResolver, securityGroupProvider, subnetProvider, @@ -163,7 +158,6 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont operator.Elected(), kubeDNSIP, clusterEndpoint, - clusterCIDR, ) instanceTypeProvider := instancetype.NewProvider( *sess.Config.Region, @@ -241,23 +235,6 @@ func ResolveClusterEndpoint(ctx context.Context, eksAPI eksiface.EKSAPI) (string return *out.Cluster.Endpoint, nil } -func ResolveClusterCIDR(ctx context.Context, eksAPI eksiface.EKSAPI) (string, error) { - out, err := eksAPI.DescribeClusterWithContext(ctx, &eks.DescribeClusterInput{ - Name: aws.String(options.FromContext(ctx).ClusterName), - }) - if err != nil { - return "", fmt.Errorf("failed to resolve cluster CIDR, %w", err) - } - - if ipv4CIDR := out.Cluster.KubernetesNetworkConfig.ServiceIpv4Cidr; ipv4CIDR != nil { - return *ipv4CIDR, nil - } - if ipv6CIDR := out.Cluster.KubernetesNetworkConfig.ServiceIpv6Cidr; ipv6CIDR != nil { - return *ipv6CIDR, nil - } - return "", fmt.Errorf("failed to resolve cluster CIDR") -} - func getCABundle(ctx context.Context, restConfig *rest.Config) (*string, error) { // Discover CA Bundle from the REST client. We could alternatively // have used the simpler client-go InClusterConfig() method. diff --git a/pkg/providers/amifamily/bootstrap/bootstrap.go b/pkg/providers/amifamily/bootstrap/bootstrap.go index fb825ed42e02..145354392515 100644 --- a/pkg/providers/amifamily/bootstrap/bootstrap.go +++ b/pkg/providers/amifamily/bootstrap/bootstrap.go @@ -34,7 +34,7 @@ import ( type Options struct { ClusterName string ClusterEndpoint string - ClusterCIDR string + ClusterCIDR *string KubeletConfig *corev1beta1.KubeletConfiguration Taints []core.Taint `hash:"set"` Labels map[string]string `hash:"set"` diff --git a/pkg/providers/amifamily/bootstrap/nodeadm.go b/pkg/providers/amifamily/bootstrap/nodeadm.go index f7cb200e5b28..687b237a349c 100644 --- a/pkg/providers/amifamily/bootstrap/nodeadm.go +++ b/pkg/providers/amifamily/bootstrap/nodeadm.go @@ -65,7 +65,7 @@ func (n Nodeadm) getNodeConfigYAML() (string, error) { Cluster: admv1alpha1.ClusterDetails{ Name: n.ClusterName, APIServerEndpoint: n.ClusterEndpoint, - CIDR: n.ClusterCIDR, + CIDR: lo.FromPtr(n.ClusterCIDR), }, }, } diff --git a/pkg/providers/amifamily/resolver.go b/pkg/providers/amifamily/resolver.go index 2c6fbbccc676..d83b602bba52 100644 --- a/pkg/providers/amifamily/resolver.go +++ b/pkg/providers/amifamily/resolver.go @@ -50,7 +50,7 @@ type Resolver struct { type Options struct { ClusterName string ClusterEndpoint string - ClusterCIDR string + ClusterCIDR *string InstanceProfile string CABundle *string `hash:"ignore"` InstanceStorePolicy *v1beta1.InstanceStorePolicy diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index 73897c3f830e..f89aa57a004e 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -29,6 +29,8 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/aws/aws-sdk-go/service/eks" + "github.com/aws/aws-sdk-go/service/eks/eksiface" "github.com/mitchellh/hashstructure/v2" "github.com/patrickmn/go-cache" "github.com/samber/lo" @@ -64,6 +66,7 @@ type LaunchTemplate struct { type Provider struct { sync.Mutex ec2api ec2iface.EC2API + eksapi eksiface.EKSAPI amiFamily *amifamily.Resolver securityGroupProvider *securitygroup.Provider subnetProvider *subnet.Provider @@ -73,14 +76,15 @@ type Provider struct { KubeDNSIP net.IP CABundle *string ClusterEndpoint string - ClusterCIDR string + ClusterCIDR *string } -func NewProvider(ctx context.Context, cache *cache.Cache, ec2api ec2iface.EC2API, amiFamily *amifamily.Resolver, +func NewProvider(ctx context.Context, cache *cache.Cache, ec2api ec2iface.EC2API, eksapi eksiface.EKSAPI, amiFamily *amifamily.Resolver, securityGroupProvider *securitygroup.Provider, subnetProvider *subnet.Provider, instanceProfileProvider *instanceprofile.Provider, - caBundle *string, startAsync <-chan struct{}, kubeDNSIP net.IP, clusterEndpoint string, clusterCIDR string) *Provider { + caBundle *string, startAsync <-chan struct{}, kubeDNSIP net.IP, clusterEndpoint string) *Provider { l := &Provider{ ec2api: ec2api, + eksapi: eksapi, amiFamily: amiFamily, securityGroupProvider: securityGroupProvider, subnetProvider: subnetProvider, @@ -90,7 +94,6 @@ func NewProvider(ctx context.Context, cache *cache.Cache, ec2api ec2iface.EC2API cm: pretty.NewChangeMonitor(), KubeDNSIP: kubeDNSIP, ClusterEndpoint: clusterEndpoint, - ClusterCIDR: clusterCIDR, } l.cache.OnEvicted(l.cachedEvictedFunc(ctx)) go func() { @@ -427,3 +430,24 @@ func (p *Provider) DeleteLaunchTemplates(ctx context.Context, nodeClass *v1beta1 } return nil } + +func (p *Provider) ResolveClusterCIDR(ctx context.Context) error { + if p.ClusterCIDR != nil { + return nil + } + out, err := p.eksapi.DescribeClusterWithContext(ctx, &eks.DescribeClusterInput{ + Name: aws.String(options.FromContext(ctx).ClusterName), + }) + if err != nil { + return fmt.Errorf("failed to resolve cluster CIDR, %w", err) + } + if ipv4CIDR := out.Cluster.KubernetesNetworkConfig.ServiceIpv4Cidr; ipv4CIDR != nil { + p.ClusterCIDR = ipv4CIDR + return nil + } + if ipv6CIDR := out.Cluster.KubernetesNetworkConfig.ServiceIpv6Cidr; ipv6CIDR != nil { + p.ClusterCIDR = ipv6CIDR + return nil + } + return fmt.Errorf("failed to resolve cluster CIDR") +} diff --git a/pkg/test/environment.go b/pkg/test/environment.go index 2ff9e0929a6f..4dd132681dc3 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -82,6 +82,7 @@ type Environment struct { func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment { // API ec2api := fake.NewEC2API() + eksapi := fake.NewEKSAPI() ssmapi := fake.NewSSMAPI() iamapi := fake.NewIAMAPI() @@ -110,6 +111,7 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment ctx, launchTemplateCache, ec2api, + eksapi, amiResolver, securityGroupProvider, subnetProvider, @@ -118,7 +120,6 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment make(chan struct{}), net.ParseIP("10.0.100.10"), "https://test-cluster", - "10.100.0.0/16", ) instanceProvider := instance.NewProvider(ctx,