From 523fcfcb48aba8b1319754a41ba0329c4decca95 Mon Sep 17 00:00:00 2001 From: Sichao Wang Date: Mon, 21 Oct 2024 04:40:31 +0000 Subject: [PATCH] Add support for static cluster --- .../internal/deployers/eksapi/cluster.go | 70 +++++++++++-------- .../internal/deployers/eksapi/deployer.go | 34 ++++++--- kubetest2/internal/deployers/eksapi/infra.go | 1 + 3 files changed, 66 insertions(+), 39 deletions(-) diff --git a/kubetest2/internal/deployers/eksapi/cluster.go b/kubetest2/internal/deployers/eksapi/cluster.go index d3b792dac..df48d9678 100644 --- a/kubetest2/internal/deployers/eksapi/cluster.go +++ b/kubetest2/internal/deployers/eksapi/cluster.go @@ -39,36 +39,50 @@ type Cluster struct { cidr string } -func (m *ClusterManager) createCluster(infra *Infrastructure, opts *deployerOptions) (*Cluster, error) { - input := eks.CreateClusterInput{ - Name: aws.String(m.resourceID), - ResourcesVpcConfig: &ekstypes.VpcConfigRequest{ - EndpointPrivateAccess: aws.Bool(true), - EndpointPublicAccess: aws.Bool(true), - SubnetIds: append(infra.subnetsPublic, infra.subnetsPrivate...), - }, - RoleArn: aws.String(infra.clusterRole), - KubernetesNetworkConfig: &ekstypes.KubernetesNetworkConfigRequest{ - IpFamily: ekstypes.IpFamily(opts.IPFamily), - }, - Version: aws.String(opts.KubernetesVersion), - } - apiOpts, err := util.NewHTTPHeaderAPIOptions(opts.UpClusterHeaders) - if err != nil { - return nil, fmt.Errorf("failed to create API options: %v", err) +func (m *ClusterManager) getOrCreateCluster(infra *Infrastructure, opts *deployerOptions) (*Cluster, error) { + targetClusterName := &opts.StaticClusterName + if *targetClusterName == "" { + klog.Infof("No StaticClusterName specified creating new cluster...") + input := eks.CreateClusterInput{ + Name: aws.String(m.resourceID), + ResourcesVpcConfig: &ekstypes.VpcConfigRequest{ + EndpointPrivateAccess: aws.Bool(true), + EndpointPublicAccess: aws.Bool(true), + SubnetIds: append(infra.subnetsPublic, infra.subnetsPrivate...), + }, + RoleArn: aws.String(infra.clusterRole), + KubernetesNetworkConfig: &ekstypes.KubernetesNetworkConfigRequest{ + IpFamily: ekstypes.IpFamily(opts.IPFamily), + }, + Version: aws.String(opts.KubernetesVersion), + } + apiOpts, err := util.NewHTTPHeaderAPIOptions(opts.UpClusterHeaders) + if err != nil { + return nil, fmt.Errorf("failed to create API options: %v", err) + } + createOutput, err := m.clients.EKS().CreateCluster(context.TODO(), &input, + func(o *eks.Options) { + o.APIOptions = apiOpts + }) + if err != nil { + return nil, fmt.Errorf("failed to create cluster: %v", err) + } + targetClusterName = createOutput.Cluster.Name + } else { + klog.Infof("reusing existing static cluster %s", *targetClusterName) } - klog.Infof("creating cluster...") - createOutput, err := m.clients.EKS().CreateCluster(context.TODO(), &input, - func(o *eks.Options) { - o.APIOptions = apiOpts - }) - if err != nil { - return nil, fmt.Errorf("failed to create cluster: %v", err) + cluster, waitErr := m.waitClusterReady(targetClusterName) + if waitErr != nil { + return nil, fmt.Errorf("failed to wait for cluster to become active: %v", waitErr) } + return cluster, nil +} + +func (m *ClusterManager) waitClusterReady(clusterName *string) (*Cluster, error) { describeInput := eks.DescribeClusterInput{ - Name: createOutput.Cluster.Name, + Name: clusterName, } - klog.Infof("waiting for cluster to be active: %s", *createOutput.Cluster.Arn) + klog.Infof("waiting for cluster to be active: %s", *clusterName) waitErr := eks.NewClusterActiveWaiter(m.clients.EKS()).Wait(context.TODO(), &describeInput, clusterCreationTimeout) describeOutput, describeErr := m.clients.EKS().DescribeCluster(context.TODO(), &describeInput) if describeErr != nil { @@ -76,9 +90,9 @@ func (m *ClusterManager) createCluster(infra *Infrastructure, opts *deployerOpti } klog.Infof("cluster details after creation: %+v", describeOutput.Cluster) if waitErr != nil { - return nil, fmt.Errorf("failed to wait for cluster to become active: %v", waitErr) + return nil, waitErr } - klog.Infof("cluster is active: %s", *createOutput.Cluster.Arn) + klog.Infof("cluster is active: %s", *describeOutput.Cluster.Arn) var cidr string switch describeOutput.Cluster.KubernetesNetworkConfig.IpFamily { case ekstypes.IpFamilyIpv4: diff --git a/kubetest2/internal/deployers/eksapi/deployer.go b/kubetest2/internal/deployers/eksapi/deployer.go index f4ee2d08e..a15159142 100644 --- a/kubetest2/internal/deployers/eksapi/deployer.go +++ b/kubetest2/internal/deployers/eksapi/deployer.go @@ -75,6 +75,7 @@ type deployerOptions struct { Nodes int `flag:"nodes" desc:"number of nodes to launch in cluster"` NodeNameStrategy string `flag:"node-name-strategy" desc:"Specifies the naming strategy for node. Allowed values: ['SessionName', 'EC2PrivateDNSName'], default to EC2PrivateDNSName"` Region string `flag:"region" desc:"AWS region for EKS cluster"` + StaticClusterName string `flag:"static-cluster-name" desc:"Optional when re-use existing cluster and node group by querying the kubeconfig and run test"` TuneVPCCNI bool `flag:"tune-vpc-cni" desc:"Apply tuning parameters to the VPC CNI DaemonSet"` UnmanagedNodes bool `flag:"unmanaged-nodes" desc:"Use an AutoScalingGroup instead of an EKS-managed nodegroup. Requires --ami"` UpClusterHeaders []string `flag:"up-cluster-header" desc:"Additional header to add to eks:CreateCluster requests. Specified in the same format as curl's -H flag."` @@ -163,12 +164,14 @@ func (d *deployer) Up() error { return err } } - if infra, err := d.infraManager.createInfrastructureStack(&d.deployerOptions); err != nil { - return err - } else { - d.infra = infra + if d.deployerOptions.StaticClusterName == "" { + if infra, err := d.infraManager.createInfrastructureStack(&d.deployerOptions); err != nil { + return err + } else { + d.infra = infra + } } - cluster, err := d.clusterManager.createCluster(d.infra, &d.deployerOptions) + cluster, err := d.clusterManager.getOrCreateCluster(d.infra, &d.deployerOptions) if err != nil { return err } @@ -181,6 +184,10 @@ func (d *deployer) Up() error { if err != nil { return err } + if d.deployerOptions.StaticClusterName != "" { + klog.Infof("inited k8sclient, skip the rest resource creation for static cluster") + return nil + } if d.UnmanagedNodes { if err := createAWSAuthConfigMap(d.k8sClient, d.NodeNameStrategy, d.infra.nodeRole); err != nil { return err @@ -236,6 +243,16 @@ func (d *deployer) verifyUpFlags() error { d.IPFamily = string(ekstypes.IpFamilyIpv4) klog.Infof("Using default IP family: %s", d.IPFamily) } + if d.NodeCreationTimeout == 0 { + d.NodeCreationTimeout = time.Minute * 20 + } + if d.NodeReadyTimeout == 0 { + d.NodeReadyTimeout = time.Minute * 5 + } + if d.StaticClusterName != "" { + klog.Infof("Skip configuration for static cluster") + return nil + } if d.UnmanagedNodes { if d.AMI == "" { return fmt.Errorf("--ami must be specified for --unmanaged-nodes") @@ -267,12 +284,6 @@ func (d *deployer) verifyUpFlags() error { klog.Infof("Using default AMI type: %s", d.AMIType) } } - if d.NodeCreationTimeout == 0 { - d.NodeCreationTimeout = time.Minute * 20 - } - if d.NodeReadyTimeout == 0 { - d.NodeReadyTimeout = time.Minute * 5 - } return nil } @@ -307,6 +318,7 @@ func deleteResources(im *InfrastructureManager, cm *ClusterManager, nm *Nodegrou // the EKS-managed cluster security group may be associated with a leaked ENI // so we need to make sure we've deleted leaked ENIs before we delete the cluster // otherwise, the cluster security group will be left behind and will block deletion of our VPC + if err := im.deleteLeakedENIs(); err != nil { return err } diff --git a/kubetest2/internal/deployers/eksapi/infra.go b/kubetest2/internal/deployers/eksapi/infra.go index 7ec7e025d..53cf59e44 100644 --- a/kubetest2/internal/deployers/eksapi/infra.go +++ b/kubetest2/internal/deployers/eksapi/infra.go @@ -168,6 +168,7 @@ func (m *InfrastructureManager) createInfrastructureStack(opts *deployerOptions) return nil, fmt.Errorf("failed to get infrastructure stack resources: %w", err) } klog.Infof("created infrastructure: %+v", infra) + return infra, nil }