From 0b39cd5da4b66641c9443ce758a18eb3dacc77f4 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Fri, 20 Oct 2023 22:14:26 -0700 Subject: [PATCH 1/9] Run beta tests for upgrade testing --- .github/workflows/e2e-upgrade.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-upgrade.yaml b/.github/workflows/e2e-upgrade.yaml index a7f1dff781ac..1e418d872335 100644 --- a/.github/workflows/e2e-upgrade.yaml +++ b/.github/workflows/e2e-upgrade.yaml @@ -152,7 +152,7 @@ jobs: - name: run the Upgrade test suite run: | aws eks update-kubeconfig --name ${{ env.CLUSTER_NAME }} - CLUSTER_NAME=${{ env.CLUSTER_NAME }} INTERRUPTION_QUEUE=${{ env.CLUSTER_NAME }} CLUSTER_ENDPOINT="$(aws eks describe-cluster --name ${{ env.CLUSTER_NAME }} --query "cluster.endpoint" --output text)" TEST_SUITE="Integration" make e2etests + CLUSTER_NAME=${{ env.CLUSTER_NAME }} INTERRUPTION_QUEUE=${{ env.CLUSTER_NAME }} CLUSTER_ENDPOINT="$(aws eks describe-cluster --name ${{ env.CLUSTER_NAME }} --query "cluster.endpoint" --output text)" TEST_SUITE="Beta/Integration" make e2etests - name: notify slack of success or failure uses: ./.github/actions/e2e/slack/notify if: (success() || failure()) && github.event_name != 'workflow_run' && github.event_name != 'conformance' From 1b0fa2b4fe232810bde2e28714e82ac1e681a287 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Fri, 20 Oct 2023 22:25:23 -0700 Subject: [PATCH 2/9] Add instance profile operations to permission boundary --- test/cloudformation/iam_cloudformation.yaml | 197 +++--------------- test/suites/alpha/drift/suite_test.go | 2 - .../alpha/expiration/expiration_test.go | 1 - .../alpha/machine/garbage_collection_test.go | 2 - .../suites/alpha/scale/deprovisioning_test.go | 1 - test/suites/beta/drift/suite_test.go | 2 - test/suites/beta/integration/cni_test.go | 1 - .../integration/extended_resources_test.go | 1 - 8 files changed, 30 insertions(+), 177 deletions(-) diff --git a/test/cloudformation/iam_cloudformation.yaml b/test/cloudformation/iam_cloudformation.yaml index 2263997cfbe2..1d6d3a3d518e 100644 --- a/test/cloudformation/iam_cloudformation.yaml +++ b/test/cloudformation/iam_cloudformation.yaml @@ -270,116 +270,7 @@ Resources: Statement: - Effect: Allow Action: - # Tag Permissions - - ec2:DescribeTags - # Internet Gateway Permissions - - ec2:DescribeEgressOnlyInternetGateways - - ec2:DescribeInternetGateways - # Elastic IP Permissions - - ec2:DescribeAddresses - # Instance Permissions - - ec2:DescribeInstanceTypeOfferings - - ec2:DescribeInstanceTypes - - ec2:DescribeInstances - - ec2:DescribeKeyPairs - # Launch Template Permissions - - ec2:DescribeLaunchTemplateVersions - - ec2:DescribeLaunchTemplates - # NAT Gateway Permissions - - ec2:DescribeNatGateways - # Network Interface Permissions - - ec2:DescribeNetworkInterfaces - # Route Table Permissions - - ec2:DescribeRouteTables - # Security Group Permissions - - ec2:DescribeSecurityGroups - # Subnet Permissions - - ec2:DescribeAvailabilityZones - - ec2:DescribeSubnets - # Volume Permissions - - ec2:DescribeVolumes - - ec2:DescribeVolumesModifications - - ec2:DescribeSnapshots - # Network ACL Permissions - - ec2:DescribeNetworkAcls - # VPC Permissions - - ec2:DescribeVpcs - # Image Permissions - - ec2:DescribeImages - # Tag Permissions - - ec2:CreateTags - - ec2:DeleteTags - # Internet Gateway Permissions - - ec2:CreateEgressOnlyInternetGateway - - ec2:DeleteEgressOnlyInternetGateway - # Elastic IP Permissions - - ec2:AllocateAddress - - ec2:ReleaseAddress - # Instance Permissions - - ec2:ModifyInstanceAttribute - - ec2:DescribeInstanceAttribute - - ec2:RunInstances - - ec2:StopInstances - - ec2:TerminateInstances - - ec2:AttachNetworkInterface - - ec2:ModifyNetworkInterfaceAttribute - - ec2:DetachNetworkInterface - # Internet Gateway Permissions - - ec2:AttachInternetGateway - - ec2:CreateInternetGateway - - ec2:DeleteInternetGateway - - ec2:DetachInternetGateway - # Launch Template Permissions - - ec2:CreateLaunchTemplate - - ec2:DeleteLaunchTemplate - # Fleet Permissions - - ec2:CreateFleet - # NAT Gateway Permissions - - ec2:CreateNatGateway - - ec2:DeleteNatGateway - # Network Interface Permissions - - ec2:AssignPrivateIpAddresses - - ec2:UnassignPrivateIpAddresses - - ec2:AssignIpv6Addresses - - ec2:UnassignIpv6Addresses - - ec2:AttachNetworkInterface - - ec2:DetachNetworkInterface - - ec2:CreateNetworkInterface - - ec2:ModifyNetworkInterfaceAttribute - - ec2:DeleteNetworkInterface - - ec2:CreateNetworkInterfacePermission - # Route Table Permissions - - ec2:CreateRoute - - ec2:CreateRouteTable - - ec2:DeleteRoute - - ec2:DeleteRouteTable - - ec2:AssociateRouteTable - - ec2:DisassociateRouteTable - # Security Group Permissions - - ec2:AuthorizeSecurityGroupIngress - - ec2:CreateSecurityGroup - - ec2:DeleteSecurityGroup - - ec2:RevokeSecurityGroupIngress - # Subnet Permissions - - ec2:CreateSubnet - - ec2:DeleteSubnet - - ec2:ModifySubnetAttribute - # Volume Permissions - - ec2:CreateSnapshot - - ec2:DeleteSnapshot - - ec2:CreateVolume - - ec2:DeleteVolume - - ec2:AttachVolume - - ec2:ModifyVolume - - ec2:DetachVolume - # VPC Permissions - - ec2:AssociateVpcCidrBlock - - ec2:DisassociateVpcCidrBlock - - ec2:CreateVpc - - ec2:DeleteVpc - - ec2:DescribeVpcAttribute - - ec2:ModifyVpcAttribute - - ec2:RunInstances + - ec2:* # Read-Only Permissions to pull ECR images needed by the NodeInstanceRole - ecr:GetAuthorizationToken - ecr:BatchCheckLayerAvailability @@ -397,75 +288,22 @@ Resources: - autoscaling:DescribeAutoScalingGroups - autoscaling:UpdateAutoScalingGroup # EKS ServiceRole permissions needed to handle LoadBalancer - - elasticloadbalancing:AddTags - - elasticloadbalancing:ApplySecurityGroupsToLoadBalancer - - elasticloadbalancing:AttachLoadBalancerToSubnets - - elasticloadbalancing:ConfigureHealthCheck - - elasticloadbalancing:CreateListener - - elasticloadbalancing:CreateLoadBalancer - - elasticloadbalancing:CreateLoadBalancerListeners - - elasticloadbalancing:CreateLoadBalancerPolicy - - elasticloadbalancing:CreateTargetGroup - - elasticloadbalancing:DeleteListener - - elasticloadbalancing:DeleteLoadBalancer - - elasticloadbalancing:DeleteLoadBalancerListeners - - elasticloadbalancing:DeleteTargetGroup - - elasticloadbalancing:DeregisterInstancesFromLoadBalancer - - elasticloadbalancing:DeregisterTargets - - elasticloadbalancing:DescribeListeners - - elasticloadbalancing:DescribeLoadBalancerAttributes - - elasticloadbalancing:DescribeLoadBalancerPolicies - - elasticloadbalancing:DescribeLoadBalancers - - elasticloadbalancing:DescribeTargetGroupAttributes - - elasticloadbalancing:DescribeTargetGroups - - elasticloadbalancing:DescribeTargetHealth - - elasticloadbalancing:DetachLoadBalancerFromSubnets - - elasticloadbalancing:ModifyListener - - elasticloadbalancing:ModifyLoadBalancerAttributes - - elasticloadbalancing:ModifyTargetGroup - - elasticloadbalancing:ModifyTargetGroupAttributes - - elasticloadbalancing:RegisterInstancesWithLoadBalancer - - elasticloadbalancing:RegisterTargets - - elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer - - elasticloadbalancing:SetLoadBalancerPoliciesOfListener + - elasticloadbalancing:* - kms:CreateGrant - kms:GenerateDataKeyWithoutPlaintext - kms:DescribeKey # SSM Permissions for AmazonSSMManagedInstanceCore policy applied to the NodeInstanceRole - - ssm:DescribeAssociation - - ssm:GetDeployablePatchSnapshotForInstance - - ssm:GetDocument - - ssm:DescribeDocument - - ssm:GetManifest - - ssm:GetParameter - - ssm:GetParameters - - ssm:ListAssociations - - ssm:ListInstanceAssociations - - ssm:PutInventory - - ssm:PutComplianceItems - - ssm:PutConfigurePackageResult - - ssm:UpdateAssociationStatus - - ssm:UpdateInstanceAssociationStatus - - ssm:UpdateInstanceInformation + - ssm:* # SSM Permissions for AmazonSSMManagedInstanceCore policy applied to the NodeInstanceRole - - ssmmessages:CreateControlChannel - - ssmmessages:CreateDataChannel - - ssmmessages:OpenControlChannel - - ssmmessages:OpenDataChannel + - ssmmessages:* # SSM Permissions for AmazonSSMManagedInstanceCore policy applied to the NodeInstanceRole - - ec2messages:AcknowledgeMessage - - ec2messages:DeleteMessage - - ec2messages:FailMessage - - ec2messages:GetEndpoint - - ec2messages:GetMessages - - ec2messages:SendReply + - ec2messages:* - sqs:DeleteMessage - sqs:GetQueueAttributes - sqs:GetQueueUrl - sqs:SendMessage - sqs:ReceiveMessage - pricing:GetProducts - - ec2:DescribeSpotPriceHistory - eks:DescribeCluster Resource: "*" - Effect: Allow @@ -473,6 +311,31 @@ Resources: Resource: - !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/KarpenterNodeRole-*" - !GetAtt FISInterruptionRole.Arn + - Effect: Allow + Action: iam:CreateInstanceProfile + Resource: "*" + Condition: + StringLike: + aws:RequestTag/karpenter.k8s.aws/ec2nodeclass: "*" + - Effect: Allow + Action: iam:TagInstanceProfile + Resource: "*" + Condition: + StringLike: + aws:RequestTag/karpenter.k8s.aws/ec2nodeclass: "*" + aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass: "*" + - Effect: Allow + Action: + - iam:AddRoleToInstanceProfile + - iam:RemoveRoleFromInstanceProfile + - iam:DeleteInstanceProfile + Resource: "*" + Condition: + StringLike: + aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass: "*" + - Effect: Allow + Action: iam:GetInstanceProfile + Resource: "*" - Effect: Allow Action: - aps:RemoteWrite diff --git a/test/suites/alpha/drift/suite_test.go b/test/suites/alpha/drift/suite_test.go index d7fae9c7acb0..c1d318919328 100644 --- a/test/suites/alpha/drift/suite_test.go +++ b/test/suites/alpha/drift/suite_test.go @@ -88,7 +88,6 @@ var _ = Describe("Drift", Label("AWS"), func() { }, }) env.ExpectSettingsOverriddenLegacy(map[string]string{"featureGates.driftEnabled": "true"}) - env.ExpectSettingsOverridden(v1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=true"}) }) It("should deprovision nodes that have drifted due to AMIs", func() { // choose an old static image @@ -122,7 +121,6 @@ var _ = Describe("Drift", Label("AWS"), func() { }) It("should not deprovision nodes that have drifted without the featureGate enabled", func() { env.ExpectSettingsOverriddenLegacy(map[string]string{"featureGates.driftEnabled": "false"}) - env.ExpectSettingsOverridden(v1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=false"}) // choose an old static image parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{ Name: awssdk.String("/aws/service/eks/optimized-ami/1.23/amazon-linux-2/amazon-eks-node-1.23-v20230322/image_id"), diff --git a/test/suites/alpha/expiration/expiration_test.go b/test/suites/alpha/expiration/expiration_test.go index 024a49a81df4..f8eb794921b5 100644 --- a/test/suites/alpha/expiration/expiration_test.go +++ b/test/suites/alpha/expiration/expiration_test.go @@ -74,7 +74,6 @@ var _ = Describe("Expiration", func() { TTLSecondsUntilExpired: ptr.Int64(30), }) env.ExpectSettingsOverriddenLegacy(map[string]string{"featureGates.driftEnabled": "false"}) - env.ExpectSettingsOverridden(v1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=true"}) }) It("should expire the node after the TTLSecondsUntilExpired is reached", func() { var numPods int32 = 1 diff --git a/test/suites/alpha/machine/garbage_collection_test.go b/test/suites/alpha/machine/garbage_collection_test.go index c4b2b0112b23..3d9b140d2c15 100644 --- a/test/suites/alpha/machine/garbage_collection_test.go +++ b/test/suites/alpha/machine/garbage_collection_test.go @@ -25,7 +25,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" @@ -136,7 +135,6 @@ var _ = Describe("NodeClaimGarbageCollection", func() { It("should succeed to garbage collect a Machine that was deleted without the cluster's knowledge", func() { // Disable the interruption queue for the garbage collection test env.ExpectSettingsOverriddenLegacy(map[string]string{"aws.interruptionQueueName": ""}) - env.ExpectSettingsOverridden(v1.EnvVar{Name: "INTERRUPTION_QUEUE", Value: ""}) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, diff --git a/test/suites/alpha/scale/deprovisioning_test.go b/test/suites/alpha/scale/deprovisioning_test.go index f80ebcb6b9bd..482176c098b3 100644 --- a/test/suites/alpha/scale/deprovisioning_test.go +++ b/test/suites/alpha/scale/deprovisioning_test.go @@ -82,7 +82,6 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), BeforeEach(func() { env.ExpectSettingsOverriddenLegacy(map[string]string{"featureGates.driftEnabled": "true"}) - env.ExpectSettingsOverridden(v1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=true"}) nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, diff --git a/test/suites/beta/drift/suite_test.go b/test/suites/beta/drift/suite_test.go index 0d26313f62ae..085d6e395703 100644 --- a/test/suites/beta/drift/suite_test.go +++ b/test/suites/beta/drift/suite_test.go @@ -103,7 +103,6 @@ var _ = Describe("Beta/Drift", Label("AWS"), func() { }, }, }) - env.ExpectSettingsOverriddenLegacy(map[string]string{"featureGates.driftEnabled": "true"}) env.ExpectSettingsOverridden(v1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=true"}) }) It("should disrupt nodes that have drifted due to AMIs", func() { @@ -137,7 +136,6 @@ var _ = Describe("Beta/Drift", Label("AWS"), func() { env.EventuallyExpectNotFound(pod, nodeClaim, node) }) It("should not disrupt nodes that have drifted without the featureGate enabled", func() { - env.ExpectSettingsOverriddenLegacy(map[string]string{"featureGates.driftEnabled": "false"}) env.ExpectSettingsOverridden(v1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=false"}) // choose an old static image parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{ diff --git a/test/suites/beta/integration/cni_test.go b/test/suites/beta/integration/cni_test.go index eb579bd67152..a0c25211081f 100644 --- a/test/suites/beta/integration/cni_test.go +++ b/test/suites/beta/integration/cni_test.go @@ -52,7 +52,6 @@ var _ = Describe("CNITests", func() { Expect(allocatablePods).To(Equal(eniLimitedPodsFor(node.Labels["node.kubernetes.io/instance-type"]))) }) It("should set maxPods when reservedENIs is set", func() { - env.ExpectSettingsOverriddenLegacy(map[string]string{"aws.reservedENIs": "1"}) env.ExpectSettingsOverridden(corev1.EnvVar{Name: "RESERVED_ENIS", Value: "1"}) pod := test.Pod() env.ExpectCreated(pod, nodeClass, nodePool) diff --git a/test/suites/beta/integration/extended_resources_test.go b/test/suites/beta/integration/extended_resources_test.go index ba4681e4ee64..d2ab03eae3e2 100644 --- a/test/suites/beta/integration/extended_resources_test.go +++ b/test/suites/beta/integration/extended_resources_test.go @@ -90,7 +90,6 @@ var _ = Describe("Extended Resources", func() { DeferCleanup(func() { env.ExpectPodENIDisabled() }) - env.ExpectSettingsOverriddenLegacy(map[string]string{"aws.enablePodENI": "true"}) // TODO: remove this requirement once VPC RC rolls out m7a.*, r7a.* ENI data (https://github.com/aws/karpenter/issues/4472) nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, []v1.NodeSelectorRequirement{ { From 7b65fa197eaec747d5ad2ab3fb8054cb3ca88b7a Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Sun, 22 Oct 2023 14:25:45 -0700 Subject: [PATCH 3/9] Fix monitor watcher for NodePool label --- test/pkg/environment/common/monitor.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/pkg/environment/common/monitor.go b/test/pkg/environment/common/monitor.go index 61a5e4a78ee5..dcb432dea6b1 100644 --- a/test/pkg/environment/common/monitor.go +++ b/test/pkg/environment/common/monitor.go @@ -29,6 +29,7 @@ import ( "github.com/samber/lo" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + "github.com/aws/karpenter-core/pkg/apis/v1beta1" "github.com/aws/karpenter-core/pkg/utils/resources" ) @@ -223,7 +224,8 @@ func (m *Monitor) nodeUtilization(resource v1.ResourceName) []float64 { for nodeName, requests := range st.nodeRequests { allocatable := st.nodes[nodeName].Status.Allocatable[resource] // skip any nodes we didn't launch - if _, ok := st.nodes[nodeName].Labels[v1alpha5.ProvisionerNameLabelKey]; !ok { + if st.nodes[nodeName].Labels[v1alpha5.ProvisionerNameLabelKey] == "" && + st.nodes[nodeName].Labels[v1beta1.NodePoolLabelKey] == "" { continue } if allocatable.IsZero() { From c3dce679d5cedf8a5a55528a99e8989b2fcd3915 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Sun, 22 Oct 2023 14:33:37 -0700 Subject: [PATCH 4/9] Change cordoned node check for beta --- test/suites/alpha/drift/suite_test.go | 4 ++-- test/suites/alpha/expiration/expiration_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/suites/alpha/drift/suite_test.go b/test/suites/alpha/drift/suite_test.go index c1d318919328..0b5e735789a1 100644 --- a/test/suites/alpha/drift/suite_test.go +++ b/test/suites/alpha/drift/suite_test.go @@ -375,7 +375,7 @@ var _ = Describe("Drift", Label("AWS"), func() { }).Should(Succeed()) // Expect nodes To get cordoned - cordonedNodes := env.EventuallyExpectCordonedNodeCount("==", 1) + cordonedNodes := env.EventuallyExpectCordonedNodeCountLegacy("==", 1) // Drift should fail and the original node should be uncordoned // TODO: reduce timeouts when deprovisioning waits are factored out @@ -433,7 +433,7 @@ var _ = Describe("Drift", Label("AWS"), func() { }).Should(Succeed()) // Expect nodes To be cordoned - cordonedNodes := env.EventuallyExpectCordonedNodeCount("==", 1) + cordonedNodes := env.EventuallyExpectCordonedNodeCountLegacy("==", 1) // Drift should fail and original node should be uncordoned // TODO: reduce timeouts when deprovisioning waits are factored outr diff --git a/test/suites/alpha/expiration/expiration_test.go b/test/suites/alpha/expiration/expiration_test.go index f8eb794921b5..15621c59d700 100644 --- a/test/suites/alpha/expiration/expiration_test.go +++ b/test/suites/alpha/expiration/expiration_test.go @@ -237,7 +237,7 @@ var _ = Describe("Expiration", func() { }).Should(Succeed()) // Expect nodes To get cordoned - cordonedNodes := env.EventuallyExpectCordonedNodeCount("==", 1) + cordonedNodes := env.EventuallyExpectCordonedNodeCountLegacy("==", 1) // Expire should fail and the original node should be uncordoned // TODO: reduce timeouts when deprovisioning waits are factored out @@ -305,7 +305,7 @@ var _ = Describe("Expiration", func() { }).Should(Succeed()) // Expect nodes To be cordoned - cordonedNodes := env.EventuallyExpectCordonedNodeCount("==", 1) + cordonedNodes := env.EventuallyExpectCordonedNodeCountLegacy("==", 1) // Expire should fail and original node should be uncordoned and no machines should be removed // TODO: reduce timeouts when deprovisioning waits are factored out From 69d3a40657fe52c9f29a2eb768ae0cc5cb5c7e24 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Sun, 22 Oct 2023 14:54:29 -0700 Subject: [PATCH 5/9] Handle uncordoning nodes for beta --- test/suites/alpha/drift/suite_test.go | 10 ++-------- test/suites/beta/drift/suite_test.go | 10 ++-------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/test/suites/alpha/drift/suite_test.go b/test/suites/alpha/drift/suite_test.go index 0b5e735789a1..d27a7fcc5a07 100644 --- a/test/suites/alpha/drift/suite_test.go +++ b/test/suites/alpha/drift/suite_test.go @@ -379,10 +379,7 @@ var _ = Describe("Drift", Label("AWS"), func() { // Drift should fail and the original node should be uncordoned // TODO: reduce timeouts when deprovisioning waits are factored out - Eventually(func(g Gomega) { - g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(cordonedNodes[0]), cordonedNodes[0])) - g.Expect(cordonedNodes[0].Spec.Unschedulable).To(BeFalse()) - }).WithTimeout(11 * time.Minute).Should(Succeed()) + env.EventuallyExpectNodesUncordonedLegacyWithTimeout(12*time.Minute, cordonedNodes...) Eventually(func(g Gomega) { machines := &v1alpha5.MachineList{} @@ -437,10 +434,7 @@ var _ = Describe("Drift", Label("AWS"), func() { // Drift should fail and original node should be uncordoned // TODO: reduce timeouts when deprovisioning waits are factored outr - Eventually(func(g Gomega) { - g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(cordonedNodes[0]), cordonedNodes[0])) - g.Expect(cordonedNodes[0].Spec.Unschedulable).To(BeFalse()) - }).WithTimeout(12 * time.Minute).Should(Succeed()) + env.EventuallyExpectNodesUncordonedLegacyWithTimeout(12*time.Minute, cordonedNodes...) // Expect that the new machine/node is kept around after the un-cordon nodeList := &v1.NodeList{} diff --git a/test/suites/beta/drift/suite_test.go b/test/suites/beta/drift/suite_test.go index 085d6e395703..a3a0530a53c3 100644 --- a/test/suites/beta/drift/suite_test.go +++ b/test/suites/beta/drift/suite_test.go @@ -431,10 +431,7 @@ var _ = Describe("Beta/Drift", Label("AWS"), func() { // Drift should fail and the original node should be uncordoned // TODO: reduce timeouts when disruption waits are factored out - Eventually(func(g Gomega) { - g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(cordonedNodes[0]), cordonedNodes[0])) - g.Expect(cordonedNodes[0].Spec.Unschedulable).To(BeFalse()) - }).WithTimeout(11 * time.Minute).Should(Succeed()) + env.EventuallyExpectNodesUncordonedWithTimeout(11*time.Minute, cordonedNodes...) Eventually(func(g Gomega) { nodeClaims := &corev1beta1.NodeClaimList{} @@ -489,10 +486,7 @@ var _ = Describe("Beta/Drift", Label("AWS"), func() { // Drift should fail and original node should be uncordoned // TODO: reduce timeouts when disruption waits are factored outr - Eventually(func(g Gomega) { - g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(cordonedNodes[0]), cordonedNodes[0])) - g.Expect(cordonedNodes[0].Spec.Unschedulable).To(BeFalse()) - }).WithTimeout(12 * time.Minute).Should(Succeed()) + env.EventuallyExpectNodesUncordonedWithTimeout(12*time.Minute, cordonedNodes...) // Expect that the new nodeClaim/node is kept around after the un-cordon nodeList := &v1.NodeList{} From 0c566973cf183d88565e1ce8e827b80e579e8e0a Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Sun, 22 Oct 2023 15:04:07 -0700 Subject: [PATCH 6/9] Fix instance profile integration tests --- test/pkg/environment/aws/expectations.go | 9 +++++++++ test/suites/beta/integration/instance_profile_test.go | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/test/pkg/environment/aws/expectations.go b/test/pkg/environment/aws/expectations.go index 6d910e99eb85..a91eafa50719 100644 --- a/test/pkg/environment/aws/expectations.go +++ b/test/pkg/environment/aws/expectations.go @@ -27,12 +27,15 @@ import ( "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/ssm" "github.com/aws/aws-sdk-go/service/sts" + "github.com/mitchellh/hashstructure/v2" . "github.com/onsi/ginkgo/v2" //nolint:revive,stylecheck . "github.com/onsi/gomega" //nolint:revive,stylecheck "github.com/samber/lo" "go.uber.org/multierr" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + + "github.com/aws/karpenter/pkg/apis/v1beta1" ) // Spot Interruption experiment details partially copied from @@ -121,6 +124,12 @@ func (env *Environment) ExpectInstanceProfileExists(profileName string) iam.Inst return lo.FromPtr(out.InstanceProfile) } +// GetInstanceProfileName gets the string for the profile name based on the cluster name, region and the NodeClass name. +// The length of this string can never exceed the maximum instance profile name limit of 128 characters. +func (env *Environment) GetInstanceProfileName(nodeClass *v1beta1.EC2NodeClass) string { + return fmt.Sprintf("%s_%d", env.ClusterName, lo.Must(hashstructure.Hash(fmt.Sprintf("%s%s", env.Region, nodeClass.Name), hashstructure.FormatV2, nil))) +} + func (env *Environment) GetInstance(nodeName string) ec2.Instance { node := env.Environment.GetNode(nodeName) return env.GetInstanceByID(env.ExpectParsedProviderID(node.Spec.ProviderID)) diff --git a/test/suites/beta/integration/instance_profile_test.go b/test/suites/beta/integration/instance_profile_test.go index b40432aae5eb..c13bbada7020 100644 --- a/test/suites/beta/integration/instance_profile_test.go +++ b/test/suites/beta/integration/instance_profile_test.go @@ -35,7 +35,7 @@ var _ = Describe("InstanceProfile Generation", func() { instance := env.GetInstance(node.Name) Expect(instance.IamInstanceProfile).ToNot(BeNil()) - Expect(instance.IamInstanceProfile.Arn).To(ContainSubstring(nodeClass.Spec.Role)) + Expect(lo.FromPtr(instance.IamInstanceProfile.Arn)).To(ContainSubstring(nodeClass.Status.InstanceProfile)) instanceProfile := env.ExpectInstanceProfileExists(instanceprofile.GetProfileName(env.Context, env.Region, nodeClass)) Expect(instanceProfile.Roles).To(HaveLen(1)) @@ -50,7 +50,7 @@ var _ = Describe("InstanceProfile Generation", func() { env.ExpectDeleted(nodePool, nodeClass) Eventually(func(g Gomega) { _, err := env.IAMAPI.GetInstanceProfileWithContext(env.Context, &iam.GetInstanceProfileInput{ - InstanceProfileName: aws.String(instanceprofile.GetProfileName(env.Context, env.Region, nodeClass)), + InstanceProfileName: aws.String(env.GetInstanceProfileName(nodeClass)), }) g.Expect(awserrors.IsNotFound(err)).To(BeTrue()) }).Should(Succeed()) From e6f4610c300177ba491abc26165e477317571ff9 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Sun, 22 Oct 2023 15:07:04 -0700 Subject: [PATCH 7/9] Fix AMISelectorTerms testing --- test/pkg/environment/aws/environment.go | 4 ++++ test/suites/alpha/drift/suite_test.go | 6 +++--- test/suites/alpha/integration/ami_test.go | 2 +- .../alpha/integration/extended_resources_test.go | 3 ++- .../suites/alpha/integration/kubelet_config_test.go | 4 ++-- test/suites/beta/drift/suite_test.go | 5 +++-- test/suites/beta/integration/ami_test.go | 13 +++++++++---- .../beta/integration/extended_resources_test.go | 3 ++- test/suites/beta/integration/kubelet_config_test.go | 4 ++-- test/suites/beta/integration/scheduling_test.go | 2 +- 10 files changed, 29 insertions(+), 17 deletions(-) diff --git a/test/pkg/environment/aws/environment.go b/test/pkg/environment/aws/environment.go index 9fc111b9759c..e5e95c962f87 100644 --- a/test/pkg/environment/aws/environment.go +++ b/test/pkg/environment/aws/environment.go @@ -42,6 +42,10 @@ import ( const WindowsDefaultImage = "mcr.microsoft.com/oss/kubernetes/pause:3.9" +// ExcludedInstanceFamilies denotes instance families that have issues during resource registration due to compatability +// issues with versions of the VPR Resource Controller +var ExcludedInstanceFamilies = []string{"m7a", "r7a", "c7a", "r7i"} + type Environment struct { *common.Environment Region string diff --git a/test/suites/alpha/drift/suite_test.go b/test/suites/alpha/drift/suite_test.go index d27a7fcc5a07..3fce8ca0e58a 100644 --- a/test/suites/alpha/drift/suite_test.go +++ b/test/suites/alpha/drift/suite_test.go @@ -379,7 +379,7 @@ var _ = Describe("Drift", Label("AWS"), func() { // Drift should fail and the original node should be uncordoned // TODO: reduce timeouts when deprovisioning waits are factored out - env.EventuallyExpectNodesUncordonedLegacyWithTimeout(12*time.Minute, cordonedNodes...) + env.EventuallyExpectNodesUncordonedLegacyWithTimeout(11*time.Minute, cordonedNodes...) Eventually(func(g Gomega) { machines := &v1alpha5.MachineList{} @@ -433,8 +433,8 @@ var _ = Describe("Drift", Label("AWS"), func() { cordonedNodes := env.EventuallyExpectCordonedNodeCountLegacy("==", 1) // Drift should fail and original node should be uncordoned - // TODO: reduce timeouts when deprovisioning waits are factored outr - env.EventuallyExpectNodesUncordonedLegacyWithTimeout(12*time.Minute, cordonedNodes...) + // TODO: reduce timeouts when deprovisioning waits are factored out + env.EventuallyExpectNodesUncordonedLegacyWithTimeout(11*time.Minute, cordonedNodes...) // Expect that the new machine/node is kept around after the un-cordon nodeList := &v1.NodeList{} diff --git a/test/suites/alpha/integration/ami_test.go b/test/suites/alpha/integration/ami_test.go index 44625235af31..40126578322c 100644 --- a/test/suites/alpha/integration/ami_test.go +++ b/test/suites/alpha/integration/ami_test.go @@ -372,7 +372,7 @@ var _ = Describe("AMI", func() { { Key: v1alpha1.LabelInstanceFamily, Operator: v1.NodeSelectorOpNotIn, - Values: []string{"m7a", "r7a", "c7a"}, + Values: awsenv.ExcludedInstanceFamilies, }, { Key: v1alpha1.LabelInstanceCategory, diff --git a/test/suites/alpha/integration/extended_resources_test.go b/test/suites/alpha/integration/extended_resources_test.go index 98de604d8e4b..e53adbbb576d 100644 --- a/test/suites/alpha/integration/extended_resources_test.go +++ b/test/suites/alpha/integration/extended_resources_test.go @@ -31,6 +31,7 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" "github.com/aws/karpenter/pkg/apis/v1alpha1" + awsenv "github.com/aws/karpenter/test/pkg/environment/aws" awstest "github.com/aws/karpenter/pkg/test" ) @@ -135,7 +136,7 @@ var _ = Describe("Extended Resources", func() { { Key: v1alpha1.LabelInstanceFamily, Operator: v1.NodeSelectorOpNotIn, - Values: []string{"m7a", "r7a"}, + Values: awsenv.ExcludedInstanceFamilies, }, }, }) diff --git a/test/suites/alpha/integration/kubelet_config_test.go b/test/suites/alpha/integration/kubelet_config_test.go index 5f9afec6a7bc..1e023001214b 100644 --- a/test/suites/alpha/integration/kubelet_config_test.go +++ b/test/suites/alpha/integration/kubelet_config_test.go @@ -107,7 +107,7 @@ var _ = Describe("KubeletConfiguration Overrides", func() { v1.NodeSelectorRequirement{ Key: v1alpha1.LabelInstanceFamily, Operator: v1.NodeSelectorOpNotIn, - Values: []string{"m7a", "r7a"}, + Values: aws.ExcludedInstanceFamilies, }) pod := test.Pod(test.PodOptions{ NodeSelector: map[string]string{ @@ -145,7 +145,7 @@ var _ = Describe("KubeletConfiguration Overrides", func() { v1.NodeSelectorRequirement{ Key: v1alpha1.LabelInstanceFamily, Operator: v1.NodeSelectorOpNotIn, - Values: []string{"m7a", "r7a", "c7a"}, + Values: aws.ExcludedInstanceFamilies, }, v1.NodeSelectorRequirement{ Key: v1alpha1.LabelInstanceCategory, diff --git a/test/suites/beta/drift/suite_test.go b/test/suites/beta/drift/suite_test.go index a3a0530a53c3..b1872f270dec 100644 --- a/test/suites/beta/drift/suite_test.go +++ b/test/suites/beta/drift/suite_test.go @@ -433,6 +433,7 @@ var _ = Describe("Beta/Drift", Label("AWS"), func() { // TODO: reduce timeouts when disruption waits are factored out env.EventuallyExpectNodesUncordonedWithTimeout(11*time.Minute, cordonedNodes...) + // We give another 6 minutes here to handle the deletion at the 15m registration timeout Eventually(func(g Gomega) { nodeClaims := &corev1beta1.NodeClaimList{} g.Expect(env.Client.List(env, nodeClaims, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) @@ -485,8 +486,8 @@ var _ = Describe("Beta/Drift", Label("AWS"), func() { cordonedNodes := env.EventuallyExpectCordonedNodeCount("==", 1) // Drift should fail and original node should be uncordoned - // TODO: reduce timeouts when disruption waits are factored outr - env.EventuallyExpectNodesUncordonedWithTimeout(12*time.Minute, cordonedNodes...) + // TODO: reduce timeouts when disruption waits are factored out + env.EventuallyExpectNodesUncordonedWithTimeout(11*time.Minute, cordonedNodes...) // Expect that the new nodeClaim/node is kept around after the un-cordon nodeList := &v1.NodeList{} diff --git a/test/suites/beta/integration/ami_test.go b/test/suites/beta/integration/ami_test.go index bc2ddc96fab3..070be1f4ff3c 100644 --- a/test/suites/beta/integration/ami_test.go +++ b/test/suites/beta/integration/ami_test.go @@ -41,8 +41,13 @@ var _ = Describe("AMI", func() { customAMI = env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", 1) }) - It("should use the AMI defined by the AMI Selector", func() { + It("should use the AMI defined by the AMI Selector Terms", func() { pod := coretest.Pod() + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + ID: customAMI, + }, + } env.ExpectCreated(pod, nodeClass, nodePool) env.EventuallyExpectHealthy(pod) env.ExpectCreatedNodeCount("==", 1) @@ -74,7 +79,7 @@ var _ = Describe("AMI", func() { env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI)))) }) - It("should support ami selector Name but fail with incorrect owners", func() { + It("should support AMI Selector Terms for Name but fail with incorrect owners", func() { output, err := env.EC2API.DescribeImages(&ec2.DescribeImagesInput{ ImageIds: []*string{aws.String(customAMI)}, }) @@ -155,7 +160,7 @@ var _ = Describe("AMI", func() { { Key: v1beta1.LabelInstanceFamily, Operator: v1.NodeSelectorOpNotIn, - Values: []string{"m7a", "r7a", "c7a"}, + Values: awsenv.ExcludedInstanceFamilies, }, { Key: v1beta1.LabelInstanceCategory, @@ -290,7 +295,7 @@ var _ = Describe("AMI", func() { { Key: v1beta1.LabelInstanceFamily, Operator: v1.NodeSelectorOpNotIn, - Values: []string{"m7a", "r7a", "c7a"}, + Values: awsenv.ExcludedInstanceFamilies, }, { Key: v1beta1.LabelInstanceCategory, diff --git a/test/suites/beta/integration/extended_resources_test.go b/test/suites/beta/integration/extended_resources_test.go index d2ab03eae3e2..f8dc328b6fe4 100644 --- a/test/suites/beta/integration/extended_resources_test.go +++ b/test/suites/beta/integration/extended_resources_test.go @@ -30,6 +30,7 @@ import ( "github.com/aws/karpenter-core/pkg/test" "github.com/aws/karpenter/pkg/apis/v1beta1" + awsenv "github.com/aws/karpenter/test/pkg/environment/aws" ) var _ = Describe("Extended Resources", func() { @@ -95,7 +96,7 @@ var _ = Describe("Extended Resources", func() { { Key: v1beta1.LabelInstanceFamily, Operator: v1.NodeSelectorOpNotIn, - Values: []string{"m7a", "r7a", "c7a"}, + Values: awsenv.ExcludedInstanceFamilies, }, { Key: v1beta1.LabelInstanceCategory, diff --git a/test/suites/beta/integration/kubelet_config_test.go b/test/suites/beta/integration/kubelet_config_test.go index 729c2b1c4576..3ec78ccd6fb6 100644 --- a/test/suites/beta/integration/kubelet_config_test.go +++ b/test/suites/beta/integration/kubelet_config_test.go @@ -90,7 +90,7 @@ var _ = Describe("KubeletConfiguration Overrides", func() { { Key: v1beta1.LabelInstanceFamily, Operator: v1.NodeSelectorOpNotIn, - Values: []string{"m7a", "r7a", "c7a"}, + Values: aws.ExcludedInstanceFamilies, }, { Key: v1beta1.LabelInstanceCategory, @@ -133,7 +133,7 @@ var _ = Describe("KubeletConfiguration Overrides", func() { { Key: v1beta1.LabelInstanceFamily, Operator: v1.NodeSelectorOpNotIn, - Values: []string{"m7a", "r7a", "c7a"}, + Values: aws.ExcludedInstanceFamilies, }, { Key: v1beta1.LabelInstanceCategory, diff --git a/test/suites/beta/integration/scheduling_test.go b/test/suites/beta/integration/scheduling_test.go index 985cc06484b5..8c3511988762 100644 --- a/test/suites/beta/integration/scheduling_test.go +++ b/test/suites/beta/integration/scheduling_test.go @@ -239,7 +239,7 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { { Key: v1beta1.LabelInstanceFamily, Operator: v1.NodeSelectorOpNotIn, - Values: []string{"m7a", "r7a", "c7a"}, + Values: aws.ExcludedInstanceFamilies, }, { Key: v1beta1.LabelInstanceCategory, From 1fdff8a0f3c44f14e469b8e715871bff94d301f1 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Sun, 22 Oct 2023 21:42:27 -0700 Subject: [PATCH 8/9] Fix other integration testing --- test/pkg/environment/aws/environment.go | 2 +- test/suites/beta/integration/emptiness_test.go | 1 - test/suites/beta/integration/instance_profile_test.go | 3 +-- test/suites/beta/integration/kubelet_config_test.go | 2 +- test/suites/beta/integration/scheduling_test.go | 1 - 5 files changed, 3 insertions(+), 6 deletions(-) diff --git a/test/pkg/environment/aws/environment.go b/test/pkg/environment/aws/environment.go index e5e95c962f87..f55bcbdbcaca 100644 --- a/test/pkg/environment/aws/environment.go +++ b/test/pkg/environment/aws/environment.go @@ -42,7 +42,7 @@ import ( const WindowsDefaultImage = "mcr.microsoft.com/oss/kubernetes/pause:3.9" -// ExcludedInstanceFamilies denotes instance families that have issues during resource registration due to compatability +// ExcludedInstanceFamilies denotes instance families that have issues during resource registration due to compatibility // issues with versions of the VPR Resource Controller var ExcludedInstanceFamilies = []string{"m7a", "r7a", "c7a", "r7i"} diff --git a/test/suites/beta/integration/emptiness_test.go b/test/suites/beta/integration/emptiness_test.go index 9c48e694a9e8..de4dc2ccc3f3 100644 --- a/test/suites/beta/integration/emptiness_test.go +++ b/test/suites/beta/integration/emptiness_test.go @@ -52,7 +52,6 @@ var _ = Describe("Emptiness", func() { By("waiting for the nodeclaim emptiness status condition to propagate") Eventually(func(g Gomega) { g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) - g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Empty)).ToNot(BeNil()) g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Empty).IsTrue()).To(BeTrue()) }).Should(Succeed()) diff --git a/test/suites/beta/integration/instance_profile_test.go b/test/suites/beta/integration/instance_profile_test.go index c13bbada7020..0d34be68507e 100644 --- a/test/suites/beta/integration/instance_profile_test.go +++ b/test/suites/beta/integration/instance_profile_test.go @@ -23,7 +23,6 @@ import ( coretest "github.com/aws/karpenter-core/pkg/test" awserrors "github.com/aws/karpenter/pkg/errors" - "github.com/aws/karpenter/pkg/providers/instanceprofile" ) var _ = Describe("InstanceProfile Generation", func() { @@ -37,7 +36,7 @@ var _ = Describe("InstanceProfile Generation", func() { Expect(instance.IamInstanceProfile).ToNot(BeNil()) Expect(lo.FromPtr(instance.IamInstanceProfile.Arn)).To(ContainSubstring(nodeClass.Status.InstanceProfile)) - instanceProfile := env.ExpectInstanceProfileExists(instanceprofile.GetProfileName(env.Context, env.Region, nodeClass)) + instanceProfile := env.ExpectInstanceProfileExists(env.GetInstanceProfileName(nodeClass)) Expect(instanceProfile.Roles).To(HaveLen(1)) Expect(lo.FromPtr(instanceProfile.Roles[0].RoleName)).To(Equal(nodeClass.Spec.Role)) }) diff --git a/test/suites/beta/integration/kubelet_config_test.go b/test/suites/beta/integration/kubelet_config_test.go index 3ec78ccd6fb6..1a7ac9c23403 100644 --- a/test/suites/beta/integration/kubelet_config_test.go +++ b/test/suites/beta/integration/kubelet_config_test.go @@ -259,7 +259,7 @@ var _ = Describe("KubeletConfiguration Overrides", func() { Values: []string{string(v1.Linux)}, }, }...) - nodePool.Spec.Template.Spec.Kubelet.PodsPerCore = ptr.Int32(1) + nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{PodsPerCore: ptr.Int32(1)} numPods := 6 dep := test.Deployment(test.DeploymentOptions{ Replicas: int32(numPods), diff --git a/test/suites/beta/integration/scheduling_test.go b/test/suites/beta/integration/scheduling_test.go index 8c3511988762..39013fb6c27b 100644 --- a/test/suites/beta/integration/scheduling_test.go +++ b/test/suites/beta/integration/scheduling_test.go @@ -69,7 +69,6 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { v1beta1.LabelInstanceCPU: "2", v1beta1.LabelInstanceMemory: "4096", v1beta1.LabelInstanceNetworkBandwidth: "750", - v1beta1.LabelInstancePods: "29", } selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { From 83c66c59f1c50a91498576c22824b5fc601f4f86 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Sun, 22 Oct 2023 23:09:46 -0700 Subject: [PATCH 9/9] Add utilization testing to integration --- .../utilization_test.go} | 23 +--------- .../beta/integration/utilization_test.go | 43 +++++++++++++++++++ 2 files changed, 44 insertions(+), 22 deletions(-) rename test/suites/alpha/{utilization/suite_test.go => integration/utilization_test.go} (81%) create mode 100644 test/suites/beta/integration/utilization_test.go diff --git a/test/suites/alpha/utilization/suite_test.go b/test/suites/alpha/integration/utilization_test.go similarity index 81% rename from test/suites/alpha/utilization/suite_test.go rename to test/suites/alpha/integration/utilization_test.go index bfaf8524a1f1..135e66a40268 100644 --- a/test/suites/alpha/utilization/suite_test.go +++ b/test/suites/alpha/integration/utilization_test.go @@ -12,13 +12,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -package utilization_test +package integration_test import ( - "testing" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/labels" @@ -29,26 +26,8 @@ import ( "github.com/aws/karpenter/test/pkg/debug" awstest "github.com/aws/karpenter/pkg/test" - "github.com/aws/karpenter/test/pkg/environment/aws" ) -var env *aws.Environment - -func TestUtilization(t *testing.T) { - RegisterFailHandler(Fail) - BeforeSuite(func() { - env = aws.NewEnvironment(t) - }) - AfterSuite(func() { - env.Stop() - }) - RunSpecs(t, "Alpha/Utilization") -} - -var _ = BeforeEach(func() { env.BeforeEach() }) -var _ = AfterEach(func() { env.Cleanup() }) -var _ = AfterEach(func() { env.AfterEach() }) - var _ = Describe("Utilization", Label(debug.NoWatch), Label(debug.NoEvents), func() { It("should provision one pod per node", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ diff --git a/test/suites/beta/integration/utilization_test.go b/test/suites/beta/integration/utilization_test.go new file mode 100644 index 000000000000..a84084cdbe90 --- /dev/null +++ b/test/suites/beta/integration/utilization_test.go @@ -0,0 +1,43 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration_test + +import ( + . "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/labels" + + "github.com/aws/karpenter-core/pkg/test" + "github.com/aws/karpenter/test/pkg/debug" +) + +var _ = Describe("Utilization", Label(debug.NoWatch), Label(debug.NoEvents), func() { + It("should provision one pod per node", func() { + nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"t3a.small"}, + }) + + deployment := test.Deployment(test.DeploymentOptions{ + Replicas: 100, + PodOptions: test.PodOptions{ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1.5")}}}}) + + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", int(*deployment.Spec.Replicas)) // One pod per node enforced by instance size + }) +})