From 531d7e00ad69bed8fa74de79abdaf8d61b476910 Mon Sep 17 00:00:00 2001 From: jigisha620 Date: Mon, 10 Jun 2024 11:03:27 -0700 Subject: [PATCH] ci: move tests out from integration suite --- .github/workflows/e2e-matrix.yaml | 6 ++ .../ami_test.go => ami/suite_test.go} | 65 +++++++++++++------ .../testdata/al2023_userdata_input.yaml | 0 .../testdata/al2_no_mime_userdata_input.sh | 0 .../testdata/al2_userdata_input.sh | 0 .../testdata/br_userdata_input.sh | 0 .../testdata/windows_userdata_input.ps1 | 0 .../suite_test.go} | 45 +++++++++---- .../storage_test.go => storage/suite_test.go} | 40 ++++++++++-- 9 files changed, 120 insertions(+), 36 deletions(-) rename test/suites/{integration/ami_test.go => ami/suite_test.go} (92%) rename test/suites/{integration => ami}/testdata/al2023_userdata_input.yaml (100%) rename test/suites/{integration => ami}/testdata/al2_no_mime_userdata_input.sh (100%) rename test/suites/{integration => ami}/testdata/al2_userdata_input.sh (100%) rename test/suites/{integration => ami}/testdata/br_userdata_input.sh (100%) rename test/suites/{integration => ami}/testdata/windows_userdata_input.ps1 (100%) rename test/suites/{integration/scheduling_test.go => scheduling/suite_test.go} (95%) rename test/suites/{integration/storage_test.go => storage/suite_test.go} (91%) diff --git a/.github/workflows/e2e-matrix.yaml b/.github/workflows/e2e-matrix.yaml index df781412619f..ebd64b4c8fe1 100644 --- a/.github/workflows/e2e-matrix.yaml +++ b/.github/workflows/e2e-matrix.yaml @@ -63,6 +63,12 @@ jobs: suite: - name: Integration region: ${{ inputs.region }} + - name: AMI + region: ${{ inputs.region }} + - name: Scheduling + region: ${{ inputs.region }} + - name: Storage + region: ${{ inputs.region }} - name: NodeClaim region: ${{ inputs.region }} - name: Consolidation diff --git a/test/suites/integration/ami_test.go b/test/suites/ami/suite_test.go similarity index 92% rename from test/suites/integration/ami_test.go rename to test/suites/ami/suite_test.go index 2bee99f8444e..655b06e95591 100644 --- a/test/suites/integration/ami_test.go +++ b/test/suites/ami/suite_test.go @@ -12,16 +12,26 @@ See the License for the specific language governing permissions and limitations under the License. */ -package integration_test +package ami_test import ( + "testing" + + awssdk "github.com/aws/aws-sdk-go/aws" + + corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "encoding/base64" "fmt" "os" "strings" "time" - "github.com/aws/aws-sdk-go/aws" + "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/aws/aws-sdk-go/service/ec2" "github.com/awslabs/operatorpkg/status" . "github.com/awslabs/operatorpkg/test/expectations" @@ -32,16 +42,33 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + environmentaws "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" + coretest "sigs.k8s.io/karpenter/pkg/test" +) - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" +var env *environmentaws.Environment +var nodeClass *v1beta1.EC2NodeClass +var nodePool *corev1beta1.NodePool - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" - awsenv "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" +func TestAMI(t *testing.T) { + RegisterFailHandler(Fail) + BeforeSuite(func() { + env = environmentaws.NewEnvironment(t) + }) + AfterSuite(func() { + env.Stop() + }) + RunSpecs(t, "Ami") +} - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) +var _ = BeforeEach(func() { + env.BeforeEach() + nodeClass = env.DefaultEC2NodeClass() + nodePool = env.DefaultNodePool(nodeClass) +}) +var _ = AfterEach(func() { env.Cleanup() }) +var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("AMI", func() { var customAMI string @@ -83,7 +110,7 @@ var _ = Describe("AMI", func() { }) It("should support AMI Selector Terms for Name but fail with incorrect owners", func() { output, err := env.EC2API.DescribeImages(&ec2.DescribeImagesInput{ - ImageIds: []*string{aws.String(customAMI)}, + ImageIds: []*string{awssdk.String(customAMI)}, }) Expect(err).To(BeNil()) Expect(output.Images).To(HaveLen(1)) @@ -101,7 +128,7 @@ var _ = Describe("AMI", func() { }) It("should support ami selector Name with default owners", func() { output, err := env.EC2API.DescribeImages(&ec2.DescribeImagesInput{ - ImageIds: []*string{aws.String(customAMI)}, + ImageIds: []*string{awssdk.String(customAMI)}, }) Expect(err).To(BeNil()) Expect(output.Images).To(HaveLen(1)) @@ -174,7 +201,7 @@ var _ = Describe("AMI", func() { NodeSelectorRequirement: v1.NodeSelectorRequirement{ Key: v1beta1.LabelInstanceFamily, Operator: v1.NodeSelectorOpNotIn, - Values: awsenv.ExcludedInstanceFamilies, + Values: environmentaws.ExcludedInstanceFamilies, }, }, ) @@ -241,7 +268,7 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/al2_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 - nodeClass.Spec.UserData = aws.String(string(content)) + nodeClass.Spec.UserData = awssdk.String(string(content)) nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} pod := coretest.Pod(coretest.PodOptions{Tolerations: []v1.Toleration{{Key: "example.com", Operator: v1.TolerationOpExists}}}) @@ -262,7 +289,7 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/al2_no_mime_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 - nodeClass.Spec.UserData = aws.String(string(content)) + nodeClass.Spec.UserData = awssdk.String(string(content)) nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} pod := coretest.Pod(coretest.PodOptions{Tolerations: []v1.Toleration{{Key: "example.com", Operator: v1.TolerationOpExists}}}) @@ -283,7 +310,7 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/br_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket - nodeClass.Spec.UserData = aws.String(string(content)) + nodeClass.Spec.UserData = awssdk.String(string(content)) nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} pod := coretest.Pod(coretest.PodOptions{Tolerations: []v1.Toleration{{Key: "example.com", Operator: v1.TolerationOpExists}}}) @@ -307,7 +334,7 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/windows_userdata_input.ps1") Expect(err).ToNot(HaveOccurred()) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyWindows2022 - nodeClass.Spec.UserData = aws.String(string(content)) + nodeClass.Spec.UserData = awssdk.String(string(content)) nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} @@ -317,7 +344,7 @@ var _ = Describe("AMI", func() { NodeSelectorRequirement: v1.NodeSelectorRequirement{ Key: v1beta1.LabelInstanceFamily, Operator: v1.NodeSelectorOpNotIn, - Values: awsenv.ExcludedInstanceFamilies, + Values: environmentaws.ExcludedInstanceFamilies, }, }, corev1beta1.NodeSelectorRequirementWithMinValues{ @@ -329,7 +356,7 @@ var _ = Describe("AMI", func() { }, ) pod := coretest.Pod(coretest.PodOptions{ - Image: awsenv.WindowsDefaultImage, + Image: environmentaws.WindowsDefaultImage, NodeSelector: map[string]string{ v1.LabelOSStable: string(v1.Windows), v1.LabelWindowsBuild: "10.0.20348", @@ -358,8 +385,8 @@ func getInstanceAttribute(nodeName string, attribute string) *ec2.DescribeInstan providerIDSplit := strings.Split(node.Spec.ProviderID, "/") instanceID := providerIDSplit[len(providerIDSplit)-1] instanceAttribute, err := env.EC2API.DescribeInstanceAttribute(&ec2.DescribeInstanceAttributeInput{ - InstanceId: aws.String(instanceID), - Attribute: aws.String(attribute), + InstanceId: awssdk.String(instanceID), + Attribute: awssdk.String(attribute), }) Expect(err).ToNot(HaveOccurred()) return instanceAttribute diff --git a/test/suites/integration/testdata/al2023_userdata_input.yaml b/test/suites/ami/testdata/al2023_userdata_input.yaml similarity index 100% rename from test/suites/integration/testdata/al2023_userdata_input.yaml rename to test/suites/ami/testdata/al2023_userdata_input.yaml diff --git a/test/suites/integration/testdata/al2_no_mime_userdata_input.sh b/test/suites/ami/testdata/al2_no_mime_userdata_input.sh similarity index 100% rename from test/suites/integration/testdata/al2_no_mime_userdata_input.sh rename to test/suites/ami/testdata/al2_no_mime_userdata_input.sh diff --git a/test/suites/integration/testdata/al2_userdata_input.sh b/test/suites/ami/testdata/al2_userdata_input.sh similarity index 100% rename from test/suites/integration/testdata/al2_userdata_input.sh rename to test/suites/ami/testdata/al2_userdata_input.sh diff --git a/test/suites/integration/testdata/br_userdata_input.sh b/test/suites/ami/testdata/br_userdata_input.sh similarity index 100% rename from test/suites/integration/testdata/br_userdata_input.sh rename to test/suites/ami/testdata/br_userdata_input.sh diff --git a/test/suites/integration/testdata/windows_userdata_input.ps1 b/test/suites/ami/testdata/windows_userdata_input.ps1 similarity index 100% rename from test/suites/integration/testdata/windows_userdata_input.ps1 rename to test/suites/ami/testdata/windows_userdata_input.ps1 diff --git a/test/suites/integration/scheduling_test.go b/test/suites/scheduling/suite_test.go similarity index 95% rename from test/suites/integration/scheduling_test.go rename to test/suites/scheduling/suite_test.go index be328ed5cdd2..14a76bb8c434 100644 --- a/test/suites/integration/scheduling_test.go +++ b/test/suites/scheduling/suite_test.go @@ -12,10 +12,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -package integration_test +package scheduling_test import ( "fmt" + "testing" "time" "github.com/samber/lo" @@ -30,12 +31,34 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/test/pkg/debug" - "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" + environmentaws "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) +var env *environmentaws.Environment +var nodeClass *v1beta1.EC2NodeClass +var nodePool *corev1beta1.NodePool + +func TestScheduling(t *testing.T) { + RegisterFailHandler(Fail) + BeforeSuite(func() { + env = environmentaws.NewEnvironment(t) + }) + AfterSuite(func() { + env.Stop() + }) + RunSpecs(t, "Scheduling") +} + +var _ = BeforeEach(func() { + env.BeforeEach() + nodeClass = env.DefaultEC2NodeClass() + nodePool = env.DefaultNodePool(nodeClass) +}) +var _ = AfterEach(func() { env.Cleanup() }) +var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { var selectors sets.Set[string] @@ -271,7 +294,7 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { NodeSelector: nodeSelector, NodePreferences: requirements, NodeRequirements: requirements, - Image: aws.WindowsDefaultImage, + Image: environmentaws.WindowsDefaultImage, }}) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyWindows2022 // TODO: remove this requirement once VPC RC rolls out m7a.*, r7a.* ENI data (https://github.com/aws/karpenter-provider-aws/issues/4472) @@ -280,7 +303,7 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { NodeSelectorRequirement: v1.NodeSelectorRequirement{ Key: v1beta1.LabelInstanceFamily, Operator: v1.NodeSelectorOpNotIn, - Values: aws.ExcludedInstanceFamilies, + Values: environmentaws.ExcludedInstanceFamilies, }, }, corev1beta1.NodeSelectorRequirementWithMinValues{ @@ -597,7 +620,7 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { }) It("should provision a node for a pod with overlapping zone and zone-id requirements", func() { - subnetInfo := lo.UniqBy(env.GetSubnetInfo(map[string]string{"karpenter.sh/discovery": env.ClusterName}), func(s aws.SubnetInfo) string { + subnetInfo := lo.UniqBy(env.GetSubnetInfo(map[string]string{"karpenter.sh/discovery": env.ClusterName}), func(s environmentaws.SubnetInfo) string { return s.Zone }) Expect(len(subnetInfo)).To(BeNumerically(">=", 3)) @@ -609,12 +632,12 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { { Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, - Values: lo.Map(subnetInfo[0:2], func(info aws.SubnetInfo, _ int) string { return info.Zone }), + Values: lo.Map(subnetInfo[0:2], func(info environmentaws.SubnetInfo, _ int) string { return info.Zone }), }, { Key: v1beta1.LabelTopologyZoneID, Operator: v1.NodeSelectorOpIn, - Values: lo.Map(subnetInfo[1:3], func(info aws.SubnetInfo, _ int) string { return info.ZoneID }), + Values: lo.Map(subnetInfo[1:3], func(info environmentaws.SubnetInfo, _ int) string { return info.ZoneID }), }, }, }) @@ -637,10 +660,10 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { }, }) - subnetInfo := lo.UniqBy(env.GetSubnetInfo(map[string]string{"karpenter.sh/discovery": env.ClusterName}), func(s aws.SubnetInfo) string { + subnetInfo := lo.UniqBy(env.GetSubnetInfo(map[string]string{"karpenter.sh/discovery": env.ClusterName}), func(s environmentaws.SubnetInfo) string { return s.Zone }) - pods := lo.Map(subnetInfo, func(info aws.SubnetInfo, _ int) *v1.Pod { + pods := lo.Map(subnetInfo, func(info environmentaws.SubnetInfo, _ int) *v1.Pod { return test.Pod(test.PodOptions{ NodeRequirements: []v1.NodeSelectorRequirement{ { @@ -666,7 +689,7 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { expectedZone, ok := node.Labels[expectedZoneLabel] Expect(ok).To(BeTrue()) Expect(node.Labels[v1.LabelTopologyZone]).To(Equal(expectedZone)) - zoneInfo, ok := lo.Find(subnetInfo, func(info aws.SubnetInfo) bool { + zoneInfo, ok := lo.Find(subnetInfo, func(info environmentaws.SubnetInfo) bool { return info.Zone == expectedZone }) Expect(ok).To(BeTrue()) @@ -678,7 +701,7 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { func ephemeralInitContainer(requirements v1.ResourceRequirements) v1.Container { return v1.Container{ - Image: aws.EphemeralInitContainerImage, + Image: environmentaws.EphemeralInitContainerImage, Command: []string{"/bin/sh"}, Args: []string{"-c", "sleep 5"}, Resources: requirements, diff --git a/test/suites/integration/storage_test.go b/test/suites/storage/suite_test.go similarity index 91% rename from test/suites/integration/storage_test.go rename to test/suites/storage/suite_test.go index 8c6f4b5e781f..d67331bc92a3 100644 --- a/test/suites/integration/storage_test.go +++ b/test/suites/storage/suite_test.go @@ -12,11 +12,21 @@ See the License for the specific language governing permissions and limitations under the License. */ -package integration_test +package storage_test import ( "fmt" "strings" + "testing" + + awssdk "github.com/aws/aws-sdk-go/aws" + corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + + "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + environmentaws "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -24,18 +34,36 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/aws/aws-sdk-go/aws" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" "github.com/samber/lo" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/errors" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/karpenter/pkg/test" ) +var env *environmentaws.Environment +var nodeClass *v1beta1.EC2NodeClass +var nodePool *corev1beta1.NodePool + +func TestStorage(t *testing.T) { + RegisterFailHandler(Fail) + BeforeSuite(func() { + env = environmentaws.NewEnvironment(t) + }) + AfterSuite(func() { + env.Stop() + }) + RunSpecs(t, "Storage") +} + +var _ = BeforeEach(func() { + env.BeforeEach() + nodeClass = env.DefaultEC2NodeClass() + nodePool = env.DefaultNodePool(nodeClass) +}) +var _ = AfterEach(func() { env.Cleanup() }) +var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("Persistent Volumes", func() { Context("Static", func() { It("should run a pod with a pre-bound persistent volume (empty storage class)", func() { @@ -127,7 +155,7 @@ var _ = Describe("Persistent Volumes", func() { ObjectMeta: metav1.ObjectMeta{ Name: "test-storage-class", }, - Provisioner: aws.String("ebs.csi.aws.com"), + Provisioner: awssdk.String("ebs.csi.aws.com"), VolumeBindingMode: lo.ToPtr(storagev1.VolumeBindingWaitForFirstConsumer), }) })