From c7fcb8bff486ec45871288eb157f7db6989137f6 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Tue, 2 Jul 2024 13:31:32 -0700 Subject: [PATCH] chore: Move native sidecar container testing out of DaemonSet Context (#1383) --- pkg/controllers/provisioning/suite_test.go | 307 +++++++++++---------- 1 file changed, 154 insertions(+), 153 deletions(-) diff --git a/pkg/controllers/provisioning/suite_test.go b/pkg/controllers/provisioning/suite_test.go index cd1f8b3a9d..ac76179d6d 100644 --- a/pkg/controllers/provisioning/suite_test.go +++ b/pkg/controllers/provisioning/suite_test.go @@ -287,6 +287,160 @@ var _ = Describe("Provisioning", func() { Expect(n.Node.Name).ToNot(Equal(node.Name)) } }) + It("should schedule based on the max resource requests of containers and initContainers with sidecar containers when initcontainer comes first", func() { + if env.Version.Minor() < 29 { + Skip("Native Sidecar containers is only on by default starting in K8s version >= 1.29.x") + } + + ExpectApplied(ctx, env.Client, test.NodePool()) + + // Add three instance types, one that's what we want, one that's slightly smaller, one that's slightly bigger. + // If we miscalculate resources, we'll schedule to the smaller instance type rather than the larger one + cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 10)), + v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 4)), + }) + cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 11)), + v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 5)), + }) + cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 12)), + v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 6)), + }) + + pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("6"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("6"), v1.ResourceMemory: resource.MustParse("2Gi")}, + }, + InitContainers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10"), v1.ResourceMemory: resource.MustParse("4Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10"), v1.ResourceMemory: resource.MustParse("4Gi")}, + }, + }, + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.9"), v1.ResourceMemory: resource.MustParse("2.9Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.9"), v1.ResourceMemory: resource.MustParse("2.9Gi")}, + }, + }, + }, + }) + + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + ExpectResources(v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("11"), + v1.ResourceMemory: resource.MustParse("5Gi"), + }, node.Status.Capacity) + }) + It("should schedule based on the max resource requests of containers and initContainers with sidecar containers when sidecar container comes first and init container resources are smaller than container resources", func() { + if env.Version.Minor() < 29 { + Skip("Native Sidecar containers is only on by default starting in K8s version >= 1.29.x") + } + + ExpectApplied(ctx, env.Client, test.NodePool()) + + // Add three instance types, one that's what we want, one that's slightly smaller, one that's slightly bigger. + // If we miscalculate resources, we'll schedule to the smaller instance type rather than the larger one + cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 10)), + v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 4)), + }) + cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 11)), + v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 5)), + }) + cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 12)), + v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 6)), + }) + + pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("6"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("6"), v1.ResourceMemory: resource.MustParse("2Gi")}, + }, + InitContainers: []v1.Container{ + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.9"), v1.ResourceMemory: resource.MustParse("2.9Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.9"), v1.ResourceMemory: resource.MustParse("2.9Gi")}, + }, + }, + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + }, + }) + + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + ExpectResources(v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("11"), + v1.ResourceMemory: resource.MustParse("5Gi"), + }, node.Status.Capacity) + }) + It("should schedule based on the max resource requests of containers and initContainers with sidecar containers when sidecar container comes first and init container resources are bigger than container resources", func() { + if env.Version.Minor() < 29 { + Skip("Native Sidecar containers is only on by default starting in K8s version >= 1.29.x") + } + + ExpectApplied(ctx, env.Client, test.NodePool()) + + // Add three instance types, one that's what we want, one that's slightly smaller, one that's slightly bigger. + // If we miscalculate resources, we'll schedule to the smaller instance type rather than the larger one + cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 10)), + v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 4)), + }) + cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 11)), + v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 5)), + }) + cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 12)), + v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 6)), + }) + + pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("1Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("1Gi")}, + }, + InitContainers: []v1.Container{ + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.9"), v1.ResourceMemory: resource.MustParse("2.9Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.9"), v1.ResourceMemory: resource.MustParse("2.9Gi")}, + }, + }, + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("6"), v1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("6"), v1.ResourceMemory: resource.MustParse("2Gi")}, + }, + }, + }, + }) + + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + ExpectResources(v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("11"), + v1.ResourceMemory: resource.MustParse("5Gi"), + }, node.Status.Capacity) + }) + Context("Resource Limits", func() { It("should not schedule when limits are exceeded", func() { ExpectApplied(ctx, env.Client, test.NodePool(v1beta1.NodePool{ @@ -563,159 +717,6 @@ var _ = Describe("Provisioning", func() { Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4"))) Expect(*allocatable.Memory()).To(Equal(resource.MustParse("4Gi"))) }) - It("should schedule based on the max resource requests of containers and initContainers with sidecar containers when initcontainer comes first", func() { - if env.Version.Minor() < 29 { - Skip("Native Sidecar containers is only on by default starting in K8s version >= 1.29.x") - } - - ExpectApplied(ctx, env.Client, test.NodePool()) - - // Add three instance types, one that's what we want, one that's slightly smaller, one that's slightly bigger. - // If we miscalculate resources, we'll schedule to the smaller instance type rather than the larger one - cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ - v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 10)), - v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 4)), - }) - cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ - v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 11)), - v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 5)), - }) - cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ - v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 12)), - v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 6)), - }) - - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("6"), v1.ResourceMemory: resource.MustParse("2Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("6"), v1.ResourceMemory: resource.MustParse("2Gi")}, - }, - InitContainers: []v1.Container{ - { - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10"), v1.ResourceMemory: resource.MustParse("4Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10"), v1.ResourceMemory: resource.MustParse("4Gi")}, - }, - }, - { - RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.9"), v1.ResourceMemory: resource.MustParse("2.9Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.9"), v1.ResourceMemory: resource.MustParse("2.9Gi")}, - }, - }, - }, - }) - - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - ExpectResources(v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("11"), - v1.ResourceMemory: resource.MustParse("5Gi"), - }, node.Status.Capacity) - }) - It("should schedule based on the max resource requests of containers and initContainers with sidecar containers when sidecar container comes first and init container resources are smaller than container resources", func() { - if env.Version.Minor() < 29 { - Skip("Native Sidecar containers is only on by default starting in K8s version >= 1.29.x") - } - - ExpectApplied(ctx, env.Client, test.NodePool()) - - // Add three instance types, one that's what we want, one that's slightly smaller, one that's slightly bigger. - // If we miscalculate resources, we'll schedule to the smaller instance type rather than the larger one - cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ - v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 10)), - v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 4)), - }) - cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ - v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 11)), - v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 5)), - }) - cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ - v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 12)), - v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 6)), - }) - - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("6"), v1.ResourceMemory: resource.MustParse("2Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("6"), v1.ResourceMemory: resource.MustParse("2Gi")}, - }, - InitContainers: []v1.Container{ - { - RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.9"), v1.ResourceMemory: resource.MustParse("2.9Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.9"), v1.ResourceMemory: resource.MustParse("2.9Gi")}, - }, - }, - { - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("1Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("1Gi")}, - }, - }, - }, - }) - - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - ExpectResources(v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("11"), - v1.ResourceMemory: resource.MustParse("5Gi"), - }, node.Status.Capacity) - }) - It("should schedule based on the max resource requests of containers and initContainers with sidecar containers when sidecar container comes first and init container resources are bigger than container resources", func() { - if env.Version.Minor() < 29 { - Skip("Native Sidecar containers is only on by default starting in K8s version >= 1.29.x") - } - - ExpectApplied(ctx, env.Client, test.NodePool()) - - // Add three instance types, one that's what we want, one that's slightly smaller, one that's slightly bigger. - // If we miscalculate resources, we'll schedule to the smaller instance type rather than the larger one - cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ - v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 10)), - v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 4)), - }) - cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ - v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 11)), - v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 5)), - }) - cloudProvider.InstanceTypes = AddInstanceResources(cloudProvider.InstanceTypes, v1.ResourceList{ - v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", 12)), - v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", 6)), - }) - - pod := test.UnschedulablePod(test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("1Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5"), v1.ResourceMemory: resource.MustParse("1Gi")}, - }, - InitContainers: []v1.Container{ - { - RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.9"), v1.ResourceMemory: resource.MustParse("2.9Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.9"), v1.ResourceMemory: resource.MustParse("2.9Gi")}, - }, - }, - { - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("6"), v1.ResourceMemory: resource.MustParse("2Gi")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("6"), v1.ResourceMemory: resource.MustParse("2Gi")}, - }, - }, - }, - }) - - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) - ExpectResources(v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("11"), - v1.ResourceMemory: resource.MustParse("5Gi"), - }, node.Status.Capacity) - }) It("should not schedule if combined max resources are too large for any node", func() { ExpectApplied(ctx, env.Client, test.NodePool(), test.DaemonSet( test.DaemonSetOptions{PodOptions: test.PodOptions{