From 46bc2acb8543d367f9a97672fe197e80428a1028 Mon Sep 17 00:00:00 2001 From: Jason Deal Date: Fri, 12 Jul 2024 00:17:42 -0700 Subject: [PATCH] remaining changes --- .../karpenter.k8s.aws_ec2nodeclasses.yaml | 4 +- pkg/apis/crds/karpenter.sh_nodeclaims.yaml | 4 +- pkg/apis/crds/karpenter.sh_nodepools.yaml | 4 +- pkg/apis/v1/ec2nodeclass.go | 1 + pkg/apis/v1/labels.go | 26 +++---- pkg/apis/v1beta1/ec2nodeclass.go | 1 - pkg/apis/v1beta1/labels.go | 26 +++---- pkg/cloudprovider/suite_test.go | 6 +- pkg/controllers/controllers.go | 4 +- pkg/controllers/interruption/suite_test.go | 6 +- .../nodeclaim/tagging/controller.go | 2 +- pkg/controllers/nodeclass/status/ami_test.go | 4 +- pkg/operator/operator.go | 2 +- pkg/providers/amifamily/resolver.go | 13 ++-- pkg/providers/instancetype/suite_test.go | 75 +++++++++++-------- pkg/providers/instancetype/types.go | 12 +-- pkg/providers/launchtemplate/suite_test.go | 14 ++-- pkg/test/environment.go | 2 +- pkg/test/nodeclass.go | 2 +- pkg/utils/utils.go | 5 +- test/suites/chaos/suite_test.go | 2 +- test/suites/drift/suite_test.go | 4 +- test/suites/integration/cni_test.go | 3 +- 23 files changed, 119 insertions(+), 103 deletions(-) diff --git a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml index 95959d19e704..cc061eb57cf9 100644 --- a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -717,7 +717,7 @@ spec: type: object type: object served: true - storage: false + storage: true subresources: status: {} - name: v1beta1 @@ -1293,7 +1293,7 @@ spec: type: object type: object served: true - storage: true + storage: false subresources: status: {} conversion: diff --git a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml index 6834866ff541..9be03992d05a 100644 --- a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +++ b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml @@ -348,7 +348,7 @@ spec: - spec type: object served: true - storage: false + storage: true subresources: status: {} - additionalPrinterColumns: @@ -790,7 +790,7 @@ spec: - spec type: object served: true - storage: true + storage: false subresources: status: {} conversion: diff --git a/pkg/apis/crds/karpenter.sh_nodepools.yaml b/pkg/apis/crds/karpenter.sh_nodepools.yaml index 9ae64056f616..f14bdd8883a3 100644 --- a/pkg/apis/crds/karpenter.sh_nodepools.yaml +++ b/pkg/apis/crds/karpenter.sh_nodepools.yaml @@ -488,7 +488,7 @@ spec: - spec type: object served: true - storage: false + storage: true subresources: status: {} - additionalPrinterColumns: @@ -1068,7 +1068,7 @@ spec: - spec type: object served: true - storage: true + storage: false subresources: status: {} conversion: diff --git a/pkg/apis/v1/ec2nodeclass.go b/pkg/apis/v1/ec2nodeclass.go index db32453e6057..c0950eaca00d 100644 --- a/pkg/apis/v1/ec2nodeclass.go +++ b/pkg/apis/v1/ec2nodeclass.go @@ -399,6 +399,7 @@ const ( // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="" // +kubebuilder:printcolumn:name="Role",type="string",JSONPath=".spec.role",priority=1,description="" // +kubebuilder:resource:path=ec2nodeclasses,scope=Cluster,categories=karpenter,shortName={ec2nc,ec2ncs} +// +kubebuilder:storageversion // +kubebuilder:subresource:status type EC2NodeClass struct { metav1.TypeMeta `json:",inline"` diff --git a/pkg/apis/v1/labels.go b/pkg/apis/v1/labels.go index fced9dae1138..58a48828dfa4 100644 --- a/pkg/apis/v1/labels.go +++ b/pkg/apis/v1/labels.go @@ -56,7 +56,7 @@ func init() { var ( TerminationFinalizer = apis.Group + "/termination" AWSToKubeArchitectures = map[string]string{ - "x86_64": karpv1.ArchitectureAmd64, + "x86_64": karpv1.ArchitectureAmd64, karpv1.ArchitectureArm64: karpv1.ArchitectureArm64, } WellKnownArchitectures = sets.NewString( @@ -75,18 +75,18 @@ var ( regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(LabelNodeClass))), regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(TagNodeClaim))), } - AMIFamilyBottlerocket = "Bottlerocket" - AMIFamilyAL2 = "AL2" - AMIFamilyAL2023 = "AL2023" - AMIFamilyUbuntu = "Ubuntu" - AMIFamilyWindows2019 = "Windows2019" - AMIFamilyWindows2022 = "Windows2022" - AMIFamilyCustom = "Custom" - Windows2019 = "2019" - Windows2022 = "2022" - WindowsCore = "Core" - Windows2019Build = "10.0.17763" - Windows2022Build = "10.0.20348" + AMIFamilyBottlerocket = "Bottlerocket" + AMIFamilyAL2 = "AL2" + AMIFamilyAL2023 = "AL2023" + AMIFamilyUbuntu = "Ubuntu" + AMIFamilyWindows2019 = "Windows2019" + AMIFamilyWindows2022 = "Windows2022" + AMIFamilyCustom = "Custom" + Windows2019 = "2019" + Windows2022 = "2022" + WindowsCore = "Core" + Windows2019Build = "10.0.17763" + Windows2022Build = "10.0.20348" ResourceNVIDIAGPU corev1.ResourceName = "nvidia.com/gpu" ResourceAMDGPU corev1.ResourceName = "amd.com/gpu" ResourceAWSNeuron corev1.ResourceName = "aws.amazon.com/neuron" diff --git a/pkg/apis/v1beta1/ec2nodeclass.go b/pkg/apis/v1beta1/ec2nodeclass.go index 4dba5f4f0adf..833dcdd5683e 100644 --- a/pkg/apis/v1beta1/ec2nodeclass.go +++ b/pkg/apis/v1beta1/ec2nodeclass.go @@ -318,7 +318,6 @@ const ( // EC2NodeClass is the Schema for the EC2NodeClass API // +kubebuilder:object:root=true -// +kubebuilder:storageversion // +kubebuilder:resource:path=ec2nodeclasses,scope=Cluster,categories=karpenter,shortName={ec2nc,ec2ncs} // +kubebuilder:subresource:status type EC2NodeClass struct { diff --git a/pkg/apis/v1beta1/labels.go b/pkg/apis/v1beta1/labels.go index afa6b2829aaf..d7ae21c84230 100644 --- a/pkg/apis/v1beta1/labels.go +++ b/pkg/apis/v1beta1/labels.go @@ -56,7 +56,7 @@ func init() { var ( TerminationFinalizer = apis.Group + "/termination" AWSToKubeArchitectures = map[string]string{ - "x86_64": karpv1beta1.ArchitectureAmd64, + "x86_64": karpv1beta1.ArchitectureAmd64, karpv1beta1.ArchitectureArm64: karpv1beta1.ArchitectureArm64, } WellKnownArchitectures = sets.NewString( @@ -75,18 +75,18 @@ var ( regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(LabelNodeClass))), regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(TagNodeClaim))), } - AMIFamilyBottlerocket = "Bottlerocket" - AMIFamilyAL2 = "AL2" - AMIFamilyAL2023 = "AL2023" - AMIFamilyUbuntu = "Ubuntu" - AMIFamilyWindows2019 = "Windows2019" - AMIFamilyWindows2022 = "Windows2022" - AMIFamilyCustom = "Custom" - Windows2019 = "2019" - Windows2022 = "2022" - WindowsCore = "Core" - Windows2019Build = "10.0.17763" - Windows2022Build = "10.0.20348" + AMIFamilyBottlerocket = "Bottlerocket" + AMIFamilyAL2 = "AL2" + AMIFamilyAL2023 = "AL2023" + AMIFamilyUbuntu = "Ubuntu" + AMIFamilyWindows2019 = "Windows2019" + AMIFamilyWindows2022 = "Windows2022" + AMIFamilyCustom = "Custom" + Windows2019 = "2019" + Windows2022 = "2022" + WindowsCore = "Core" + Windows2019Build = "10.0.17763" + Windows2022Build = "10.0.20348" ResourceNVIDIAGPU corev1.ResourceName = "nvidia.com/gpu" ResourceAMDGPU corev1.ResourceName = "amd.com/gpu" ResourceAWSNeuron corev1.ResourceName = "aws.amazon.com/neuron" diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index fea3efee3181..f28739327e15 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -1129,7 +1129,11 @@ var _ = Describe("CloudProvider", func() { Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}}}, }}) controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider) - nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{MaxPods: aws.Int32(1)} + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ + MaxPods: aws.Int32(1), + } + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) ExpectObjectReconciled(ctx, env.Client, controller, nodeClass) pod1 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{corev1.LabelTopologyZone: "test-zone-1a"}}) pod2 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{corev1.LabelTopologyZone: "test-zone-1a"}}) diff --git a/pkg/controllers/controllers.go b/pkg/controllers/controllers.go index dbeaf727a282..c45f00938299 100644 --- a/pkg/controllers/controllers.go +++ b/pkg/controllers/controllers.go @@ -22,7 +22,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/karpenter/pkg/cloudprovider" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" nodeclasshash "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/hash" nodeclassstatus "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/status" nodeclasstermination "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/termination" @@ -66,7 +66,7 @@ func NewControllers(ctx context.Context, mgr manager.Manager, sess *session.Sess nodeclaimtagging.NewController(kubeClient, instanceProvider), controllerspricing.NewController(pricingProvider), controllersinstancetype.NewController(instanceTypeProvider), - status.NewController[*v1beta1.EC2NodeClass](kubeClient, mgr.GetEventRecorderFor("karpenter")), + status.NewController[*v1.EC2NodeClass](kubeClient, mgr.GetEventRecorderFor("karpenter")), } if options.FromContext(ctx).InterruptionQueue != "" { sqsapi := servicesqs.New(sess) diff --git a/pkg/controllers/interruption/suite_test.go b/pkg/controllers/interruption/suite_test.go index e68203580081..4f3ff299cb28 100644 --- a/pkg/controllers/interruption/suite_test.go +++ b/pkg/controllers/interruption/suite_test.go @@ -214,9 +214,9 @@ var _ = Describe("InterruptionHandling", func() { }) It("should mark the ICE cache for the offering when getting a spot interruption warning", func() { nodeClaim.Labels = lo.Assign(nodeClaim.Labels, map[string]string{ - corev1.LabelTopologyZone: "coretest-zone-1a", - corev1.LabelInstanceTypeStable: "t3.large", - karpv1.CapacityTypeLabelKey: karpv1.CapacityTypeSpot, + corev1.LabelTopologyZone: "coretest-zone-1a", + corev1.LabelInstanceTypeStable: "t3.large", + karpv1.CapacityTypeLabelKey: karpv1.CapacityTypeSpot, }) ExpectMessagesCreated(spotInterruptionMessage(lo.Must(utils.ParseInstanceID(nodeClaim.Status.ProviderID)))) ExpectApplied(ctx, env.Client, nodeClaim, node) diff --git a/pkg/controllers/nodeclaim/tagging/controller.go b/pkg/controllers/nodeclaim/tagging/controller.go index 2e42f99bbf44..d11d97442c5b 100644 --- a/pkg/controllers/nodeclaim/tagging/controller.go +++ b/pkg/controllers/nodeclaim/tagging/controller.go @@ -33,7 +33,7 @@ import ( "github.com/awslabs/operatorpkg/reasonable" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/providers/instance" "github.com/aws/karpenter-provider-aws/pkg/utils" diff --git a/pkg/controllers/nodeclass/status/ami_test.go b/pkg/controllers/nodeclass/status/ami_test.go index 8e7ed7a40e81..0ab001dd7d4c 100644 --- a/pkg/controllers/nodeclass/status/ami_test.go +++ b/pkg/controllers/nodeclass/status/ami_test.go @@ -92,8 +92,8 @@ var _ = Describe("NodeClass AMI Status Controller", func() { version := lo.Must(awsEnv.VersionProvider.Get(ctx)) awsEnv.SSMAPI.Parameters = map[string]string{ - fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", version): "ami-id-123", - fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-gpu/recommended/image_id", version): "ami-id-456", + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", version): "ami-id-123", + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-gpu/recommended/image_id", version): "ami-id-456", fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, fmt.Sprintf("-%s", karpv1.ArchitectureArm64)): "ami-id-789", } diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 8d8115cc6636..1158569fe44c 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -48,8 +48,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" - karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1" karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/operator" awscache "github.com/aws/karpenter-provider-aws/pkg/cache" diff --git a/pkg/providers/amifamily/resolver.go b/pkg/providers/amifamily/resolver.go index 294d6013d389..78bb5373f163 100644 --- a/pkg/providers/amifamily/resolver.go +++ b/pkg/providers/amifamily/resolver.go @@ -20,7 +20,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/imdario/mergo" "github.com/samber/lo" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -208,14 +207,14 @@ func (r Resolver) defaultClusterDNS(opts *Options, kubeletConfig *v1.KubeletConf func (r Resolver) resolveLaunchTemplate(nodeClass *v1.EC2NodeClass, nodeClaim *karpv1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, capacityType string, amiFamily AMIFamily, amiID string, maxPods int, efaCount int, options *Options) (*LaunchTemplate, error) { - kubeletConfig := &v1.KubeletConfiguration{} - if kc, err := utils.GetKubeletConfigurationWithNodeClaim(nodeClaim, nodeClass); err != nil { + kubeletConfig, err := utils.GetKubeletConfigurationWithNodeClaim(nodeClaim, nodeClass) + if err != nil { return nil, fmt.Errorf("resolving kubelet configuration, %w", err) - } else if kc != nil { - if err := mergo.Merge(kubeletConfig, kc); err != nil { - return nil, err - } } + if kubeletConfig == nil { + kubeletConfig = &v1.KubeletConfiguration{} + } + fmt.Printf("resolved kubelet config with maxPods %d\n", lo.FromPtr(kubeletConfig.MaxPods)) if kubeletConfig.MaxPods == nil { kubeletConfig.MaxPods = lo.ToPtr(int32(maxPods)) } diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index 64a28a9f2944..569647e2b99a 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -37,6 +37,7 @@ import ( "github.com/samber/lo" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" clock "k8s.io/utils/clock/testing" @@ -217,13 +218,13 @@ var _ = Describe("InstanceTypeProvider", func() { nodeSelector := map[string]string{ // Well known - karpv1.NodePoolLabelKey: nodePool.Name, - corev1.LabelTopologyRegion: fake.DefaultRegion, - corev1.LabelTopologyZone: "test-zone-1a", - corev1.LabelInstanceTypeStable: "g4dn.8xlarge", - corev1.LabelOSStable: "linux", - corev1.LabelArchStable: "amd64", - karpv1.CapacityTypeLabelKey: "on-demand", + karpv1.NodePoolLabelKey: nodePool.Name, + corev1.LabelTopologyRegion: fake.DefaultRegion, + corev1.LabelTopologyZone: "test-zone-1a", + corev1.LabelInstanceTypeStable: "g4dn.8xlarge", + corev1.LabelOSStable: "linux", + corev1.LabelArchStable: "amd64", + karpv1.CapacityTypeLabelKey: "on-demand", // Well Known to AWS v1.LabelInstanceHypervisor: "nitro", v1.LabelInstanceEncryptionInTransitSupported: "true", @@ -248,10 +249,10 @@ var _ = Describe("InstanceTypeProvider", func() { // Deprecated Labels corev1.LabelFailureDomainBetaRegion: fake.DefaultRegion, corev1.LabelFailureDomainBetaZone: "test-zone-1a", - "beta.kubernetes.io/arch": "amd64", - "beta.kubernetes.io/os": "linux", + "beta.kubernetes.io/arch": "amd64", + "beta.kubernetes.io/os": "linux", corev1.LabelInstanceType: "g4dn.8xlarge", - "topology.ebs.csi.aws.com/zone": "test-zone-1a", + "topology.ebs.csi.aws.com/zone": "test-zone-1a", corev1.LabelWindowsBuild: v1.Windows2022Build, } @@ -272,13 +273,13 @@ var _ = Describe("InstanceTypeProvider", func() { nodeSelector := map[string]string{ // Well known - karpv1.NodePoolLabelKey: nodePool.Name, - corev1.LabelTopologyRegion: fake.DefaultRegion, - corev1.LabelTopologyZone: "test-zone-1a", - corev1.LabelInstanceTypeStable: "g4dn.8xlarge", - corev1.LabelOSStable: "linux", - corev1.LabelArchStable: "amd64", - karpv1.CapacityTypeLabelKey: "on-demand", + karpv1.NodePoolLabelKey: nodePool.Name, + corev1.LabelTopologyRegion: fake.DefaultRegion, + corev1.LabelTopologyZone: "test-zone-1a", + corev1.LabelInstanceTypeStable: "g4dn.8xlarge", + corev1.LabelOSStable: "linux", + corev1.LabelArchStable: "amd64", + karpv1.CapacityTypeLabelKey: "on-demand", // Well Known to AWS v1.LabelInstanceHypervisor: "nitro", v1.LabelInstanceEncryptionInTransitSupported: "true", @@ -300,10 +301,10 @@ var _ = Describe("InstanceTypeProvider", func() { // Deprecated Labels corev1.LabelFailureDomainBetaRegion: fake.DefaultRegion, corev1.LabelFailureDomainBetaZone: "test-zone-1a", - "beta.kubernetes.io/arch": "amd64", - "beta.kubernetes.io/os": "linux", + "beta.kubernetes.io/arch": "amd64", + "beta.kubernetes.io/os": "linux", corev1.LabelInstanceType: "g4dn.8xlarge", - "topology.ebs.csi.aws.com/zone": "test-zone-1a", + "topology.ebs.csi.aws.com/zone": "test-zone-1a", } // Ensure that we're exercising all well known labels except for accelerator labels @@ -325,13 +326,13 @@ var _ = Describe("InstanceTypeProvider", func() { nodeSelector := map[string]string{ // Well known - karpv1.NodePoolLabelKey: nodePool.Name, - corev1.LabelTopologyRegion: fake.DefaultRegion, - corev1.LabelTopologyZone: "test-zone-1a", - corev1.LabelInstanceTypeStable: "inf1.2xlarge", - corev1.LabelOSStable: "linux", - corev1.LabelArchStable: "amd64", - karpv1.CapacityTypeLabelKey: "on-demand", + karpv1.NodePoolLabelKey: nodePool.Name, + corev1.LabelTopologyRegion: fake.DefaultRegion, + corev1.LabelTopologyZone: "test-zone-1a", + corev1.LabelInstanceTypeStable: "inf1.2xlarge", + corev1.LabelOSStable: "linux", + corev1.LabelArchStable: "amd64", + karpv1.CapacityTypeLabelKey: "on-demand", // Well Known to AWS v1.LabelInstanceHypervisor: "nitro", v1.LabelInstanceEncryptionInTransitSupported: "true", @@ -351,10 +352,10 @@ var _ = Describe("InstanceTypeProvider", func() { // Deprecated Labels corev1.LabelFailureDomainBetaRegion: fake.DefaultRegion, corev1.LabelFailureDomainBetaZone: "test-zone-1a", - "beta.kubernetes.io/arch": "amd64", - "beta.kubernetes.io/os": "linux", + "beta.kubernetes.io/arch": "amd64", + "beta.kubernetes.io/os": "linux", corev1.LabelInstanceType: "inf1.2xlarge", - "topology.ebs.csi.aws.com/zone": "test-zone-1a", + "topology.ebs.csi.aws.com/zone": "test-zone-1a", } // Ensure that we're exercising all well known labels except for gpu labels and nvme @@ -907,6 +908,7 @@ var _ = Describe("InstanceTypeProvider", func() { It("should not set pods to 110 if using ENI-based pod density", func() { instanceInfo, err := awsEnv.EC2API.DescribeInstanceTypesWithContext(ctx, &ec2.DescribeInstanceTypesInput{}) Expect(err).To(BeNil()) + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{} for _, info := range instanceInfo.InstanceTypes { amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, @@ -929,7 +931,7 @@ var _ = Describe("InstanceTypeProvider", func() { It("should set pods to 110 if AMI Family doesn't support", func() { instanceInfo, err := awsEnv.EC2API.DescribeInstanceTypesWithContext(ctx, &ec2.DescribeInstanceTypesInput{}) Expect(err).To(BeNil()) - + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{} for _, info := range instanceInfo.InstanceTypes { amiFamily := amifamily.GetAMIFamily(windowsNodeClass.Spec.AMIFamily, &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, @@ -1051,6 +1053,7 @@ var _ = Describe("InstanceTypeProvider", func() { Context("System Reserved Resources", func() { It("should use defaults when no kubelet is specified", func() { amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{} it := instancetype.NewInstanceType(ctx, info, fake.DefaultRegion, @@ -1522,6 +1525,7 @@ var _ = Describe("InstanceTypeProvider", func() { It("should default max pods based off of network interfaces", func() { instanceInfo, err := awsEnv.EC2API.DescribeInstanceTypesWithContext(ctx, &ec2.DescribeInstanceTypesInput{}) Expect(err).To(BeNil()) + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{} for _, info := range instanceInfo.InstanceTypes { if *info.InstanceType == "t3.large" { amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) @@ -1623,6 +1627,7 @@ var _ = Describe("InstanceTypeProvider", func() { }) Expect(ok).To(Equal(true)) amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{} it := instancetype.NewInstanceType(ctx, t3Large, fake.DefaultRegion, @@ -1657,6 +1662,7 @@ var _ = Describe("InstanceTypeProvider", func() { }) Expect(ok).To(Equal(true)) amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{} it := instancetype.NewInstanceType(ctx, t3Large, fake.DefaultRegion, @@ -2088,7 +2094,7 @@ var _ = Describe("InstanceTypeProvider", func() { }, NodeSelector: map[string]string{ karpv1.CapacityTypeLabelKey: ct, - corev1.LabelTopologyZone: zone, + corev1.LabelTopologyZone: zone, }, })) } @@ -2319,7 +2325,10 @@ var _ = Describe("InstanceTypeProvider", func() { SystemReserved: map[string]string{string(corev1.ResourceCPU): "1"}, EvictionHard: map[string]string{"memory.available": "5%"}, EvictionSoft: map[string]string{"nodefs.available": "10%"}, - MaxPods: aws.Int32(10), + EvictionSoftGracePeriod: map[string]metav1.Duration{ + "nodefs.available": {Duration: time.Minute}, + }, + MaxPods: aws.Int32(10), } kubeletChanges := []*v1.KubeletConfiguration{ {}, // Testing the base case black EC2NodeClass diff --git a/pkg/providers/instancetype/types.go b/pkg/providers/instancetype/types.go index 3f2e665e52ea..bce08d2be7ed 100644 --- a/pkg/providers/instancetype/types.go +++ b/pkg/providers/instancetype/types.go @@ -199,12 +199,12 @@ func computeCapacity(ctx context.Context, info *ec2.InstanceTypeInfo, amiFamily corev1.ResourceMemory: *memory(ctx, info), corev1.ResourceEphemeralStorage: *ephemeralStorage(info, amiFamily, blockDeviceMapping, instanceStorePolicy), corev1.ResourcePods: *pods(ctx, info, amiFamily, maxPods, podsPerCore), - v1.ResourceAWSPodENI: *awsPodENI(aws.StringValue(info.InstanceType)), - v1.ResourceNVIDIAGPU: *nvidiaGPUs(info), - v1.ResourceAMDGPU: *amdGPUs(info), - v1.ResourceAWSNeuron: *awsNeurons(info), - v1.ResourceHabanaGaudi: *habanaGaudis(info), - v1.ResourceEFA: *efas(info), + v1.ResourceAWSPodENI: *awsPodENI(aws.StringValue(info.InstanceType)), + v1.ResourceNVIDIAGPU: *nvidiaGPUs(info), + v1.ResourceAMDGPU: *amdGPUs(info), + v1.ResourceAWSNeuron: *awsNeurons(info), + v1.ResourceHabanaGaudi: *habanaGaudis(info), + v1.ResourceEFA: *efas(info), } return resourceList } diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index 00ffea6a6dbf..a0f7b24dccfb 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -127,6 +127,9 @@ var _ = Describe("LaunchTemplate Provider", func() { BeforeEach(func() { nodeClass = test.EC2NodeClass( v1.EC2NodeClass{ + // Spec: v1.EC2NodeClassSpec{ + // Kubelet: &v1.KubeletConfiguration{}, + // }, Status: v1.EC2NodeClassStatus{ InstanceProfile: "test-profile", SecurityGroups: []v1.SecurityGroup{ @@ -1009,6 +1012,7 @@ var _ = Describe("LaunchTemplate Provider", func() { })) nodeClass.Spec.AMIFamily = &v1.AMIFamilyAL2 + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{} amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, @@ -1062,6 +1066,7 @@ var _ = Describe("LaunchTemplate Provider", func() { })) nodeClass.Spec.AMIFamily = &v1.AMIFamilyBottlerocket + nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{} amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, @@ -1422,7 +1427,6 @@ var _ = Describe("LaunchTemplate Provider", func() { ExpectNotScheduled(ctx, env.Client, pod) }) It("should override system reserved values in user data", func() { - ExpectApplied(ctx, env.Client, nodeClass) nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ SystemReserved: map[string]string{ string(corev1.ResourceCPU): "2", @@ -1430,7 +1434,7 @@ var _ = Describe("LaunchTemplate Provider", func() { string(corev1.ResourceEphemeralStorage): "10Gi", }, } - ExpectApplied(ctx, env.Client, nodePool) + ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1447,7 +1451,6 @@ var _ = Describe("LaunchTemplate Provider", func() { }) }) It("should override kube reserved values in user data", func() { - ExpectApplied(ctx, env.Client, nodeClass) nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ KubeReserved: map[string]string{ string(corev1.ResourceCPU): "2", @@ -1455,7 +1458,7 @@ var _ = Describe("LaunchTemplate Provider", func() { string(corev1.ResourceEphemeralStorage): "10Gi", }, } - ExpectApplied(ctx, env.Client, nodePool) + ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -1472,7 +1475,6 @@ var _ = Describe("LaunchTemplate Provider", func() { }) }) It("should override kube reserved values in user data", func() { - ExpectApplied(ctx, env.Client, nodeClass) nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{ EvictionHard: map[string]string{ "memory.available": "10%", @@ -1480,7 +1482,7 @@ var _ = Describe("LaunchTemplate Provider", func() { "nodefs.inodesFree": "5%", }, } - ExpectApplied(ctx, env.Client, nodePool) + ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) diff --git a/pkg/test/environment.go b/pkg/test/environment.go index dd38a3b5eee3..015d5ae9a5eb 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -23,7 +23,7 @@ import ( corev1 "k8s.io/api/core/v1" karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1" + karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" awscache "github.com/aws/karpenter-provider-aws/pkg/cache" "github.com/aws/karpenter-provider-aws/pkg/fake" diff --git a/pkg/test/nodeclass.go b/pkg/test/nodeclass.go index 203e5ef8fc44..6b8858832a29 100644 --- a/pkg/test/nodeclass.go +++ b/pkg/test/nodeclass.go @@ -27,7 +27,7 @@ import ( karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" ) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index f2c04668ef57..4b15fc5615b6 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -22,10 +22,11 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/samber/lo" - karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" ) var ( diff --git a/test/suites/chaos/suite_test.go b/test/suites/chaos/suite_test.go index 2a5103464624..aa35422b2c76 100644 --- a/test/suites/chaos/suite_test.go +++ b/test/suites/chaos/suite_test.go @@ -37,7 +37,7 @@ import ( coretest "sigs.k8s.io/karpenter/pkg/test" nodeutils "sigs.k8s.io/karpenter/pkg/utils/node" - "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/test/pkg/debug" "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" diff --git a/test/suites/drift/suite_test.go b/test/suites/drift/suite_test.go index b2e41a663171..961004cf1ae0 100644 --- a/test/suites/drift/suite_test.go +++ b/test/suites/drift/suite_test.go @@ -544,8 +544,8 @@ var _ = Describe("Drift", func() { Spec: karpv1.NodeClaimSpec{ NodeClassRef: &karpv1.NodeClassReference{ Group: object.GVK(nodeClass).Group, - Kind: object.GVK(nodeClass).Kind, - Name: nodeClass.Name, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }, diff --git a/test/suites/integration/cni_test.go b/test/suites/integration/cni_test.go index adec377a9b6d..41d65f04dae3 100644 --- a/test/suites/integration/cni_test.go +++ b/test/suites/integration/cni_test.go @@ -19,13 +19,14 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/samber/lo" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + "sigs.k8s.io/karpenter/pkg/test" )