From 2a657d8e07846806710fdf8ca6a9402acd362937 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Tue, 14 May 2024 16:49:51 -0500 Subject: [PATCH] Add ordering fix for inflightIPs --- .github/workflows/release.yaml | 2 +- .../karpenter.k8s.aws_awsnodetemplates.yaml | 268 ++++++--------- .../karpenter.k8s.aws_ec2nodeclasses.yaml | 322 ++++++++---------- pkg/providers/subnet/subnet.go | 5 +- 4 files changed, 249 insertions(+), 348 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2fc1cee5e199..c01227a094b5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -20,7 +20,7 @@ jobs: with: fetch-depth: 0 - name: Create GitHub Release - uses: 'marvinpinto/action-automatic-releases@latest' + uses: marvinpinto/action-automatic-releases@919008cf3f741b179569b7a6fb4d8860689ab7f0 # v1.2.1 with: repo_token: '${{ secrets.GITHUB_TOKEN }}' prerelease: false diff --git a/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml b/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml index f68a599b7649..20cf10f8b3d2 100644 --- a/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml +++ b/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.13.0 name: awsnodetemplates.karpenter.k8s.aws spec: group: karpenter.k8s.aws @@ -22,26 +22,21 @@ spec: description: AWSNodeTemplate is the Schema for the AWSNodeTemplate API properties: apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: - description: |- - AWSNodeTemplateSpec is the top level specification for the AWS Karpenter Provider. - This will contain configuration necessary to launch instances in AWS. + description: AWSNodeTemplateSpec is the top level specification for the + AWS Karpenter Provider. This will contain configuration necessary to + launch instances in AWS. properties: amiFamily: description: AMIFamily is the AMI family that instances use. @@ -52,11 +47,9 @@ spec: description: AMISelector discovers AMIs to be used by Amazon EC2 tags. type: object apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string blockDeviceMappings: description: BlockDeviceMappings to be applied to provisioned nodes. @@ -74,38 +67,27 @@ spec: volume is deleted on instance termination. type: boolean encrypted: - description: |- - Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only - be attached to instances that support Amazon EBS encryption. If you are creating - a volume from a snapshot, you can't specify an encryption value. + description: Encrypted indicates whether the EBS volume + is encrypted. Encrypted volumes can only be attached to + instances that support Amazon EBS encryption. If you are + creating a volume from a snapshot, you can't specify an + encryption value. type: boolean iops: - description: |- - IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, - this represents the number of IOPS that are provisioned for the volume. For - gp2 volumes, this represents the baseline performance of the volume and the - rate at which the volume accumulates I/O credits for bursting. - - - The following are the supported values for each volume type: - - - * gp3: 3,000-16,000 IOPS - - - * io1: 100-64,000 IOPS - - - * io2: 100-64,000 IOPS - - - For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built - on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). - Other instance families guarantee performance up to 32,000 IOPS. - - - This parameter is supported for io1, io2, and gp3 volumes only. This parameter - is not supported for gp2, st1, sc1, or standard volumes. + description: "IOPS is the number of I/O operations per second + (IOPS). For gp3, io1, and io2 volumes, this represents + the number of IOPS that are provisioned for the volume. + For gp2 volumes, this represents the baseline performance + of the volume and the rate at which the volume accumulates + I/O credits for bursting. \n The following are the supported + values for each volume type: \n * gp3: 3,000-16,000 IOPS + \n * io1: 100-64,000 IOPS \n * io2: 100-64,000 IOPS \n + For io1 and io2 volumes, we guarantee 64,000 IOPS only + for Instances built on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 + IOPS. \n This parameter is supported for io1, io2, and + gp3 volumes only. This parameter is not supported for + gp2, st1, sc1, or standard volumes." format: int64 type: integer kmsKeyID: @@ -116,46 +98,32 @@ spec: description: SnapshotID is the ID of an EBS snapshot type: string throughput: - description: |- - Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. - Valid Range: Minimum value of 125. Maximum value of 1000. + description: 'Throughput to provision for a gp3 volume, + with a maximum of 1,000 MiB/s. Valid Range: Minimum value + of 125. Maximum value of 1000.' format: int64 type: integer volumeSize: anyOf: - type: integer - type: string - description: |- - VolumeSize in GiBs. You must specify either a snapshot ID or - a volume size. The following are the supported volumes sizes for each volume - type: - - - * gp2 and gp3: 1-16,384 - - - * io1 and io2: 4-16,384 - - - * st1 and sc1: 125-16,384 - - - * standard: 1-1,024 + description: "VolumeSize in GiBs. You must specify either + a snapshot ID or a volume size. The following are the + supported volumes sizes for each volume type: \n * gp2 + and gp3: 1-16,384 \n * io1 and io2: 4-16,384 \n * st1 + and sc1: 125-16,384 \n * standard: 1-1,024" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true volumeType: - description: |- - VolumeType of the block device. - For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + description: VolumeType of the block device. For more information, + see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) in the Amazon Elastic Compute Cloud User Guide. type: string type: object type: object type: array context: - description: |- - Context is a Reserved field in EC2 APIs - https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + description: Context is a Reserved field in EC2 APIs https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html type: string detailedMonitoring: description: DetailedMonitoring controls if detailed monitoring is @@ -165,81 +133,63 @@ spec: description: InstanceProfile is the AWS identity that instances use. type: string kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint + the client submits requests to. Cannot be updated. In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string launchTemplate: - description: |- - LaunchTemplateName for the node. If not specified, a launch template will be generated. - NOTE: This field is for specifying a custom launch template and is exposed in the Spec - as `launchTemplate` for backwards compatibility. + description: 'LaunchTemplateName for the node. If not specified, a + launch template will be generated. NOTE: This field is for specifying + a custom launch template and is exposed in the Spec as `launchTemplate` + for backwards compatibility.' type: string metadataOptions: - description: |- - MetadataOptions for the generated launch template of provisioned nodes. - - - This specifies the exposure of the Instance Metadata Service to - provisioned EC2 nodes. For more information, - see Instance Metadata and User Data - (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) - in the Amazon Elastic Compute Cloud User Guide. - - - Refer to recommended, security best practices - (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + description: "MetadataOptions for the generated launch template of + provisioned nodes. \n This specifies the exposure of the Instance + Metadata Service to provisioned EC2 nodes. For more information, + see Instance Metadata and User Data (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. \n Refer to recommended, + security best practices (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) for limiting exposure of Instance Metadata and User Data to pods. If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 - disabled, with httpPutResponseLimit of 2, and with httpTokens - required. + disabled, with httpPutResponseLimit of 2, and with httpTokens required." properties: httpEndpoint: - description: |- - HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned - nodes. If metadata options is non-nil, but this parameter is not specified, - the default state is "enabled". - - - If you specify a value of "disabled", instance metadata will not be accessible - on the node. + description: "HTTPEndpoint enables or disables the HTTP metadata + endpoint on provisioned nodes. If metadata options is non-nil, + but this parameter is not specified, the default state is \"enabled\". + \n If you specify a value of \"disabled\", instance metadata + will not be accessible on the node." type: string httpProtocolIPv6: - description: |- - HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata - service on provisioned nodes. If metadata options is non-nil, but this parameter - is not specified, the default state is "disabled". + description: HTTPProtocolIPv6 enables or disables the IPv6 endpoint + for the instance metadata service on provisioned nodes. If metadata + options is non-nil, but this parameter is not specified, the + default state is "disabled". type: string httpPutResponseHopLimit: - description: |- - HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for - instance metadata requests. The larger the number, the further instance - metadata requests can travel. Possible values are integers from 1 to 64. - If metadata options is non-nil, but this parameter is not specified, the - default value is 1. + description: HTTPPutResponseHopLimit is the desired HTTP PUT response + hop limit for instance metadata requests. The larger the number, + the further instance metadata requests can travel. Possible + values are integers from 1 to 64. If metadata options is non-nil, + but this parameter is not specified, the default value is 1. format: int64 type: integer httpTokens: - description: |- - HTTPTokens determines the state of token usage for instance metadata - requests. If metadata options is non-nil, but this parameter is not - specified, the default state is "optional". - - - If the state is optional, one can choose to retrieve instance metadata with - or without a signed token header on the request. If one retrieves the IAM - role credentials without a token, the version 1.0 role credentials are - returned. If one retrieves the IAM role credentials using a valid signed - token, the version 2.0 role credentials are returned. - - - If the state is "required", one must send a signed token header with any - instance metadata retrieval requests. In this state, retrieving the IAM - role credentials always returns the version 2.0 credentials; the version - 1.0 credentials are not available. + description: "HTTPTokens determines the state of token usage for + instance metadata requests. If metadata options is non-nil, + but this parameter is not specified, the default state is \"optional\". + \n If the state is optional, one can choose to retrieve instance + metadata with or without a signed token header on the request. + If one retrieves the IAM role credentials without a token, the + version 1.0 role credentials are returned. If one retrieves + the IAM role credentials using a valid signed token, the version + 2.0 role credentials are returned. \n If the state is \"required\", + one must send a signed token header with any instance metadata + retrieval requests. In this state, retrieving the IAM role credentials + always returns the version 2.0 credentials; the version 1.0 + credentials are not available." type: string type: object securityGroupSelector: @@ -260,10 +210,10 @@ spec: launch templates. type: object userData: - description: |- - UserData to be applied to the provisioned nodes. - It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into - this UserData to ensure nodes are being provisioned with the correct configuration. + description: UserData to be applied to the provisioned nodes. It must + be in the appropriate format based on the AMIFamily in use. Karpenter + will merge certain fields into this UserData to ensure nodes are + being provisioned with the correct configuration. type: string type: object status: @@ -271,9 +221,8 @@ spec: AWSNodeTemplate properties: amis: - description: |- - AMI contains the current AMI values that are available to the - cluster under the AMI selectors. + description: AMI contains the current AMI values that are available + to the cluster under the AMI selectors. items: description: AMI contains resolved AMI selector values utilized for node launch @@ -288,25 +237,26 @@ spec: description: Requirements of the AMI to be utilized on an instance type items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. + description: A node selector requirement is a selector that + contains values, a key, and an operator that relates the + key and values. properties: key: description: The label key that the selector applies to. type: string operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: Represents a key's relationship to a set + of values. Valid operators are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. type: string values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, the + values array must have a single element, which will + be interpreted as an integer. This array is replaced + during a strategic merge patch. items: type: string type: array @@ -321,9 +271,8 @@ spec: type: object type: array securityGroups: - description: |- - SecurityGroups contains the current Security Groups values that are available to the - cluster under the SecurityGroups selectors. + description: SecurityGroups contains the current Security Groups values + that are available to the cluster under the SecurityGroups selectors. items: description: SecurityGroup contains resolved SecurityGroup selector values utilized for node launch @@ -339,9 +288,8 @@ spec: type: object type: array subnets: - description: |- - Subnets contains the current Subnet values that are available to the - cluster under the subnet selectors. + description: Subnets contains the current Subnet values that are available + to the cluster under the subnet selectors. items: description: Subnet contains resolved Subnet selector values utilized for node launch diff --git a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml index 329aba7fba0b..467ff450d4f2 100644 --- a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.13.0 name: ec2nodeclasses.karpenter.k8s.aws spec: group: karpenter.k8s.aws @@ -25,26 +25,21 @@ spec: description: EC2NodeClass is the Schema for the EC2NodeClass API properties: apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: - description: |- - EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. - This will contain configuration necessary to launch instances in AWS. + description: EC2NodeClassSpec is the top level specification for the AWS + Karpenter Provider. This will contain configuration necessary to launch + instances in AWS. properties: amiFamily: description: AMIFamily is the AMI family that instances use. @@ -60,30 +55,28 @@ spec: description: AMISelectorTerms is a list of or ami selector terms. The terms are ORed. items: - description: |- - AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. - If multiple fields are used for selection, the requirements are ANDed. + description: AMISelectorTerm defines selection logic for an ami + used by Karpenter to launch nodes. If multiple fields are used + for selection, the requirements are ANDed. properties: id: description: ID is the ami id in EC2 pattern: ami-[0-9a-z]+ type: string name: - description: |- - Name is the ami name in EC2. - This value is the name field, which is different from the name tag. + description: Name is the ami name in EC2. This value is the + name field, which is different from the name tag. type: string owner: - description: |- - Owner is the owner for the ami. - You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + description: Owner is the owner for the ami. You can specify + a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" type: string tags: additionalProperties: type: string - description: |- - Tags is a map of key/value tags used to select subnets - Specifying '*' for a value selects all values for a given tag key. + description: Tags is a map of key/value tags used to select + subnets Specifying '*' for a value selects all values for + a given tag key. maxProperties: 20 type: object x-kubernetes-validations: @@ -115,38 +108,27 @@ spec: volume is deleted on instance termination. type: boolean encrypted: - description: |- - Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only - be attached to instances that support Amazon EBS encryption. If you are creating - a volume from a snapshot, you can't specify an encryption value. + description: Encrypted indicates whether the EBS volume + is encrypted. Encrypted volumes can only be attached to + instances that support Amazon EBS encryption. If you are + creating a volume from a snapshot, you can't specify an + encryption value. type: boolean iops: - description: |- - IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, - this represents the number of IOPS that are provisioned for the volume. For - gp2 volumes, this represents the baseline performance of the volume and the - rate at which the volume accumulates I/O credits for bursting. - - - The following are the supported values for each volume type: - - - * gp3: 3,000-16,000 IOPS - - - * io1: 100-64,000 IOPS - - - * io2: 100-64,000 IOPS - - - For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built - on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). - Other instance families guarantee performance up to 32,000 IOPS. - - - This parameter is supported for io1, io2, and gp3 volumes only. This parameter - is not supported for gp2, st1, sc1, or standard volumes. + description: "IOPS is the number of I/O operations per second + (IOPS). For gp3, io1, and io2 volumes, this represents + the number of IOPS that are provisioned for the volume. + For gp2 volumes, this represents the baseline performance + of the volume and the rate at which the volume accumulates + I/O credits for bursting. \n The following are the supported + values for each volume type: \n * gp3: 3,000-16,000 IOPS + \n * io1: 100-64,000 IOPS \n * io2: 100-64,000 IOPS \n + For io1 and io2 volumes, we guarantee 64,000 IOPS only + for Instances built on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 + IOPS. \n This parameter is supported for io1, io2, and + gp3 volumes only. This parameter is not supported for + gp2, st1, sc1, or standard volumes." format: int64 type: integer kmsKeyID: @@ -157,9 +139,9 @@ spec: description: SnapshotID is the ID of an EBS snapshot type: string throughput: - description: |- - Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. - Valid Range: Minimum value of 125. Maximum value of 1000. + description: 'Throughput to provision for a gp3 volume, + with a maximum of 1,000 MiB/s. Valid Range: Minimum value + of 125. Maximum value of 1000.' format: int64 type: integer volumeSize: @@ -169,27 +151,15 @@ spec: anyOf: - type: integer - type: string - description: |- - VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or - a volume size. The following are the supported volumes sizes for each volume - type: - - - * gp2 and gp3: 1-16,384 - - - * io1 and io2: 4-16,384 - - - * st1 and sc1: 125-16,384 - - - * standard: 1-1,024 + description: "VolumeSize in `Gi`, `G`, `Ti`, or `T`. You + must specify either a snapshot ID or a volume size. The + following are the supported volumes sizes for each volume + type: \n * gp2 and gp3: 1-16,384 \n * io1 and io2: 4-16,384 + \n * st1 and sc1: 125-16,384 \n * standard: 1-1,024" x-kubernetes-int-or-string: true volumeType: - description: |- - VolumeType of the block device. - For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + description: VolumeType of the block device. For more information, + see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) in the Amazon Elastic Compute Cloud User Guide. enum: - standard @@ -205,9 +175,9 @@ spec: - message: snapshotID or volumeSize must be defined rule: has(self.snapshotID) || has(self.volumeSize) rootVolume: - description: |- - RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can - configure at most one root volume in BlockDeviceMappings. + description: RootVolume is a flag indicating if this device + is mounted as kubelet root dir. You can configure at most + one root volume in BlockDeviceMappings. type: boolean type: object maxItems: 50 @@ -217,20 +187,18 @@ spec: rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1 context: - description: |- - Context is a Reserved field in EC2 APIs - https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + description: Context is a Reserved field in EC2 APIs https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html type: string detailedMonitoring: description: DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched type: boolean instanceProfile: - description: |- - InstanceProfile is the AWS entity that instances use. - This field is mutually exclusive from role. - The instance profile should already have a role assigned to it that Karpenter - has PassRole permission on for instance launch using this instanceProfile to succeed. + description: InstanceProfile is the AWS entity that instances use. + This field is mutually exclusive from role. The instance profile + should already have a role assigned to it that Karpenter has PassRole + permission on for instance launch using this instanceProfile to + succeed. type: string x-kubernetes-validations: - message: instanceProfile cannot be empty @@ -241,91 +209,76 @@ spec: httpProtocolIPv6: disabled httpPutResponseHopLimit: 2 httpTokens: required - description: |- - MetadataOptions for the generated launch template of provisioned nodes. - - - This specifies the exposure of the Instance Metadata Service to - provisioned EC2 nodes. For more information, - see Instance Metadata and User Data - (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) - in the Amazon Elastic Compute Cloud User Guide. - - - Refer to recommended, security best practices - (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + description: "MetadataOptions for the generated launch template of + provisioned nodes. \n This specifies the exposure of the Instance + Metadata Service to provisioned EC2 nodes. For more information, + see Instance Metadata and User Data (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. \n Refer to recommended, + security best practices (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) for limiting exposure of Instance Metadata and User Data to pods. If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 - disabled, with httpPutResponseLimit of 2, and with httpTokens - required. + disabled, with httpPutResponseLimit of 2, and with httpTokens required." properties: httpEndpoint: default: enabled - description: |- - HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned - nodes. If metadata options is non-nil, but this parameter is not specified, - the default state is "enabled". - - - If you specify a value of "disabled", instance metadata will not be accessible - on the node. + description: "HTTPEndpoint enables or disables the HTTP metadata + endpoint on provisioned nodes. If metadata options is non-nil, + but this parameter is not specified, the default state is \"enabled\". + \n If you specify a value of \"disabled\", instance metadata + will not be accessible on the node." enum: - enabled - disabled type: string httpProtocolIPv6: default: disabled - description: |- - HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata - service on provisioned nodes. If metadata options is non-nil, but this parameter - is not specified, the default state is "disabled". + description: HTTPProtocolIPv6 enables or disables the IPv6 endpoint + for the instance metadata service on provisioned nodes. If metadata + options is non-nil, but this parameter is not specified, the + default state is "disabled". enum: - enabled - disabled type: string httpPutResponseHopLimit: default: 2 - description: |- - HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for - instance metadata requests. The larger the number, the further instance - metadata requests can travel. Possible values are integers from 1 to 64. - If metadata options is non-nil, but this parameter is not specified, the - default value is 2. + description: HTTPPutResponseHopLimit is the desired HTTP PUT response + hop limit for instance metadata requests. The larger the number, + the further instance metadata requests can travel. Possible + values are integers from 1 to 64. If metadata options is non-nil, + but this parameter is not specified, the default value is 2. format: int64 maximum: 64 minimum: 1 type: integer httpTokens: default: required - description: |- - HTTPTokens determines the state of token usage for instance metadata - requests. If metadata options is non-nil, but this parameter is not - specified, the default state is "required". - - - If the state is optional, one can choose to retrieve instance metadata with - or without a signed token header on the request. If one retrieves the IAM - role credentials without a token, the version 1.0 role credentials are - returned. If one retrieves the IAM role credentials using a valid signed - token, the version 2.0 role credentials are returned. - - - If the state is "required", one must send a signed token header with any - instance metadata retrieval requests. In this state, retrieving the IAM - role credentials always returns the version 2.0 credentials; the version - 1.0 credentials are not available. + description: "HTTPTokens determines the state of token usage for + instance metadata requests. If metadata options is non-nil, + but this parameter is not specified, the default state is \"required\". + \n If the state is optional, one can choose to retrieve instance + metadata with or without a signed token header on the request. + If one retrieves the IAM role credentials without a token, the + version 1.0 role credentials are returned. If one retrieves + the IAM role credentials using a valid signed token, the version + 2.0 role credentials are returned. \n If the state is \"required\", + one must send a signed token header with any instance metadata + retrieval requests. In this state, retrieving the IAM role credentials + always returns the version 2.0 credentials; the version 1.0 + credentials are not available." enum: - required - optional type: string type: object role: - description: |- - Role is the AWS identity that nodes use. This field is immutable. - This field is mutually exclusive from instanceProfile. - Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. - This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented - for the old instance profiles on an update. + description: Role is the AWS identity that nodes use. This field is + immutable. This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating + managed instance profiles from running instances. This field may + be made mutable in the future, assuming the correct garbage collection + and drift handling is implemented for the old instance profiles + on an update. type: string x-kubernetes-validations: - message: role cannot be empty @@ -336,25 +289,24 @@ spec: description: SecurityGroupSelectorTerms is a list of or security group selector terms. The terms are ORed. items: - description: |- - SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. - If multiple fields are used for selection, the requirements are ANDed. + description: SecurityGroupSelectorTerm defines selection logic for + a security group used by Karpenter to launch nodes. If multiple + fields are used for selection, the requirements are ANDed. properties: id: description: ID is the security group id in EC2 pattern: sg-[0-9a-z]+ type: string name: - description: |- - Name is the security group name in EC2. - This value is the name field, which is different from the name tag. + description: Name is the security group name in EC2. This value + is the name field, which is different from the name tag. type: string tags: additionalProperties: type: string - description: |- - Tags is a map of key/value tags used to select subnets - Specifying '*' for a value selects all values for a given tag key. + description: Tags is a map of key/value tags used to select + subnets Specifying '*' for a value selects all values for + a given tag key. maxProperties: 20 type: object x-kubernetes-validations: @@ -378,9 +330,9 @@ spec: description: SubnetSelectorTerms is a list of or subnet selector terms. The terms are ORed. items: - description: |- - SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. - If multiple fields are used for selection, the requirements are ANDed. + description: SubnetSelectorTerm defines selection logic for a subnet + used by Karpenter to launch nodes. If multiple fields are used + for selection, the requirements are ANDed. properties: id: description: ID is the subnet id in EC2 @@ -389,9 +341,9 @@ spec: tags: additionalProperties: type: string - description: |- - Tags is a map of key/value tags used to select subnets - Specifying '*' for a value selects all values for a given tag key. + description: Tags is a map of key/value tags used to select + subnets Specifying '*' for a value selects all values for + a given tag key. maxProperties: 20 type: object x-kubernetes-validations: @@ -426,10 +378,10 @@ spec: - message: tag contains a restricted tag matching karpenter.sh/managed-by rule: self.all(k, k !='karpenter.sh/managed-by') userData: - description: |- - UserData to be applied to the provisioned nodes. - It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into - this UserData to ensure nodes are being provisioned with the correct configuration. + description: UserData to be applied to the provisioned nodes. It must + be in the appropriate format based on the AMIFamily in use. Karpenter + will merge certain fields into this UserData to ensure nodes are + being provisioned with the correct configuration. type: string required: - amiFamily @@ -452,9 +404,8 @@ spec: description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass properties: amis: - description: |- - AMI contains the current AMI values that are available to the - cluster under the AMI selectors. + description: AMI contains the current AMI values that are available + to the cluster under the AMI selectors. items: description: AMI contains resolved AMI selector values utilized for node launch @@ -469,25 +420,26 @@ spec: description: Requirements of the AMI to be utilized on an instance type items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. + description: A node selector requirement is a selector that + contains values, a key, and an operator that relates the + key and values. properties: key: description: The label key that the selector applies to. type: string operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: Represents a key's relationship to a set + of values. Valid operators are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. type: string values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, the + values array must have a single element, which will + be interpreted as an integer. This array is replaced + during a strategic merge patch. items: type: string type: array @@ -506,9 +458,8 @@ spec: for the role type: string securityGroups: - description: |- - SecurityGroups contains the current Security Groups values that are available to the - cluster under the SecurityGroups selectors. + description: SecurityGroups contains the current Security Groups values + that are available to the cluster under the SecurityGroups selectors. items: description: SecurityGroup contains resolved SecurityGroup selector values utilized for node launch @@ -524,9 +475,8 @@ spec: type: object type: array subnets: - description: |- - Subnets contains the current Subnet values that are available to the - cluster under the subnet selectors. + description: Subnets contains the current Subnet values that are available + to the cluster under the subnet selectors. items: description: Subnet contains resolved Subnet selector values utilized for node launch diff --git a/pkg/providers/subnet/subnet.go b/pkg/providers/subnet/subnet.go index c51be4b04946..2e95804f1e1a 100644 --- a/pkg/providers/subnet/subnet.go +++ b/pkg/providers/subnet/subnet.go @@ -131,7 +131,10 @@ func (p *Provider) ZonalSubnetsForLaunch(ctx context.Context, nodeClass *v1beta1 if ips, ok := p.inflightIPs[*subnets[j].SubnetId]; ok { jIPs = ips } - return iIPs < jIPs + if iIPs != jIPs { + return iIPs < jIPs + } + return *subnets[i].SubnetId < *subnets[j].SubnetId }) for _, subnet := range subnets { zonalSubnets[*subnet.AvailabilityZone] = subnet