diff --git a/pkg/apis/apis.go b/pkg/apis/apis.go index f99bd99dd8f6..29203dba69e2 100644 --- a/pkg/apis/apis.go +++ b/pkg/apis/apis.go @@ -18,9 +18,10 @@ package apis import ( _ "embed" - v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/samber/lo" @@ -33,6 +34,7 @@ var ( // Builder includes all types within the apis package Builder = runtime.NewSchemeBuilder( v1beta1.SchemeBuilder.AddToScheme, + v1.SchemeBuilder.AddToScheme, ) // AddToScheme may be used to add all resources defined in the project to a Scheme AddToScheme = Builder.AddToScheme @@ -43,6 +45,6 @@ var ( //go:embed crds/karpenter.k8s.aws_ec2nodeclasses.yaml EC2NodeClassCRD []byte CRDs = append(apis.CRDs, - lo.Must(functional.Unmarshal[v1.CustomResourceDefinition](EC2NodeClassCRD)), + lo.Must(functional.Unmarshal[apiextensionsv1.CustomResourceDefinition](EC2NodeClassCRD)), ) ) diff --git a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml index 01582e9bbbd5..a3464c3df20b 100644 --- a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -19,6 +19,606 @@ spec: singular: ec2nodeclass scope: Cluster versions: + - name: v1 + schema: + openAPIV3Schema: + description: EC2NodeClass is the Schema for the EC2NodeClass API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. + This will contain configuration necessary to launch instances in AWS. + properties: + amiFamily: + description: AMIFamily is the AMI family that instances use. + enum: + - AL2 + - AL2023 + - Bottlerocket + - Ubuntu + - Custom + - Windows2019 + - Windows2022 + type: string + amiSelectorTerms: + description: AMISelectorTerms is a list of or ami selector terms. + The terms are ORed. + items: + description: |- + AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the ami id in EC2 + pattern: ami-[0-9a-z]+ + type: string + name: + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. + type: string + owner: + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a combination + of other fields in amiSelectorTerms' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name) || + has(x.owner)))' + associatePublicIPAddress: + description: AssociatePublicIPAddress controls if public IP addresses + are assigned to instances that are launched with the nodeclass. + type: boolean + blockDeviceMappings: + description: BlockDeviceMappings to be applied to provisioned nodes. + items: + properties: + deviceName: + description: The device name (for example, /dev/sdh or xvdh). + type: string + ebs: + description: EBS contains parameters used to automatically set + up EBS volumes when an instance is launched. + properties: + deleteOnTermination: + description: DeleteOnTermination indicates whether the EBS + volume is deleted on instance termination. + type: boolean + encrypted: + description: |- + Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + be attached to instances that support Amazon EBS encryption. If you are creating + a volume from a snapshot, you can't specify an encryption value. + type: boolean + iops: + description: |- + IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + this represents the number of IOPS that are provisioned for the volume. For + gp2 volumes, this represents the baseline performance of the volume and the + rate at which the volume accumulates I/O credits for bursting. + + + The following are the supported values for each volume type: + + + * gp3: 3,000-16,000 IOPS + + + * io1: 100-64,000 IOPS + + + * io2: 100-64,000 IOPS + + + For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 IOPS. + + + This parameter is supported for io1, io2, and gp3 volumes only. This parameter + is not supported for gp2, st1, sc1, or standard volumes. + format: int64 + type: integer + kmsKeyID: + description: KMSKeyID (ARN) of the symmetric Key Management + Service (KMS) CMK used for encryption. + type: string + snapshotID: + description: SnapshotID is the ID of an EBS snapshot + type: string + throughput: + description: |- + Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + Valid Range: Minimum value of 125. Maximum value of 1000. + format: int64 + type: integer + volumeSize: + description: |- + VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + a volume size. The following are the supported volumes sizes for each volume + type: + + + * gp2 and gp3: 1-16,384 + + + * io1 and io2: 4-16,384 + + + * st1 and sc1: 125-16,384 + + + * standard: 1-1,024 + pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ + type: string + volumeType: + description: |- + VolumeType of the block device. + For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + in the Amazon Elastic Compute Cloud User Guide. + enum: + - standard + - io1 + - io2 + - gp2 + - sc1 + - st1 + - gp3 + type: string + type: object + x-kubernetes-validations: + - message: snapshotID or volumeSize must be defined + rule: has(self.snapshotID) || has(self.volumeSize) + rootVolume: + description: |- + RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + configure at most one root volume in BlockDeviceMappings. + type: boolean + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: must have only one blockDeviceMappings with rootVolume + rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() + <= 1 + context: + description: |- + Context is a Reserved field in EC2 APIs + https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + type: string + detailedMonitoring: + description: DetailedMonitoring controls if detailed monitoring is + enabled for instances that are launched + type: boolean + instanceProfile: + description: |- + InstanceProfile is the AWS entity that instances use. + This field is mutually exclusive from role. + The instance profile should already have a role assigned to it that Karpenter + has PassRole permission on for instance launch using this instanceProfile to succeed. + type: string + x-kubernetes-validations: + - message: instanceProfile cannot be empty + rule: self != '' + instanceStorePolicy: + description: InstanceStorePolicy specifies how to handle instance-store + disks. + enum: + - RAID0 + type: string + metadataOptions: + default: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: required + description: |- + MetadataOptions for the generated launch template of provisioned nodes. + + + This specifies the exposure of the Instance Metadata Service to + provisioned EC2 nodes. For more information, + see Instance Metadata and User Data + (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. + + + Refer to recommended, security best practices + (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + for limiting exposure of Instance Metadata and User Data to pods. + If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + disabled, with httpPutResponseLimit of 2, and with httpTokens + required. + properties: + httpEndpoint: + default: enabled + description: |- + HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + nodes. If metadata options is non-nil, but this parameter is not specified, + the default state is "enabled". + + + If you specify a value of "disabled", instance metadata will not be accessible + on the node. + enum: + - enabled + - disabled + type: string + httpProtocolIPv6: + default: disabled + description: |- + HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + service on provisioned nodes. If metadata options is non-nil, but this parameter + is not specified, the default state is "disabled". + enum: + - enabled + - disabled + type: string + httpPutResponseHopLimit: + default: 2 + description: |- + HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + instance metadata requests. The larger the number, the further instance + metadata requests can travel. Possible values are integers from 1 to 64. + If metadata options is non-nil, but this parameter is not specified, the + default value is 2. + format: int64 + maximum: 64 + minimum: 1 + type: integer + httpTokens: + default: required + description: |- + HTTPTokens determines the state of token usage for instance metadata + requests. If metadata options is non-nil, but this parameter is not + specified, the default state is "required". + + + If the state is optional, one can choose to retrieve instance metadata with + or without a signed token header on the request. If one retrieves the IAM + role credentials without a token, the version 1.0 role credentials are + returned. If one retrieves the IAM role credentials using a valid signed + token, the version 2.0 role credentials are returned. + + + If the state is "required", one must send a signed token header with any + instance metadata retrieval requests. In this state, retrieving the IAM + role credentials always returns the version 2.0 credentials; the version + 1.0 credentials are not available. + enum: + - required + - optional + type: string + type: object + role: + description: |- + Role is the AWS identity that nodes use. This field is immutable. + This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + for the old instance profiles on an update. + type: string + x-kubernetes-validations: + - message: role cannot be empty + rule: self != '' + - message: immutable field changed + rule: self == oldSelf + securityGroupSelectorTerms: + description: SecurityGroupSelectorTerms is a list of or security group + selector terms. The terms are ORed. + items: + description: |- + SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the security group id in EC2 + pattern: sg-[0-9a-z]+ + type: string + name: + description: |- + Name is the security group name in EC2. + This value is the name field, which is different from the name tag. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: securityGroupSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a combination + of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' + - message: '''name'' is mutually exclusive, cannot be set with a combination + of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' + subnetSelectorTerms: + description: SubnetSelectorTerms is a list of or subnet selector terms. + The terms are ORed. + items: + description: |- + SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the subnet id in EC2 + pattern: subnet-[0-9a-z]+ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: subnetSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set with a combination + of other fields in subnetSelectorTerms' + rule: '!self.all(x, has(x.id) && has(x.tags))' + tags: + additionalProperties: + type: string + description: Tags to be applied on ec2 resources like instances and + launch templates. + type: object + x-kubernetes-validations: + - message: empty tag keys aren't supported + rule: self.all(k, k != '') + - message: tag contains a restricted tag matching kubernetes.io/cluster/ + rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) + - message: tag contains a restricted tag matching karpenter.sh/nodepool + rule: self.all(k, k != 'karpenter.sh/nodepool') + - message: tag contains a restricted tag matching karpenter.sh/managed-by + rule: self.all(k, k !='karpenter.sh/managed-by') + - message: tag contains a restricted tag matching karpenter.sh/nodeclaim + rule: self.all(k, k !='karpenter.sh/nodeclaim') + - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass + rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') + userData: + description: |- + UserData to be applied to the provisioned nodes. + It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + this UserData to ensure nodes are being provisioned with the correct configuration. + type: string + required: + - amiFamily + - securityGroupSelectorTerms + - subnetSelectorTerms + type: object + x-kubernetes-validations: + - message: amiSelectorTerms is required when amiFamily == 'Custom' + rule: 'self.amiFamily == ''Custom'' ? self.amiSelectorTerms.size() != + 0 : true' + - message: must specify exactly one of ['role', 'instanceProfile'] + rule: (has(self.role) && !has(self.instanceProfile)) || (!has(self.role) + && has(self.instanceProfile)) + - message: changing from 'instanceProfile' to 'role' is not supported. + You must delete and recreate this node class if you want to change + this. + rule: (has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) + && has(self.instanceProfile)) + status: + description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass + properties: + amis: + description: |- + AMI contains the current AMI values that are available to the + cluster under the AMI selectors. + items: + description: AMI contains resolved AMI selector values utilized + for node launch + properties: + id: + description: ID of the AMI + type: string + name: + description: Name of the AMI + type: string + requirements: + description: Requirements of the AMI to be utilized on an instance + type + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + required: + - id + - requirements + type: object + type: array + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional + helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + instanceProfile: + description: InstanceProfile contains the resolved instance profile + for the role + type: string + securityGroups: + description: |- + SecurityGroups contains the current Security Groups values that are available to the + cluster under the SecurityGroups selectors. + items: + description: SecurityGroup contains resolved SecurityGroup selector + values utilized for node launch + properties: + id: + description: ID of the security group + type: string + name: + description: Name of the security group + type: string + required: + - id + type: object + type: array + subnets: + description: |- + Subnets contains the current Subnet values that are available to the + cluster under the subnet selectors. + items: + description: Subnet contains resolved Subnet selector values utilized + for node launch + properties: + id: + description: ID of the subnet + type: string + zone: + description: The associated availability zone + type: string + required: + - id + - zone + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} - name: v1beta1 schema: openAPIV3Schema: @@ -616,6 +1216,6 @@ spec: type: object type: object served: true - storage: true + storage: false subresources: status: {} diff --git a/pkg/apis/v1/doc.go b/pkg/apis/v1/doc.go new file mode 100644 index 000000000000..170c72b672d6 --- /dev/null +++ b/pkg/apis/v1/doc.go @@ -0,0 +1,19 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen=package,register +// +k8s:defaulter-gen=TypeMeta +// +groupName=karpenter.k8s.aws +package v1 // doc.go is discovered by codegen diff --git a/pkg/apis/v1/ec2nodeclass.go b/pkg/apis/v1/ec2nodeclass.go new file mode 100644 index 000000000000..01fb7cb67789 --- /dev/null +++ b/pkg/apis/v1/ec2nodeclass.go @@ -0,0 +1,370 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "fmt" + + "github.com/mitchellh/hashstructure/v2" + "github.com/samber/lo" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" +) + +// EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. +// This will contain configuration necessary to launch instances in AWS. +type EC2NodeClassSpec struct { + // SubnetSelectorTerms is a list of or subnet selector terms. The terms are ORed. + // +kubebuilder:validation:XValidation:message="subnetSelectorTerms cannot be empty",rule="self.size() != 0" + // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id']",rule="self.all(x, has(x.tags) || has(x.id))" + // +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set with a combination of other fields in subnetSelectorTerms",rule="!self.all(x, has(x.id) && has(x.tags))" + // +kubebuilder:validation:MaxItems:=30 + // +required + SubnetSelectorTerms []SubnetSelectorTerm `json:"subnetSelectorTerms" hash:"ignore"` + // SecurityGroupSelectorTerms is a list of or security group selector terms. The terms are ORed. + // +kubebuilder:validation:XValidation:message="securityGroupSelectorTerms cannot be empty",rule="self.size() != 0" + // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id', 'name']",rule="self.all(x, has(x.tags) || has(x.id) || has(x.name))" + // +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms",rule="!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))" + // +kubebuilder:validation:XValidation:message="'name' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms",rule="!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))" + // +kubebuilder:validation:MaxItems:=30 + // +required + SecurityGroupSelectorTerms []SecurityGroupSelectorTerm `json:"securityGroupSelectorTerms" hash:"ignore"` + // AssociatePublicIPAddress controls if public IP addresses are assigned to instances that are launched with the nodeclass. + // +optional + AssociatePublicIPAddress *bool `json:"associatePublicIPAddress,omitempty"` + // AMISelectorTerms is a list of or ami selector terms. The terms are ORed. + // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id', 'name']",rule="self.all(x, has(x.tags) || has(x.id) || has(x.name))" + // +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms",rule="!self.all(x, has(x.id) && (has(x.tags) || has(x.name) || has(x.owner)))" + // +kubebuilder:validation:MaxItems:=30 + // +optional + AMISelectorTerms []AMISelectorTerm `json:"amiSelectorTerms,omitempty" hash:"ignore"` + // AMIFamily is the AMI family that instances use. + // +kubebuilder:validation:Enum:={AL2,AL2023,Bottlerocket,Ubuntu,Custom,Windows2019,Windows2022} + // +required + AMIFamily *string `json:"amiFamily"` + // UserData to be applied to the provisioned nodes. + // It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + // this UserData to ensure nodes are being provisioned with the correct configuration. + // +optional + UserData *string `json:"userData,omitempty"` + // Role is the AWS identity that nodes use. This field is immutable. + // This field is mutually exclusive from instanceProfile. + // Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + // This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + // for the old instance profiles on an update. + // +kubebuilder:validation:XValidation:rule="self != ''",message="role cannot be empty" + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="immutable field changed" + // +optional + Role string `json:"role,omitempty"` + // InstanceProfile is the AWS entity that instances use. + // This field is mutually exclusive from role. + // The instance profile should already have a role assigned to it that Karpenter + // has PassRole permission on for instance launch using this instanceProfile to succeed. + // +kubebuilder:validation:XValidation:rule="self != ''",message="instanceProfile cannot be empty" + // +optional + InstanceProfile *string `json:"instanceProfile,omitempty"` + // Tags to be applied on ec2 resources like instances and launch templates. + // +kubebuilder:validation:XValidation:message="empty tag keys aren't supported",rule="self.all(k, k != '')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching kubernetes.io/cluster/",rule="self.all(k, !k.startsWith('kubernetes.io/cluster') )" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.sh/nodepool",rule="self.all(k, k != 'karpenter.sh/nodepool')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.sh/managed-by",rule="self.all(k, k !='karpenter.sh/managed-by')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.sh/nodeclaim",rule="self.all(k, k !='karpenter.sh/nodeclaim')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass",rule="self.all(k, k !='karpenter.k8s.aws/ec2nodeclass')" + // +optional + Tags map[string]string `json:"tags,omitempty"` + // BlockDeviceMappings to be applied to provisioned nodes. + // +kubebuilder:validation:XValidation:message="must have only one blockDeviceMappings with rootVolume",rule="self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1" + // +kubebuilder:validation:MaxItems:=50 + // +optional + BlockDeviceMappings []*BlockDeviceMapping `json:"blockDeviceMappings,omitempty"` + // InstanceStorePolicy specifies how to handle instance-store disks. + // +optional + InstanceStorePolicy *InstanceStorePolicy `json:"instanceStorePolicy,omitempty"` + // DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched + // +optional + DetailedMonitoring *bool `json:"detailedMonitoring,omitempty"` + // MetadataOptions for the generated launch template of provisioned nodes. + // + // This specifies the exposure of the Instance Metadata Service to + // provisioned EC2 nodes. For more information, + // see Instance Metadata and User Data + // (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + // in the Amazon Elastic Compute Cloud User Guide. + // + // Refer to recommended, security best practices + // (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + // for limiting exposure of Instance Metadata and User Data to pods. + // If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + // disabled, with httpPutResponseLimit of 2, and with httpTokens + // required. + // +kubebuilder:default={"httpEndpoint":"enabled","httpProtocolIPv6":"disabled","httpPutResponseHopLimit":2,"httpTokens":"required"} + // +optional + MetadataOptions *MetadataOptions `json:"metadataOptions,omitempty"` + // Context is a Reserved field in EC2 APIs + // https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + // +optional + Context *string `json:"context,omitempty"` +} + +// SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. +// If multiple fields are used for selection, the requirements are ANDed. +type SubnetSelectorTerm struct { + // Tags is a map of key/value tags used to select subnets + // Specifying '*' for a value selects all values for a given tag key. + // +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')" + // +kubebuilder:validation:MaxProperties:=20 + // +optional + Tags map[string]string `json:"tags,omitempty"` + // ID is the subnet id in EC2 + // +kubebuilder:validation:Pattern="subnet-[0-9a-z]+" + // +optional + ID string `json:"id,omitempty"` +} + +// SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. +// If multiple fields are used for selection, the requirements are ANDed. +type SecurityGroupSelectorTerm struct { + // Tags is a map of key/value tags used to select subnets + // Specifying '*' for a value selects all values for a given tag key. + // +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')" + // +kubebuilder:validation:MaxProperties:=20 + // +optional + Tags map[string]string `json:"tags,omitempty"` + // ID is the security group id in EC2 + // +kubebuilder:validation:Pattern:="sg-[0-9a-z]+" + // +optional + ID string `json:"id,omitempty"` + // Name is the security group name in EC2. + // This value is the name field, which is different from the name tag. + Name string `json:"name,omitempty"` +} + +// AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. +// If multiple fields are used for selection, the requirements are ANDed. +type AMISelectorTerm struct { + // Tags is a map of key/value tags used to select subnets + // Specifying '*' for a value selects all values for a given tag key. + // +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')" + // +kubebuilder:validation:MaxProperties:=20 + // +optional + Tags map[string]string `json:"tags,omitempty"` + // ID is the ami id in EC2 + // +kubebuilder:validation:Pattern:="ami-[0-9a-z]+" + // +optional + ID string `json:"id,omitempty"` + // Name is the ami name in EC2. + // This value is the name field, which is different from the name tag. + // +optional + Name string `json:"name,omitempty"` + // Owner is the owner for the ami. + // You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + // +optional + Owner string `json:"owner,omitempty"` +} + +// MetadataOptions contains parameters for specifying the exposure of the +// Instance Metadata Service to provisioned EC2 nodes. +type MetadataOptions struct { + // HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + // nodes. If metadata options is non-nil, but this parameter is not specified, + // the default state is "enabled". + // + // If you specify a value of "disabled", instance metadata will not be accessible + // on the node. + // +kubebuilder:default=enabled + // +kubebuilder:validation:Enum:={enabled,disabled} + // +optional + HTTPEndpoint *string `json:"httpEndpoint,omitempty"` + // HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + // service on provisioned nodes. If metadata options is non-nil, but this parameter + // is not specified, the default state is "disabled". + // +kubebuilder:default=disabled + // +kubebuilder:validation:Enum:={enabled,disabled} + // +optional + HTTPProtocolIPv6 *string `json:"httpProtocolIPv6,omitempty"` + // HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + // instance metadata requests. The larger the number, the further instance + // metadata requests can travel. Possible values are integers from 1 to 64. + // If metadata options is non-nil, but this parameter is not specified, the + // default value is 2. + // +kubebuilder:default=2 + // +kubebuilder:validation:Minimum:=1 + // +kubebuilder:validation:Maximum:=64 + // +optional + HTTPPutResponseHopLimit *int64 `json:"httpPutResponseHopLimit,omitempty"` + // HTTPTokens determines the state of token usage for instance metadata + // requests. If metadata options is non-nil, but this parameter is not + // specified, the default state is "required". + // + // If the state is optional, one can choose to retrieve instance metadata with + // or without a signed token header on the request. If one retrieves the IAM + // role credentials without a token, the version 1.0 role credentials are + // returned. If one retrieves the IAM role credentials using a valid signed + // token, the version 2.0 role credentials are returned. + // + // If the state is "required", one must send a signed token header with any + // instance metadata retrieval requests. In this state, retrieving the IAM + // role credentials always returns the version 2.0 credentials; the version + // 1.0 credentials are not available. + // +kubebuilder:default=required + // +kubebuilder:validation:Enum:={required,optional} + // +optional + HTTPTokens *string `json:"httpTokens,omitempty"` +} + +type BlockDeviceMapping struct { + // The device name (for example, /dev/sdh or xvdh). + // +required + DeviceName *string `json:"deviceName,omitempty"` + // EBS contains parameters used to automatically set up EBS volumes when an instance is launched. + // +kubebuilder:validation:XValidation:message="snapshotID or volumeSize must be defined",rule="has(self.snapshotID) || has(self.volumeSize)" + // +required + EBS *BlockDevice `json:"ebs,omitempty"` + // RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + // configure at most one root volume in BlockDeviceMappings. + RootVolume bool `json:"rootVolume,omitempty"` +} + +type BlockDevice struct { + // DeleteOnTermination indicates whether the EBS volume is deleted on instance termination. + // +optional + DeleteOnTermination *bool `json:"deleteOnTermination,omitempty"` + // Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + // be attached to instances that support Amazon EBS encryption. If you are creating + // a volume from a snapshot, you can't specify an encryption value. + // +optional + Encrypted *bool `json:"encrypted,omitempty"` + // IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + // this represents the number of IOPS that are provisioned for the volume. For + // gp2 volumes, this represents the baseline performance of the volume and the + // rate at which the volume accumulates I/O credits for bursting. + // + // The following are the supported values for each volume type: + // + // * gp3: 3,000-16,000 IOPS + // + // * io1: 100-64,000 IOPS + // + // * io2: 100-64,000 IOPS + // + // For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + // on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + // Other instance families guarantee performance up to 32,000 IOPS. + // + // This parameter is supported for io1, io2, and gp3 volumes only. This parameter + // is not supported for gp2, st1, sc1, or standard volumes. + // +optional + IOPS *int64 `json:"iops,omitempty"` + // KMSKeyID (ARN) of the symmetric Key Management Service (KMS) CMK used for encryption. + // +optional + KMSKeyID *string `json:"kmsKeyID,omitempty"` + // SnapshotID is the ID of an EBS snapshot + // +optional + SnapshotID *string `json:"snapshotID,omitempty"` + // Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + // Valid Range: Minimum value of 125. Maximum value of 1000. + // +optional + Throughput *int64 `json:"throughput,omitempty"` + // VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + // a volume size. The following are the supported volumes sizes for each volume + // type: + // + // * gp2 and gp3: 1-16,384 + // + // * io1 and io2: 4-16,384 + // + // * st1 and sc1: 125-16,384 + // + // * standard: 1-1,024 + // + TODO: Add the CEL resources.quantity type after k8s 1.29 + // + https://github.com/kubernetes/apiserver/commit/b137c256373aec1c5d5810afbabb8932a19ecd2a#diff-838176caa5882465c9d6061febd456397a3e2b40fb423ed36f0cabb1847ecb4dR190 + // +kubebuilder:validation:Pattern:="^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$" + // +kubebuilder:validation:Schemaless + // +kubebuilder:validation:Type:=string + // +optional + VolumeSize *resource.Quantity `json:"volumeSize,omitempty" hash:"string"` + // VolumeType of the block device. + // For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + // in the Amazon Elastic Compute Cloud User Guide. + // +kubebuilder:validation:Enum:={standard,io1,io2,gp2,sc1,st1,gp3} + // +optional + VolumeType *string `json:"volumeType,omitempty"` +} + +// InstanceStorePolicy enumerates options for configuring instance store disks. +// +kubebuilder:validation:Enum={RAID0} +type InstanceStorePolicy string + +const ( + // InstanceStorePolicyRAID0 configures a RAID-0 array that includes all ephemeral NVMe instance storage disks. + // The containerd and kubelet state directories (`/var/lib/containerd` and `/var/lib/kubelet`) will then use the + // ephemeral storage for more and faster node ephemeral-storage. The node's ephemeral storage can be shared among + // pods that request ephemeral storage and container images that are downloaded to the node. + InstanceStorePolicyRAID0 InstanceStorePolicy = "RAID0" +) + +// EC2NodeClass is the Schema for the EC2NodeClass API +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=ec2nodeclasses,scope=Cluster,categories=karpenter,shortName={ec2nc,ec2ncs} +// +kubebuilder:subresource:status +type EC2NodeClass struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // +kubebuilder:validation:XValidation:message="amiSelectorTerms is required when amiFamily == 'Custom'",rule="self.amiFamily == 'Custom' ? self.amiSelectorTerms.size() != 0 : true" + // +kubebuilder:validation:XValidation:message="must specify exactly one of ['role', 'instanceProfile']",rule="(has(self.role) && !has(self.instanceProfile)) || (!has(self.role) && has(self.instanceProfile))" + // +kubebuilder:validation:XValidation:message="changing from 'instanceProfile' to 'role' is not supported. You must delete and recreate this node class if you want to change this.",rule="(has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) && has(self.instanceProfile))" + Spec EC2NodeClassSpec `json:"spec,omitempty"` + Status EC2NodeClassStatus `json:"status,omitempty"` +} + +// We need to bump the EC2NodeClassHashVersion when we make an update to the EC2NodeClass CRD under these conditions: +// 1. A field changes its default value for an existing field that is already hashed +// 2. A field is added to the hash calculation with an already-set value +// 3. A field is removed from the hash calculations +const EC2NodeClassHashVersion = "v2" + +func (in *EC2NodeClass) Hash() string { + return fmt.Sprint(lo.Must(hashstructure.Hash(in.Spec, hashstructure.FormatV2, &hashstructure.HashOptions{ + SlicesAsSets: true, + IgnoreZeroValue: true, + ZeroNil: true, + }))) +} + +func (in *EC2NodeClass) InstanceProfileName(clusterName, region string) string { + return fmt.Sprintf("%s_%d", clusterName, lo.Must(hashstructure.Hash(fmt.Sprintf("%s%s", region, in.Name), hashstructure.FormatV2, nil))) +} + +func (in *EC2NodeClass) InstanceProfileRole() string { + return in.Spec.Role +} + +func (in *EC2NodeClass) InstanceProfileTags(clusterName string) map[string]string { + return lo.Assign(in.Spec.Tags, map[string]string{ + fmt.Sprintf("kubernetes.io/cluster/%s", clusterName): "owned", + corev1beta1.ManagedByAnnotationKey: clusterName, + LabelNodeClass: in.Name, + }) +} + +// EC2NodeClassList contains a list of EC2NodeClass +// +kubebuilder:object:root=true +type EC2NodeClassList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []EC2NodeClass `json:"items"` +} diff --git a/pkg/apis/v1/ec2nodeclass_defaults.go b/pkg/apis/v1/ec2nodeclass_defaults.go new file mode 100644 index 000000000000..8820e836bd0e --- /dev/null +++ b/pkg/apis/v1/ec2nodeclass_defaults.go @@ -0,0 +1,22 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" +) + +// SetDefaults for the EC2NodeClass +func (in *EC2NodeClass) SetDefaults(_ context.Context) {} diff --git a/pkg/apis/v1/ec2nodeclass_hash_test.go b/pkg/apis/v1/ec2nodeclass_hash_test.go new file mode 100644 index 000000000000..afed0631aaed --- /dev/null +++ b/pkg/apis/v1/ec2nodeclass_hash_test.go @@ -0,0 +1,212 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1_test + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/imdario/mergo" + "github.com/samber/lo" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/karpenter/pkg/test" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Hash", func() { + const staticHash = "10790156025840984195" + var nodeClass *v1.EC2NodeClass + BeforeEach(func() { + nodeClass = &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}), + Spec: v1.EC2NodeClassSpec{ + AMIFamily: lo.ToPtr(v1.AMIFamilyAL2023), + Role: "role-1", + Tags: map[string]string{ + "keyTag-1": "valueTag-1", + "keyTag-2": "valueTag-2", + }, + Context: lo.ToPtr("fake-context"), + DetailedMonitoring: lo.ToPtr(false), + AssociatePublicIPAddress: lo.ToPtr(false), + MetadataOptions: &v1.MetadataOptions{ + HTTPEndpoint: lo.ToPtr("disabled"), + HTTPProtocolIPv6: lo.ToPtr("disabled"), + HTTPPutResponseHopLimit: lo.ToPtr(int64(1)), + HTTPTokens: lo.ToPtr("optional"), + }, + BlockDeviceMappings: []*v1.BlockDeviceMapping{ + { + DeviceName: lo.ToPtr("map-device-1"), + RootVolume: false, + EBS: &v1.BlockDevice{ + DeleteOnTermination: lo.ToPtr(false), + Encrypted: lo.ToPtr(false), + IOPS: lo.ToPtr(int64(0)), + KMSKeyID: lo.ToPtr("fakeKMSKeyID"), + SnapshotID: lo.ToPtr("fakeSnapshot"), + Throughput: lo.ToPtr(int64(0)), + VolumeSize: resource.NewScaledQuantity(2, resource.Giga), + VolumeType: lo.ToPtr("standard"), + }, + }, + { + DeviceName: lo.ToPtr("map-device-2"), + }, + }, + UserData: aws.String("userdata-test-1"), + }, + } + }) + DescribeTable( + "should match static hash on field value change", + func(hash string, changes v1.EC2NodeClass) { + Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride, mergo.WithSliceDeepCopy)).To(Succeed()) + Expect(nodeClass.Hash()).To(Equal(hash)) + }, + Entry("Base EC2NodeClass", staticHash, v1.EC2NodeClass{}), + // Static fields, expect changed hash from base + Entry("UserData", "18317182711135792962", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{UserData: aws.String("userdata-test-2")}}), + Entry("Tags", "7254882043893135054", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), + Entry("Context", "17271601354348855032", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Context: aws.String("context-2")}}), + Entry("DetailedMonitoring", "3320998103335094348", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), + Entry("AMIFamily", "11029247967399146065", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{AMIFamily: aws.String(v1.AMIFamilyBottlerocket)}}), + Entry("InstanceStorePolicy", "15591048753403695860", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{InstanceStorePolicy: lo.ToPtr(v1.InstanceStorePolicyRAID0)}}), + Entry("AssociatePublicIPAddress", "8788624850560996180", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{AssociatePublicIPAddress: lo.ToPtr(true)}}), + Entry("MetadataOptions HTTPEndpoint", "12130088184516131939", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPEndpoint: lo.ToPtr("enabled")}}}), + Entry("MetadataOptions HTTPProtocolIPv6", "9851778617676567202", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPProtocolIPv6: lo.ToPtr("enabled")}}}), + Entry("MetadataOptions HTTPPutResponseHopLimit", "10114972825726256442", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPPutResponseHopLimit: lo.ToPtr(int64(10))}}}), + Entry("MetadataOptions HTTPTokens", "15328515228245883488", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPTokens: lo.ToPtr("required")}}}), + Entry("BlockDeviceMapping DeviceName", "14855383487702710824", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{DeviceName: lo.ToPtr("map-device-test-3")}}}}), + Entry("BlockDeviceMapping RootVolume", "9591488558660758449", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{RootVolume: true}}}}), + Entry("BlockDeviceMapping DeleteOnTermination", "2802222466202766732", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{DeleteOnTermination: lo.ToPtr(true)}}}}}), + Entry("BlockDeviceMapping Encrypted", "16743053872042184219", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{Encrypted: lo.ToPtr(true)}}}}}), + Entry("BlockDeviceMapping IOPS", "17284705682110195253", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{IOPS: lo.ToPtr(int64(10))}}}}}), + Entry("BlockDeviceMapping KMSKeyID", "9151019926310241707", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{KMSKeyID: lo.ToPtr("test")}}}}}), + Entry("BlockDeviceMapping SnapshotID", "5250341140179985875", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{SnapshotID: lo.ToPtr("test")}}}}}), + Entry("BlockDeviceMapping Throughput", "16711481758638864953", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{Throughput: lo.ToPtr(int64(10))}}}}}), + Entry("BlockDeviceMapping VolumeType", "488614640133725370", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{VolumeType: lo.ToPtr("io1")}}}}}), + + // Behavior / Dynamic fields, expect same hash as base + Entry("Modified AMISelector", staticHash, v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{AMISelectorTerms: []v1.AMISelectorTerm{{Tags: map[string]string{"ami-test-key": "ami-test-value"}}}}}), + Entry("Modified SubnetSelector", staticHash, v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{SubnetSelectorTerms: []v1.SubnetSelectorTerm{{Tags: map[string]string{"subnet-test-key": "subnet-test-value"}}}}}), + Entry("Modified SecurityGroupSelector", staticHash, v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{SecurityGroupSelectorTerms: []v1.SecurityGroupSelectorTerm{{Tags: map[string]string{"security-group-test-key": "security-group-test-value"}}}}}), + ) + // We create a separate test for updating blockDeviceMapping volumeSize, since resource.Quantity is a struct, and mergo.WithSliceDeepCopy + // doesn't work well with unexported fields, like the ones that are present in resource.Quantity + It("should match static hash when updating blockDeviceMapping volumeSize", func() { + nodeClass.Spec.BlockDeviceMappings[0].EBS.VolumeSize = resource.NewScaledQuantity(10, resource.Giga) + Expect(nodeClass.Hash()).To(Equal("4802236799448001710")) + }) + It("should match static hash for instanceProfile", func() { + nodeClass.Spec.Role = "" + nodeClass.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") + Expect(nodeClass.Hash()).To(Equal("7914642030762404205")) + }) + It("should match static hash when reordering tags", func() { + nodeClass.Spec.Tags = map[string]string{"keyTag-2": "valueTag-2", "keyTag-1": "valueTag-1"} + Expect(nodeClass.Hash()).To(Equal(staticHash)) + }) + It("should match static hash when reordering blockDeviceMappings", func() { + nodeClass.Spec.BlockDeviceMappings[0], nodeClass.Spec.BlockDeviceMappings[1] = nodeClass.Spec.BlockDeviceMappings[1], nodeClass.Spec.BlockDeviceMappings[0] + Expect(nodeClass.Hash()).To(Equal(staticHash)) + }) + DescribeTable("should change hash when static fields are updated", func(changes v1.EC2NodeClass) { + hash := nodeClass.Hash() + Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride, mergo.WithSliceDeepCopy)).To(Succeed()) + updatedHash := nodeClass.Hash() + Expect(hash).ToNot(Equal(updatedHash)) + }, + Entry("UserData", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{UserData: aws.String("userdata-test-2")}}), + Entry("Tags", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), + Entry("Context", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{Context: aws.String("context-2")}}), + Entry("DetailedMonitoring", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), + Entry("AMIFamily", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{AMIFamily: aws.String(v1.AMIFamilyBottlerocket)}}), + Entry("InstanceStorePolicy", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{InstanceStorePolicy: lo.ToPtr(v1.InstanceStorePolicyRAID0)}}), + Entry("AssociatePublicIPAddress", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{AssociatePublicIPAddress: lo.ToPtr(true)}}), + Entry("MetadataOptions HTTPEndpoint", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPEndpoint: lo.ToPtr("enabled")}}}), + Entry("MetadataOptions HTTPProtocolIPv6", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPProtocolIPv6: lo.ToPtr("enabled")}}}), + Entry("MetadataOptions HTTPPutResponseHopLimit", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPPutResponseHopLimit: lo.ToPtr(int64(10))}}}), + Entry("MetadataOptions HTTPTokens", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{MetadataOptions: &v1.MetadataOptions{HTTPTokens: lo.ToPtr("required")}}}), + Entry("BlockDeviceMapping DeviceName", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{DeviceName: lo.ToPtr("map-device-test-3")}}}}), + Entry("BlockDeviceMapping RootVolume", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{RootVolume: true}}}}), + Entry("BlockDeviceMapping DeleteOnTermination", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{DeleteOnTermination: lo.ToPtr(true)}}}}}), + Entry("BlockDeviceMapping Encrypted", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{Encrypted: lo.ToPtr(true)}}}}}), + Entry("BlockDeviceMapping IOPS", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{IOPS: lo.ToPtr(int64(10))}}}}}), + Entry("BlockDeviceMapping KMSKeyID", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{KMSKeyID: lo.ToPtr("test")}}}}}), + Entry("BlockDeviceMapping SnapshotID", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{SnapshotID: lo.ToPtr("test")}}}}}), + Entry("BlockDeviceMapping Throughput", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{Throughput: lo.ToPtr(int64(10))}}}}}), + Entry("BlockDeviceMapping VolumeType", v1.EC2NodeClass{Spec: v1.EC2NodeClassSpec{BlockDeviceMappings: []*v1.BlockDeviceMapping{{EBS: &v1.BlockDevice{VolumeType: lo.ToPtr("io1")}}}}}), + ) + // We create a separate test for updating blockDeviceMapping volumeSize, since resource.Quantity is a struct, and mergo.WithSliceDeepCopy + // doesn't work well with unexported fields, like the ones that are present in resource.Quantity + It("should change hash blockDeviceMapping volumeSize is updated", func() { + hash := nodeClass.Hash() + nodeClass.Spec.BlockDeviceMappings[0].EBS.VolumeSize = resource.NewScaledQuantity(10, resource.Giga) + updatedHash := nodeClass.Hash() + Expect(hash).ToNot(Equal(updatedHash)) + }) + It("should change hash when instanceProfile is updated", func() { + nodeClass.Spec.Role = "" + nodeClass.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") + hash := nodeClass.Hash() + nodeClass.Spec.InstanceProfile = lo.ToPtr("other-instance-profile") + updatedHash := nodeClass.Hash() + Expect(hash).ToNot(Equal(updatedHash)) + }) + It("should not change hash when tags are re-ordered", func() { + hash := nodeClass.Hash() + nodeClass.Spec.Tags = map[string]string{"keyTag-2": "valueTag-2", "keyTag-1": "valueTag-1"} + updatedHash := nodeClass.Hash() + Expect(hash).To(Equal(updatedHash)) + }) + It("should not change hash when blockDeviceMappings are re-ordered", func() { + hash := nodeClass.Hash() + nodeClass.Spec.BlockDeviceMappings[0], nodeClass.Spec.BlockDeviceMappings[1] = nodeClass.Spec.BlockDeviceMappings[1], nodeClass.Spec.BlockDeviceMappings[0] + updatedHash := nodeClass.Hash() + Expect(hash).To(Equal(updatedHash)) + }) + It("should not change hash when behavior/dynamic fields are updated", func() { + hash := nodeClass.Hash() + + // Update a behavior/dynamic field + nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{"subnet-test-key": "subnet-test-value"}, + }, + } + nodeClass.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{"sg-test-key": "sg-test-value"}, + }, + } + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Tags: map[string]string{"ami-test-key": "ami-test-value"}, + }, + } + updatedHash := nodeClass.Hash() + Expect(hash).To(Equal(updatedHash)) + }) + It("should expect two EC2NodeClasses with the same spec to have the same hash", func() { + otherNodeClass := &v1.EC2NodeClass{ + Spec: nodeClass.Spec, + } + Expect(nodeClass.Hash()).To(Equal(otherNodeClass.Hash())) + }) +}) diff --git a/pkg/apis/v1/ec2nodeclass_status.go b/pkg/apis/v1/ec2nodeclass_status.go new file mode 100644 index 000000000000..8071bac7a1c1 --- /dev/null +++ b/pkg/apis/v1/ec2nodeclass_status.go @@ -0,0 +1,92 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + op "github.com/awslabs/operatorpkg/status" + v1 "k8s.io/api/core/v1" +) + +// Subnet contains resolved Subnet selector values utilized for node launch +type Subnet struct { + // ID of the subnet + // +required + ID string `json:"id"` + // The associated availability zone + // +required + Zone string `json:"zone"` +} + +// SecurityGroup contains resolved SecurityGroup selector values utilized for node launch +type SecurityGroup struct { + // ID of the security group + // +required + ID string `json:"id"` + // Name of the security group + // +optional + Name string `json:"name,omitempty"` +} + +// AMI contains resolved AMI selector values utilized for node launch +type AMI struct { + // ID of the AMI + // +required + ID string `json:"id"` + // Name of the AMI + // +optional + Name string `json:"name,omitempty"` + // Requirements of the AMI to be utilized on an instance type + // +required + Requirements []v1.NodeSelectorRequirement `json:"requirements"` +} + +// EC2NodeClassStatus contains the resolved state of the EC2NodeClass +type EC2NodeClassStatus struct { + // Subnets contains the current Subnet values that are available to the + // cluster under the subnet selectors. + // +optional + Subnets []Subnet `json:"subnets,omitempty"` + // SecurityGroups contains the current Security Groups values that are available to the + // cluster under the SecurityGroups selectors. + // +optional + SecurityGroups []SecurityGroup `json:"securityGroups,omitempty"` + // AMI contains the current AMI values that are available to the + // cluster under the AMI selectors. + // +optional + AMIs []AMI `json:"amis,omitempty"` + // InstanceProfile contains the resolved instance profile for the role + // +optional + InstanceProfile string `json:"instanceProfile,omitempty"` + // Conditions contains signals for health and readiness + // +optional + Conditions []op.Condition `json:"conditions,omitempty"` +} + +const ( + // ConditionTypeNodeClassReady = "Ready" condition indicates that subnets, security groups, AMIs and instance profile for nodeClass were resolved + ConditionTypeNodeClassReady = "Ready" +) + +func (in *EC2NodeClass) StatusConditions() op.ConditionSet { + return op.NewReadyConditions(ConditionTypeNodeClassReady).For(in) +} + +func (in *EC2NodeClass) GetConditions() []op.Condition { + return in.Status.Conditions +} + +func (in *EC2NodeClass) SetConditions(conditions []op.Condition) { + in.Status.Conditions = conditions +} diff --git a/pkg/apis/v1/ec2nodeclass_validation.go b/pkg/apis/v1/ec2nodeclass_validation.go new file mode 100644 index 000000000000..f0e7dcc6e5cc --- /dev/null +++ b/pkg/apis/v1/ec2nodeclass_validation.go @@ -0,0 +1,307 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/service/ec2" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + "k8s.io/apimachinery/pkg/api/resource" + "knative.dev/pkg/apis" +) + +const ( + subnetSelectorTermsPath = "subnetSelectorTerms" + securityGroupSelectorTermsPath = "securityGroupSelectorTerms" + amiSelectorTermsPath = "amiSelectorTerms" + amiFamilyPath = "amiFamily" + tagsPath = "tags" + metadataOptionsPath = "metadataOptions" + blockDeviceMappingsPath = "blockDeviceMappings" + rolePath = "role" + instanceProfilePath = "instanceProfile" +) + +var ( + minVolumeSize = *resource.NewScaledQuantity(1, resource.Giga) + maxVolumeSize = *resource.NewScaledQuantity(64, resource.Tera) +) + +func (in *EC2NodeClass) SupportedVerbs() []admissionregistrationv1.OperationType { + return []admissionregistrationv1.OperationType{ + admissionregistrationv1.Create, + admissionregistrationv1.Update, + } +} + +func (in *EC2NodeClass) Validate(ctx context.Context) (errs *apis.FieldError) { + if apis.IsInUpdate(ctx) { + original := apis.GetBaseline(ctx).(*EC2NodeClass) + errs = in.validateImmutableFields(original) + } + return errs.Also( + apis.ValidateObjectMetadata(in).ViaField("metadata"), + in.Spec.validate(ctx).ViaField("spec"), + ) +} + +func (in *EC2NodeClass) validateImmutableFields(original *EC2NodeClass) (errs *apis.FieldError) { + return errs.Also( + in.Spec.validateRoleImmutability(&original.Spec).ViaField("spec"), + ) +} + +func (in *EC2NodeClassSpec) validate(_ context.Context) (errs *apis.FieldError) { + if in.Role != "" && in.InstanceProfile != nil { + errs = errs.Also(apis.ErrMultipleOneOf(rolePath, instanceProfilePath)) + } + if in.Role == "" && in.InstanceProfile == nil { + errs = errs.Also(apis.ErrMissingOneOf(rolePath, instanceProfilePath)) + } + return errs.Also( + in.validateSubnetSelectorTerms().ViaField(subnetSelectorTermsPath), + in.validateSecurityGroupSelectorTerms().ViaField(securityGroupSelectorTermsPath), + in.validateAMISelectorTerms().ViaField(amiSelectorTermsPath), + in.validateMetadataOptions().ViaField(metadataOptionsPath), + in.validateAMIFamily().ViaField(amiFamilyPath), + in.validateBlockDeviceMappings().ViaField(blockDeviceMappingsPath), + in.validateTags().ViaField(tagsPath), + ) +} + +func (in *EC2NodeClassSpec) validateSubnetSelectorTerms() (errs *apis.FieldError) { + if len(in.SubnetSelectorTerms) == 0 { + errs = errs.Also(apis.ErrMissingOneOf()) + } + for i, term := range in.SubnetSelectorTerms { + errs = errs.Also(term.validate()).ViaIndex(i) + } + return errs +} + +func (in *SubnetSelectorTerm) validate() (errs *apis.FieldError) { + errs = errs.Also(validateTags(in.Tags).ViaField("tags")) + if len(in.Tags) == 0 && in.ID == "" { + errs = errs.Also(apis.ErrGeneric("expected at least one, got none", "tags", "id")) + } else if in.ID != "" && len(in.Tags) > 0 { + errs = errs.Also(apis.ErrGeneric(`"id" is mutually exclusive, cannot be set with a combination of other fields in`)) + } + return errs +} + +func (in *EC2NodeClassSpec) validateSecurityGroupSelectorTerms() (errs *apis.FieldError) { + if len(in.SecurityGroupSelectorTerms) == 0 { + errs = errs.Also(apis.ErrMissingOneOf()) + } + for _, term := range in.SecurityGroupSelectorTerms { + errs = errs.Also(term.validate()) + } + return errs +} + +//nolint:gocyclo +func (in *SecurityGroupSelectorTerm) validate() (errs *apis.FieldError) { + errs = errs.Also(validateTags(in.Tags).ViaField("tags")) + if len(in.Tags) == 0 && in.ID == "" && in.Name == "" { + errs = errs.Also(apis.ErrGeneric("expect at least one, got none", "tags", "id", "name")) + } else if in.ID != "" && (len(in.Tags) > 0 || in.Name != "") { + errs = errs.Also(apis.ErrGeneric(`"id" is mutually exclusive, cannot be set with a combination of other fields in`)) + } else if in.Name != "" && (len(in.Tags) > 0 || in.ID != "") { + errs = errs.Also(apis.ErrGeneric(`"name" is mutually exclusive, cannot be set with a combination of other fields in`)) + } + return errs +} + +func (in *EC2NodeClassSpec) validateAMISelectorTerms() (errs *apis.FieldError) { + for _, term := range in.AMISelectorTerms { + errs = errs.Also(term.validate()) + } + return errs +} + +//nolint:gocyclo +func (in *AMISelectorTerm) validate() (errs *apis.FieldError) { + errs = errs.Also(validateTags(in.Tags).ViaField("tags")) + if len(in.Tags) == 0 && in.ID == "" && in.Name == "" { + errs = errs.Also(apis.ErrGeneric("expect at least one, got none", "tags", "id", "name")) + } else if in.ID != "" && (len(in.Tags) > 0 || in.Name != "" || in.Owner != "") { + errs = errs.Also(apis.ErrGeneric(`"id" is mutually exclusive, cannot be set with a combination of other fields in`)) + } + return errs +} + +func validateTags(m map[string]string) (errs *apis.FieldError) { + for k, v := range m { + if k == "" { + errs = errs.Also(apis.ErrInvalidKeyName(`""`, "")) + } + if v == "" { + errs = errs.Also(apis.ErrInvalidValue(`""`, k)) + } + } + return errs +} + +func (in *EC2NodeClassSpec) validateMetadataOptions() (errs *apis.FieldError) { + if in.MetadataOptions == nil { + return nil + } + return errs.Also( + in.validateHTTPEndpoint(), + in.validateHTTPProtocolIpv6(), + in.validateHTTPPutResponseHopLimit(), + in.validateHTTPTokens(), + ) +} + +func (in *EC2NodeClassSpec) validateHTTPEndpoint() *apis.FieldError { + if in.MetadataOptions.HTTPEndpoint == nil { + return nil + } + return in.validateStringEnum(*in.MetadataOptions.HTTPEndpoint, "httpEndpoint", ec2.LaunchTemplateInstanceMetadataEndpointState_Values()) +} + +func (in *EC2NodeClassSpec) validateHTTPProtocolIpv6() *apis.FieldError { + if in.MetadataOptions.HTTPProtocolIPv6 == nil { + return nil + } + return in.validateStringEnum(*in.MetadataOptions.HTTPProtocolIPv6, "httpProtocolIPv6", ec2.LaunchTemplateInstanceMetadataProtocolIpv6_Values()) +} + +func (in *EC2NodeClassSpec) validateHTTPPutResponseHopLimit() *apis.FieldError { + if in.MetadataOptions.HTTPPutResponseHopLimit == nil { + return nil + } + limit := *in.MetadataOptions.HTTPPutResponseHopLimit + if limit < 1 || limit > 64 { + return apis.ErrOutOfBoundsValue(limit, 1, 64, "httpPutResponseHopLimit") + } + return nil +} + +func (in *EC2NodeClassSpec) validateHTTPTokens() *apis.FieldError { + if in.MetadataOptions.HTTPTokens == nil { + return nil + } + return in.validateStringEnum(*in.MetadataOptions.HTTPTokens, "httpTokens", ec2.LaunchTemplateHttpTokensState_Values()) +} + +func (in *EC2NodeClassSpec) validateStringEnum(value, field string, validValues []string) *apis.FieldError { + for _, validValue := range validValues { + if value == validValue { + return nil + } + } + return apis.ErrInvalidValue(fmt.Sprintf("%s not in %v", value, strings.Join(validValues, ", ")), field) +} + +func (in *EC2NodeClassSpec) validateBlockDeviceMappings() (errs *apis.FieldError) { + numRootVolume := 0 + for i, blockDeviceMapping := range in.BlockDeviceMappings { + if err := in.validateBlockDeviceMapping(blockDeviceMapping); err != nil { + errs = errs.Also(err.ViaFieldIndex(blockDeviceMappingsPath, i)) + } + if blockDeviceMapping.RootVolume { + numRootVolume++ + } + } + if numRootVolume > 1 { + errs = errs.Also(apis.ErrMultipleOneOf("more than 1 root volume configured")) + } + return errs +} + +func (in *EC2NodeClassSpec) validateBlockDeviceMapping(blockDeviceMapping *BlockDeviceMapping) (errs *apis.FieldError) { + return errs.Also(in.validateDeviceName(blockDeviceMapping), in.validateEBS(blockDeviceMapping)) +} + +func (in *EC2NodeClassSpec) validateDeviceName(blockDeviceMapping *BlockDeviceMapping) *apis.FieldError { + if blockDeviceMapping.DeviceName == nil { + return apis.ErrMissingField("deviceName") + } + return nil +} + +func (in *EC2NodeClassSpec) validateEBS(blockDeviceMapping *BlockDeviceMapping) (errs *apis.FieldError) { + if blockDeviceMapping.EBS == nil { + return apis.ErrMissingField("ebs") + } + for _, err := range []*apis.FieldError{ + in.validateVolumeType(blockDeviceMapping), + in.validateVolumeSize(blockDeviceMapping), + } { + if err != nil { + errs = errs.Also(err.ViaField("ebs")) + } + } + return errs +} + +func (in *EC2NodeClassSpec) validateVolumeType(blockDeviceMapping *BlockDeviceMapping) *apis.FieldError { + if blockDeviceMapping.EBS.VolumeType != nil { + return in.validateStringEnum(*blockDeviceMapping.EBS.VolumeType, "volumeType", ec2.VolumeType_Values()) + } + return nil +} + +func (in *EC2NodeClassSpec) validateVolumeSize(blockDeviceMapping *BlockDeviceMapping) *apis.FieldError { + // If an EBS mapping is present, one of volumeSize or snapshotID must be present + if blockDeviceMapping.EBS.SnapshotID != nil && blockDeviceMapping.EBS.VolumeSize == nil { + return nil + } else if blockDeviceMapping.EBS.VolumeSize == nil { + return apis.ErrMissingField("volumeSize") + } else if blockDeviceMapping.EBS.VolumeSize.Cmp(minVolumeSize) == -1 || blockDeviceMapping.EBS.VolumeSize.Cmp(maxVolumeSize) == 1 { + return apis.ErrOutOfBoundsValue(blockDeviceMapping.EBS.VolumeSize.String(), minVolumeSize.String(), maxVolumeSize.String(), "volumeSize") + } + return nil +} + +func (in *EC2NodeClassSpec) validateAMIFamily() (errs *apis.FieldError) { + if in.AMIFamily == nil { + return nil + } + if *in.AMIFamily == AMIFamilyCustom && len(in.AMISelectorTerms) == 0 { + errs = errs.Also(apis.ErrMissingField(amiSelectorTermsPath)) + } + return errs +} + +func (in *EC2NodeClassSpec) validateTags() (errs *apis.FieldError) { + for k, v := range in.Tags { + if k == "" { + errs = errs.Also(apis.ErrInvalidValue(fmt.Sprintf( + "the tag with key : '' and value : '%s' is invalid because empty tag keys aren't supported", v), "tags")) + } + for _, pattern := range RestrictedTagPatterns { + if pattern.MatchString(k) { + errs = errs.Also(apis.ErrInvalidKeyName(k, "tags", fmt.Sprintf("tag contains in restricted tag matching %q", pattern.String()))) + } + } + } + return errs +} + +func (in *EC2NodeClassSpec) validateRoleImmutability(originalSpec *EC2NodeClassSpec) *apis.FieldError { + if in.Role != originalSpec.Role { + return &apis.FieldError{ + Message: "Immutable field changed", + Paths: []string{"role"}, + } + } + return nil +} diff --git a/pkg/apis/v1/ec2nodeclass_validation_cel_test.go b/pkg/apis/v1/ec2nodeclass_validation_cel_test.go new file mode 100644 index 000000000000..c08b75996feb --- /dev/null +++ b/pkg/apis/v1/ec2nodeclass_validation_cel_test.go @@ -0,0 +1,712 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1_test + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/samber/lo" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "sigs.k8s.io/karpenter/pkg/test" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("CEL/Validation", func() { + var nc *v1.EC2NodeClass + + BeforeEach(func() { + if env.Version.Minor() < 25 { + Skip("CEL Validation is for 1.25>") + } + nc = &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}), + Spec: v1.EC2NodeClassSpec{ + AMIFamily: lo.ToPtr(v1.AMIFamilyAL2023), + Role: "role-1", + SecurityGroupSelectorTerms: []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "*": "*", + }, + }, + }, + SubnetSelectorTerms: []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "*": "*", + }, + }, + }, + }, + } + }) + It("should succeed if just specifying role", func() { + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed if just specifying instance profile", func() { + nc.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") + nc.Spec.Role = "" + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail if specifying both instance profile and role", func() { + nc.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail if not specifying one of instance profile and role", func() { + nc.Spec.Role = "" + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + Context("UserData", func() { + It("should succeed if user data is empty", func() { + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + }) + Context("Tags", func() { + It("should succeed when tags are empty", func() { + nc.Spec.Tags = map[string]string{} + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed if tags aren't in restricted tag keys", func() { + nc.Spec.Tags = map[string]string{ + "karpenter.sh/custom-key": "value", + "karpenter.sh/managed": "true", + "kubernetes.io/role/key": "value", + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail if tags contain a restricted domain key", func() { + nc.Spec.Tags = map[string]string{ + corev1beta1.NodePoolLabelKey: "value", + } + Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + "kubernetes.io/cluster/test": "value", + } + Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + corev1beta1.ManagedByAnnotationKey: "test", + } + Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + v1.LabelNodeClass: "test", + } + Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + "karpenter.sh/nodeclaim": "test", + } + Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) + }) + }) + Context("SubnetSelectorTerms", func() { + It("should succeed with a valid subnet selector on tags", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed with a valid subnet selector on id", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + ID: "subnet-12345749", + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail when subnet selector terms is set to nil", func() { + nc.Spec.SubnetSelectorTerms = nil + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when no subnet selector terms exist", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{} + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has no values", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + {}, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has no tag map values", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{}, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has a tag map key that is empty", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "test": "", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has a tag map value that is empty", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when the last subnet selector is invalid", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + { + Tags: map[string]string{ + "test2": "testvalue2", + }, + }, + { + Tags: map[string]string{ + "test3": "testvalue3", + }, + }, + { + Tags: map[string]string{ + "": "testvalue4", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when specifying id with tags", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + ID: "subnet-12345749", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + }) + Context("SecurityGroupSelectorTerms", func() { + It("should succeed with a valid security group selector on tags", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed with a valid security group selector on id", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + ID: "sg-12345749", + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed with a valid security group selector on name", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Name: "testname", + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail when security group selector terms is set to nil", func() { + nc.Spec.SecurityGroupSelectorTerms = nil + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when no security group selector terms exist", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{} + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has no values", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + {}, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has no tag map values", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{}, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has a tag map key that is empty", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "test": "", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has a tag map value that is empty", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when the last security group selector is invalid", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + { + Tags: map[string]string{ + "test2": "testvalue2", + }, + }, + { + Tags: map[string]string{ + "test3": "testvalue3", + }, + }, + { + Tags: map[string]string{ + "": "testvalue4", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when specifying id with tags", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + ID: "sg-12345749", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when specifying id with name", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + ID: "sg-12345749", + Name: "my-security-group", + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when specifying name with tags", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Name: "my-security-group", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + }) + Context("AMISelectorTerms", func() { + It("should succeed with a valid ami selector on tags", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed with a valid ami selector on id", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + ID: "ami-12345749", + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed with a valid ami selector on name", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Name: "testname", + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed with a valid ami selector on name and owner", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Name: "testname", + Owner: "testowner", + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed when an ami selector term has an owner key with tags", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Owner: "testowner", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail when a ami selector term has no values", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + {}, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a ami selector term has no tag map values", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Tags: map[string]string{}, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a ami selector term has a tag map key that is empty", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Tags: map[string]string{ + "test": "", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a ami selector term has a tag map value that is empty", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Tags: map[string]string{ + "": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when the last ami selector is invalid", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + { + Tags: map[string]string{ + "test2": "testvalue2", + }, + }, + { + Tags: map[string]string{ + "test3": "testvalue3", + }, + }, + { + Tags: map[string]string{ + "": "testvalue4", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when specifying id with tags", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + ID: "ami-12345749", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when specifying id with name", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + ID: "ami-12345749", + Name: "my-custom-ami", + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when specifying id with owner", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + ID: "ami-12345749", + Owner: "123456789", + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when AMIFamily is Custom and not AMISelectorTerms", func() { + nc.Spec.AMIFamily = &v1.AMIFamilyCustom + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + }) + Context("MetadataOptions", func() { + It("should succeed for valid inputs", func() { + nc.Spec.MetadataOptions = &v1.MetadataOptions{ + HTTPEndpoint: aws.String("disabled"), + HTTPProtocolIPv6: aws.String("enabled"), + HTTPPutResponseHopLimit: aws.Int64(34), + HTTPTokens: aws.String("optional"), + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail for invalid for HTTPEndpoint", func() { + nc.Spec.MetadataOptions = &v1.MetadataOptions{ + HTTPEndpoint: aws.String("test"), + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail for invalid for HTTPProtocolIPv6", func() { + nc.Spec.MetadataOptions = &v1.MetadataOptions{ + HTTPProtocolIPv6: aws.String("test"), + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail for invalid for HTTPPutResponseHopLimit", func() { + nc.Spec.MetadataOptions = &v1.MetadataOptions{ + HTTPPutResponseHopLimit: aws.Int64(-5), + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail for invalid for HTTPTokens", func() { + nc.Spec.MetadataOptions = &v1.MetadataOptions{ + HTTPTokens: aws.String("test"), + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + }) + Context("BlockDeviceMappings", func() { + It("should succeed if more than one root volume is specified", func() { + nodeClass := &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}), + Spec: v1.EC2NodeClassSpec{ + AMIFamily: nc.Spec.AMIFamily, + SubnetSelectorTerms: nc.Spec.SubnetSelectorTerms, + SecurityGroupSelectorTerms: nc.Spec.SecurityGroupSelectorTerms, + Role: nc.Spec.Role, + BlockDeviceMappings: []*v1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(500, resource.Giga), + }, + + RootVolume: true, + }, + { + DeviceName: aws.String("map-device-2"), + EBS: &v1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(50, resource.Tera), + }, + + RootVolume: false, + }, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).To(Succeed()) + }) + It("should succeed for valid VolumeSize in G", func() { + nodeClass := &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}), + Spec: v1.EC2NodeClassSpec{ + AMIFamily: nc.Spec.AMIFamily, + SubnetSelectorTerms: nc.Spec.SubnetSelectorTerms, + SecurityGroupSelectorTerms: nc.Spec.SecurityGroupSelectorTerms, + Role: nc.Spec.Role, + BlockDeviceMappings: []*v1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(58, resource.Giga), + }, + RootVolume: false, + }, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).To(Succeed()) + }) + It("should succeed for valid VolumeSize in T", func() { + nodeClass := &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}), + Spec: v1.EC2NodeClassSpec{ + AMIFamily: nc.Spec.AMIFamily, + SubnetSelectorTerms: nc.Spec.SubnetSelectorTerms, + SecurityGroupSelectorTerms: nc.Spec.SecurityGroupSelectorTerms, + Role: nc.Spec.Role, + BlockDeviceMappings: []*v1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(45, resource.Tera), + }, + RootVolume: false, + }, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).To(Succeed()) + }) + It("should fail if more than one root volume is specified", func() { + nodeClass := &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}), + Spec: v1.EC2NodeClassSpec{ + AMIFamily: nc.Spec.AMIFamily, + SubnetSelectorTerms: nc.Spec.SubnetSelectorTerms, + SecurityGroupSelectorTerms: nc.Spec.SecurityGroupSelectorTerms, + Role: nc.Spec.Role, + BlockDeviceMappings: []*v1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(50, resource.Giga), + }, + RootVolume: true, + }, + { + DeviceName: aws.String("map-device-2"), + EBS: &v1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(50, resource.Giga), + }, + RootVolume: true, + }, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed())) + }) + It("should fail VolumeSize is less then 1Gi/1G", func() { + nodeClass := &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}), + Spec: v1.EC2NodeClassSpec{ + AMIFamily: nc.Spec.AMIFamily, + SubnetSelectorTerms: nc.Spec.SubnetSelectorTerms, + SecurityGroupSelectorTerms: nc.Spec.SecurityGroupSelectorTerms, + Role: nc.Spec.Role, + BlockDeviceMappings: []*v1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(1, resource.Milli), + }, + RootVolume: false, + }, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed())) + }) + It("should fail VolumeSize is greater then 64T", func() { + nodeClass := &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}), + Spec: v1.EC2NodeClassSpec{ + AMIFamily: nc.Spec.AMIFamily, + SubnetSelectorTerms: nc.Spec.SubnetSelectorTerms, + SecurityGroupSelectorTerms: nc.Spec.SecurityGroupSelectorTerms, + Role: nc.Spec.Role, + BlockDeviceMappings: []*v1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(100, resource.Tera), + }, + RootVolume: false, + }, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed())) + }) + It("should fail for VolumeSize that do not parse into quantity values", func() { + nodeClass := &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}), + Spec: v1.EC2NodeClassSpec{ + AMIFamily: nc.Spec.AMIFamily, + SubnetSelectorTerms: nc.Spec.SubnetSelectorTerms, + SecurityGroupSelectorTerms: nc.Spec.SecurityGroupSelectorTerms, + Role: nc.Spec.Role, + BlockDeviceMappings: []*v1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1.BlockDevice{ + VolumeSize: &resource.Quantity{}, + }, + RootVolume: false, + }, + }, + }, + } + Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed())) + }) + }) + Context("Role Immutability", func() { + It("should fail if role is not defined", func() { + nc.Spec.Role = "" + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when updating the role", func() { + nc.Spec.Role = "test-role" + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + + nc.Spec.Role = "test-role2" + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail to switch between an unmanaged and managed instance profile", func() { + nc.Spec.Role = "" + nc.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + + nc.Spec.Role = "test-role" + nc.Spec.InstanceProfile = nil + Expect(env.Client.Update(ctx, nc)).ToNot(Succeed()) + }) + It("should fail to switch between a managed and unmanaged instance profile", func() { + nc.Spec.Role = "test-role" + nc.Spec.InstanceProfile = nil + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + + nc.Spec.Role = "" + nc.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") + Expect(env.Client.Update(ctx, nc)).ToNot(Succeed()) + }) + }) +}) diff --git a/pkg/apis/v1/ec2nodeclass_validation_webhook_test.go b/pkg/apis/v1/ec2nodeclass_validation_webhook_test.go new file mode 100644 index 000000000000..59f388d3dffc --- /dev/null +++ b/pkg/apis/v1/ec2nodeclass_validation_webhook_test.go @@ -0,0 +1,546 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1_test + +import ( + "github.com/samber/lo" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "knative.dev/pkg/apis" + "sigs.k8s.io/karpenter/pkg/test" + + "github.com/aws/aws-sdk-go/aws" + + v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Webhook/Validation", func() { + var nc *v1.EC2NodeClass + + BeforeEach(func() { + nc = &v1.EC2NodeClass{ + ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}), + Spec: v1.EC2NodeClassSpec{ + AMIFamily: lo.ToPtr(v1.AMIFamilyAL2023), + Role: "role-1", + SecurityGroupSelectorTerms: []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "*": "*", + }, + }, + }, + SubnetSelectorTerms: []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "*": "*", + }, + }, + }, + }, + } + }) + It("should succeed if just specifying role", func() { + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should succeed if just specifying instance profile", func() { + nc.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") + nc.Spec.Role = "" + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should fail if specifying both instance profile and role", func() { + nc.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail if not specifying one of instance profile and role", func() { + nc.Spec.Role = "" + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + Context("UserData", func() { + It("should succeed if user data is empty", func() { + Expect(nc.Validate(ctx)).To(Succeed()) + }) + }) + Context("Tags", func() { + It("should succeed when tags are empty", func() { + nc.Spec.Tags = map[string]string{} + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should succeed if tags aren't in restricted tag keys", func() { + nc.Spec.Tags = map[string]string{ + "karpenter.sh/custom-key": "value", + "karpenter.sh/managed": "true", + "kubernetes.io/role/key": "value", + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should succeed by validating that regex is properly escaped", func() { + nc.Spec.Tags = map[string]string{ + "karpenterzsh/nodepool": "value", + } + Expect(nc.Validate(ctx)).To(Succeed()) + nc.Spec.Tags = map[string]string{ + "kubernetesbio/cluster/test": "value", + } + Expect(nc.Validate(ctx)).To(Succeed()) + nc.Spec.Tags = map[string]string{ + "karpenterzsh/managed-by": "test", + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should fail if tags contain a restricted domain key", func() { + nc.Spec.Tags = map[string]string{ + "karpenter.sh/nodepool": "value", + } + Expect(nc.Validate(ctx)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + "kubernetes.io/cluster/test": "value", + } + Expect(nc.Validate(ctx)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + "karpenter.sh/managed-by": "test", + } + Expect(nc.Validate(ctx)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + v1.LabelNodeClass: "test", + } + Expect(nc.Validate(ctx)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + "karpenter.sh/nodeclaim": "test", + } + Expect(nc.Validate(ctx)).To(Not(Succeed())) + }) + }) + Context("SubnetSelectorTerms", func() { + It("should succeed with a valid subnet selector on tags", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should succeed with a valid subnet selector on id", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + ID: "subnet-12345749", + }, + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should fail when subnet selector terms is set to nil", func() { + nc.Spec.SubnetSelectorTerms = nil + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when no subnet selector terms exist", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{} + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has no values", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + {}, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has no tag map values", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{}, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has a tag map key that is empty", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "test": "", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has a tag map value that is empty", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when the last subnet selector is invalid", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + { + Tags: map[string]string{ + "test2": "testvalue2", + }, + }, + { + Tags: map[string]string{ + "test3": "testvalue3", + }, + }, + { + Tags: map[string]string{ + "": "testvalue4", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when specifying id with tags", func() { + nc.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{ + { + ID: "subnet-12345749", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + }) + Context("SecurityGroupSelectorTerms", func() { + It("should succeed with a valid security group selector on tags", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should succeed with a valid security group selector on id", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + ID: "sg-12345749", + }, + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should succeed with a valid security group selector on name", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Name: "testname", + }, + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should fail when security group selector terms is set to nil", func() { + nc.Spec.SecurityGroupSelectorTerms = nil + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when no security group selector terms exist", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{} + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has no values", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + {}, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has no tag map values", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{}, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has a tag map key that is empty", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "test": "", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has a tag map value that is empty", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when the last security group selector is invalid", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + { + Tags: map[string]string{ + "test2": "testvalue2", + }, + }, + { + Tags: map[string]string{ + "test3": "testvalue3", + }, + }, + { + Tags: map[string]string{ + "": "testvalue4", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when specifying id with tags", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + ID: "sg-12345749", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when specifying id with name", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + ID: "sg-12345749", + Name: "my-security-group", + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when specifying name with tags", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1.SecurityGroupSelectorTerm{ + { + Name: "my-security-group", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + }) + Context("AMISelectorTerms", func() { + It("should succeed with a valid ami selector on tags", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should succeed with a valid ami selector on id", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + ID: "sg-12345749", + }, + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should succeed with a valid ami selector on name", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Name: "testname", + }, + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should succeed with a valid ami selector on name and owner", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Name: "testname", + Owner: "testowner", + }, + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should succeed when an ami selector term has an owner key with tags", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Owner: "testowner", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should fail when a ami selector term has no values", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + {}, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a ami selector term has no tag map values", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Tags: map[string]string{}, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a ami selector term has a tag map key that is empty", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Tags: map[string]string{ + "test": "", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a ami selector term has a tag map value that is empty", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Tags: map[string]string{ + "": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when the last ami selector is invalid", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + { + Tags: map[string]string{ + "test2": "testvalue2", + }, + }, + { + Tags: map[string]string{ + "test3": "testvalue3", + }, + }, + { + Tags: map[string]string{ + "": "testvalue4", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when specifying id with tags", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + ID: "ami-12345749", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when specifying id with name", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + ID: "ami-12345749", + Name: "my-custom-ami", + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when specifying id with owner", func() { + nc.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + ID: "ami-12345749", + Owner: "123456789", + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + }) + Context("BlockDeviceMappings", func() { + It("should fail if more than one root volume is specified", func() { + nodeClass := &v1.EC2NodeClass{ + Spec: v1.EC2NodeClassSpec{ + BlockDeviceMappings: []*v1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(50, resource.Giga), + }, + + RootVolume: true, + }, + { + DeviceName: aws.String("map-device-2"), + EBS: &v1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(50, resource.Giga), + }, + + RootVolume: true, + }, + }, + }, + } + Expect(nodeClass.Validate(ctx)).To(Not(Succeed())) + }) + }) + Context("Role Immutability", func() { + It("should fail when updating the role", func() { + nc.Spec.Role = "test-role" + Expect(nc.Validate(ctx)).To(Succeed()) + + updateCtx := apis.WithinUpdate(ctx, nc.DeepCopy()) + nc.Spec.Role = "test-role2" + Expect(nc.Validate(updateCtx)).ToNot(Succeed()) + }) + It("should fail to switch between an unmanaged and managed instance profile", func() { + nc.Spec.Role = "" + nc.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") + Expect(nc.Validate(ctx)).To(Succeed()) + + updateCtx := apis.WithinUpdate(ctx, nc.DeepCopy()) + nc.Spec.Role = "test-role" + nc.Spec.InstanceProfile = nil + Expect(nc.Validate(updateCtx)).ToNot(Succeed()) + }) + It("should fail to switch between a managed and unmanaged instance profile", func() { + nc.Spec.Role = "test-role" + nc.Spec.InstanceProfile = nil + Expect(nc.Validate(ctx)).To(Succeed()) + + updateCtx := apis.WithinUpdate(ctx, nc.DeepCopy()) + nc.Spec.Role = "" + nc.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") + Expect(nc.Validate(updateCtx)).ToNot(Succeed()) + }) + }) +}) diff --git a/pkg/apis/v1/labels.go b/pkg/apis/v1/labels.go new file mode 100644 index 000000000000..c723ae379300 --- /dev/null +++ b/pkg/apis/v1/labels.go @@ -0,0 +1,123 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "fmt" + "regexp" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + + "sigs.k8s.io/karpenter/pkg/apis/v1beta1" +) + +func init() { + v1beta1.RestrictedLabelDomains = v1beta1.RestrictedLabelDomains.Insert(RestrictedLabelDomains...) + v1beta1.WellKnownLabels = v1beta1.WellKnownLabels.Insert( + LabelInstanceHypervisor, + LabelInstanceEncryptionInTransitSupported, + LabelInstanceCategory, + LabelInstanceFamily, + LabelInstanceGeneration, + LabelInstanceSize, + LabelInstanceLocalNVME, + LabelInstanceCPU, + LabelInstanceCPUManufacturer, + LabelInstanceMemory, + LabelInstanceEBSBandwidth, + LabelInstanceNetworkBandwidth, + LabelInstanceGPUName, + LabelInstanceGPUManufacturer, + LabelInstanceGPUCount, + LabelInstanceGPUMemory, + LabelInstanceAcceleratorName, + LabelInstanceAcceleratorManufacturer, + LabelInstanceAcceleratorCount, + v1.LabelWindowsBuild, + ) +} + +var ( + TerminationFinalizer = Group + "/termination" + AWSToKubeArchitectures = map[string]string{ + "x86_64": v1beta1.ArchitectureAmd64, + v1beta1.ArchitectureArm64: v1beta1.ArchitectureArm64, + } + WellKnownArchitectures = sets.NewString( + v1beta1.ArchitectureAmd64, + v1beta1.ArchitectureArm64, + ) + RestrictedLabelDomains = []string{ + Group, + } + RestrictedTagPatterns = []*regexp.Regexp{ + // Adheres to cluster name pattern matching as specified in the API spec + // https://docs.aws.amazon.com/eks/latest/APIReference/API_CreateCluster.html + regexp.MustCompile(`^kubernetes\.io/cluster/[0-9A-Za-z][A-Za-z0-9\-_]*$`), + regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(v1beta1.NodePoolLabelKey))), + regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(v1beta1.ManagedByAnnotationKey))), + regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(LabelNodeClass))), + regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(TagNodeClaim))), + } + AMIFamilyBottlerocket = "Bottlerocket" + AMIFamilyAL2 = "AL2" + AMIFamilyAL2023 = "AL2023" + AMIFamilyUbuntu = "Ubuntu" + AMIFamilyWindows2019 = "Windows2019" + AMIFamilyWindows2022 = "Windows2022" + AMIFamilyCustom = "Custom" + Windows2019 = "2019" + Windows2022 = "2022" + WindowsCore = "Core" + Windows2019Build = "10.0.17763" + Windows2022Build = "10.0.20348" + ResourceNVIDIAGPU v1.ResourceName = "nvidia.com/gpu" + ResourceAMDGPU v1.ResourceName = "amd.com/gpu" + ResourceAWSNeuron v1.ResourceName = "aws.amazon.com/neuron" + ResourceHabanaGaudi v1.ResourceName = "habana.ai/gaudi" + ResourceAWSPodENI v1.ResourceName = "vpc.amazonaws.com/pod-eni" + ResourcePrivateIPv4Address v1.ResourceName = "vpc.amazonaws.com/PrivateIPv4Address" + ResourceEFA v1.ResourceName = "vpc.amazonaws.com/efa" + + LabelNodeClass = Group + "/ec2nodeclass" + + LabelInstanceHypervisor = Group + "/instance-hypervisor" + LabelInstanceEncryptionInTransitSupported = Group + "/instance-encryption-in-transit-supported" + LabelInstanceCategory = Group + "/instance-category" + LabelInstanceFamily = Group + "/instance-family" + LabelInstanceGeneration = Group + "/instance-generation" + LabelInstanceLocalNVME = Group + "/instance-local-nvme" + LabelInstanceSize = Group + "/instance-size" + LabelInstanceCPU = Group + "/instance-cpu" + LabelInstanceCPUManufacturer = Group + "/instance-cpu-manufacturer" + LabelInstanceMemory = Group + "/instance-memory" + LabelInstanceEBSBandwidth = Group + "/instance-ebs-bandwidth" + LabelInstanceNetworkBandwidth = Group + "/instance-network-bandwidth" + LabelInstanceGPUName = Group + "/instance-gpu-name" + LabelInstanceGPUManufacturer = Group + "/instance-gpu-manufacturer" + LabelInstanceGPUCount = Group + "/instance-gpu-count" + LabelInstanceGPUMemory = Group + "/instance-gpu-memory" + LabelInstanceAcceleratorName = Group + "/instance-accelerator-name" + LabelInstanceAcceleratorManufacturer = Group + "/instance-accelerator-manufacturer" + LabelInstanceAcceleratorCount = Group + "/instance-accelerator-count" + AnnotationEC2NodeClassHash = Group + "/ec2nodeclass-hash" + AnnotationEC2NodeClassHashVersion = Group + "/ec2nodeclass-hash-version" + AnnotationInstanceTagged = Group + "/tagged" + + TagNodeClaim = v1beta1.Group + "/nodeclaim" + TagManagedLaunchTemplate = Group + "/cluster" + TagName = "Name" +) diff --git a/pkg/apis/v1/nodepool_validation_cel_test.go b/pkg/apis/v1/nodepool_validation_cel_test.go new file mode 100644 index 000000000000..0ab0af052989 --- /dev/null +++ b/pkg/apis/v1/nodepool_validation_cel_test.go @@ -0,0 +1,112 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1_test + +import ( + "strings" + + "github.com/Pallinder/go-randomdata" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + + "sigs.k8s.io/karpenter/pkg/apis/v1beta1" +) + +// TODO @engedaam: Updated NodePool and a NodeClaim to use the v1 API +var _ = Describe("CEL/Validation", func() { + var nodePool *v1beta1.NodePool + + BeforeEach(func() { + if env.Version.Minor() < 25 { + Skip("CEL Validation is for 1.25>") + } + nodePool = &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, + Spec: v1beta1.NodePoolSpec{ + Template: v1beta1.NodeClaimTemplate{ + Spec: v1beta1.NodeClaimSpec{ + NodeClassRef: &v1beta1.NodeClassReference{ + Kind: "NodeClaim", + Name: "default", + }, + Requirements: []v1beta1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpExists, + }, + }, + }, + }, + }, + }, + } + }) + Context("Requirements", func() { + It("should allow restricted domains exceptions", func() { + oldNodePool := nodePool.DeepCopy() + for label := range v1beta1.LabelDomainExceptions { + nodePool.Spec.Template.Spec.Requirements = []v1beta1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: label + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}}, + } + Expect(env.Client.Create(ctx, nodePool)).To(Succeed()) + Expect(nodePool.RuntimeValidate()).To(Succeed()) + Expect(env.Client.Delete(ctx, nodePool)).To(Succeed()) + nodePool = oldNodePool.DeepCopy() + } + }) + It("should allow well known label exceptions", func() { + oldNodePool := nodePool.DeepCopy() + for label := range v1beta1.WellKnownLabels.Difference(sets.New(v1beta1.NodePoolLabelKey)) { + nodePool.Spec.Template.Spec.Requirements = []v1beta1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: label, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}}, + } + Expect(env.Client.Create(ctx, nodePool)).To(Succeed()) + Expect(nodePool.RuntimeValidate()).To(Succeed()) + Expect(env.Client.Delete(ctx, nodePool)).To(Succeed()) + nodePool = oldNodePool.DeepCopy() + } + }) + }) + Context("Labels", func() { + It("should allow restricted domains exceptions", func() { + oldNodePool := nodePool.DeepCopy() + for label := range v1beta1.LabelDomainExceptions { + nodePool.Spec.Template.Labels = map[string]string{ + label: "test", + } + Expect(env.Client.Create(ctx, nodePool)).To(Succeed()) + Expect(nodePool.RuntimeValidate()).To(Succeed()) + Expect(env.Client.Delete(ctx, nodePool)).To(Succeed()) + nodePool = oldNodePool.DeepCopy() + } + }) + It("should allow well known label exceptions", func() { + oldNodePool := nodePool.DeepCopy() + for label := range v1beta1.WellKnownLabels.Difference(sets.New(v1beta1.NodePoolLabelKey)) { + nodePool.Spec.Template.Labels = map[string]string{ + label: "test", + } + Expect(env.Client.Create(ctx, nodePool)).To(Succeed()) + Expect(nodePool.RuntimeValidate()).To(Succeed()) + Expect(env.Client.Delete(ctx, nodePool)).To(Succeed()) + nodePool = oldNodePool.DeepCopy() + } + }) + }) +}) diff --git a/pkg/apis/v1/register.go b/pkg/apis/v1/register.go new file mode 100644 index 000000000000..ed82071157d8 --- /dev/null +++ b/pkg/apis/v1/register.go @@ -0,0 +1,34 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ( + Group = "karpenter.k8s.aws" + SchemeGroupVersion = schema.GroupVersion{Group: Group, Version: "v1"} + SchemeBuilder = runtime.NewSchemeBuilder(func(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &EC2NodeClass{}, + &EC2NodeClassList{}, + ) + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil + }) +) diff --git a/pkg/apis/v1/suite_test.go b/pkg/apis/v1/suite_test.go new file mode 100644 index 000000000000..b3309cbca37c --- /dev/null +++ b/pkg/apis/v1/suite_test.go @@ -0,0 +1,55 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "knative.dev/pkg/logging/testing" + + . "sigs.k8s.io/karpenter/pkg/test/expectations" + + "sigs.k8s.io/karpenter/pkg/operator/scheme" + coretest "sigs.k8s.io/karpenter/pkg/test" + + "github.com/aws/karpenter-provider-aws/pkg/apis" + "github.com/aws/karpenter-provider-aws/pkg/test" +) + +var ctx context.Context +var env *coretest.Environment +var awsEnv *test.Environment + +func TestAPIs(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "Validation") +} + +var _ = BeforeSuite(func() { + env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + awsEnv = test.NewEnvironment(ctx, env) +}) + +var _ = AfterEach(func() { + ExpectCleanedUp(ctx, env.Client) +}) + +var _ = AfterSuite(func() { + Expect(env.Stop()).To(Succeed(), "Failed to stop environment") +}) diff --git a/pkg/apis/v1/zz_generated.deepcopy.go b/pkg/apis/v1/zz_generated.deepcopy.go new file mode 100644 index 000000000000..07709df56478 --- /dev/null +++ b/pkg/apis/v1/zz_generated.deepcopy.go @@ -0,0 +1,450 @@ +//go:build !ignore_autogenerated + +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1 + +import ( + "github.com/awslabs/operatorpkg/status" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AMI) DeepCopyInto(out *AMI) { + *out = *in + if in.Requirements != nil { + in, out := &in.Requirements, &out.Requirements + *out = make([]corev1.NodeSelectorRequirement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AMI. +func (in *AMI) DeepCopy() *AMI { + if in == nil { + return nil + } + out := new(AMI) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AMISelectorTerm) DeepCopyInto(out *AMISelectorTerm) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AMISelectorTerm. +func (in *AMISelectorTerm) DeepCopy() *AMISelectorTerm { + if in == nil { + return nil + } + out := new(AMISelectorTerm) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BlockDevice) DeepCopyInto(out *BlockDevice) { + *out = *in + if in.DeleteOnTermination != nil { + in, out := &in.DeleteOnTermination, &out.DeleteOnTermination + *out = new(bool) + **out = **in + } + if in.Encrypted != nil { + in, out := &in.Encrypted, &out.Encrypted + *out = new(bool) + **out = **in + } + if in.IOPS != nil { + in, out := &in.IOPS, &out.IOPS + *out = new(int64) + **out = **in + } + if in.KMSKeyID != nil { + in, out := &in.KMSKeyID, &out.KMSKeyID + *out = new(string) + **out = **in + } + if in.SnapshotID != nil { + in, out := &in.SnapshotID, &out.SnapshotID + *out = new(string) + **out = **in + } + if in.Throughput != nil { + in, out := &in.Throughput, &out.Throughput + *out = new(int64) + **out = **in + } + if in.VolumeSize != nil { + in, out := &in.VolumeSize, &out.VolumeSize + x := (*in).DeepCopy() + *out = &x + } + if in.VolumeType != nil { + in, out := &in.VolumeType, &out.VolumeType + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlockDevice. +func (in *BlockDevice) DeepCopy() *BlockDevice { + if in == nil { + return nil + } + out := new(BlockDevice) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BlockDeviceMapping) DeepCopyInto(out *BlockDeviceMapping) { + *out = *in + if in.DeviceName != nil { + in, out := &in.DeviceName, &out.DeviceName + *out = new(string) + **out = **in + } + if in.EBS != nil { + in, out := &in.EBS, &out.EBS + *out = new(BlockDevice) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlockDeviceMapping. +func (in *BlockDeviceMapping) DeepCopy() *BlockDeviceMapping { + if in == nil { + return nil + } + out := new(BlockDeviceMapping) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EC2NodeClass) DeepCopyInto(out *EC2NodeClass) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EC2NodeClass. +func (in *EC2NodeClass) DeepCopy() *EC2NodeClass { + if in == nil { + return nil + } + out := new(EC2NodeClass) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EC2NodeClass) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EC2NodeClassList) DeepCopyInto(out *EC2NodeClassList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]EC2NodeClass, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EC2NodeClassList. +func (in *EC2NodeClassList) DeepCopy() *EC2NodeClassList { + if in == nil { + return nil + } + out := new(EC2NodeClassList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EC2NodeClassList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EC2NodeClassSpec) DeepCopyInto(out *EC2NodeClassSpec) { + *out = *in + if in.SubnetSelectorTerms != nil { + in, out := &in.SubnetSelectorTerms, &out.SubnetSelectorTerms + *out = make([]SubnetSelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.SecurityGroupSelectorTerms != nil { + in, out := &in.SecurityGroupSelectorTerms, &out.SecurityGroupSelectorTerms + *out = make([]SecurityGroupSelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.AssociatePublicIPAddress != nil { + in, out := &in.AssociatePublicIPAddress, &out.AssociatePublicIPAddress + *out = new(bool) + **out = **in + } + if in.AMISelectorTerms != nil { + in, out := &in.AMISelectorTerms, &out.AMISelectorTerms + *out = make([]AMISelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.AMIFamily != nil { + in, out := &in.AMIFamily, &out.AMIFamily + *out = new(string) + **out = **in + } + if in.UserData != nil { + in, out := &in.UserData, &out.UserData + *out = new(string) + **out = **in + } + if in.InstanceProfile != nil { + in, out := &in.InstanceProfile, &out.InstanceProfile + *out = new(string) + **out = **in + } + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.BlockDeviceMappings != nil { + in, out := &in.BlockDeviceMappings, &out.BlockDeviceMappings + *out = make([]*BlockDeviceMapping, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(BlockDeviceMapping) + (*in).DeepCopyInto(*out) + } + } + } + if in.InstanceStorePolicy != nil { + in, out := &in.InstanceStorePolicy, &out.InstanceStorePolicy + *out = new(InstanceStorePolicy) + **out = **in + } + if in.DetailedMonitoring != nil { + in, out := &in.DetailedMonitoring, &out.DetailedMonitoring + *out = new(bool) + **out = **in + } + if in.MetadataOptions != nil { + in, out := &in.MetadataOptions, &out.MetadataOptions + *out = new(MetadataOptions) + (*in).DeepCopyInto(*out) + } + if in.Context != nil { + in, out := &in.Context, &out.Context + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EC2NodeClassSpec. +func (in *EC2NodeClassSpec) DeepCopy() *EC2NodeClassSpec { + if in == nil { + return nil + } + out := new(EC2NodeClassSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EC2NodeClassStatus) DeepCopyInto(out *EC2NodeClassStatus) { + *out = *in + if in.Subnets != nil { + in, out := &in.Subnets, &out.Subnets + *out = make([]Subnet, len(*in)) + copy(*out, *in) + } + if in.SecurityGroups != nil { + in, out := &in.SecurityGroups, &out.SecurityGroups + *out = make([]SecurityGroup, len(*in)) + copy(*out, *in) + } + if in.AMIs != nil { + in, out := &in.AMIs, &out.AMIs + *out = make([]AMI, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]status.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EC2NodeClassStatus. +func (in *EC2NodeClassStatus) DeepCopy() *EC2NodeClassStatus { + if in == nil { + return nil + } + out := new(EC2NodeClassStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetadataOptions) DeepCopyInto(out *MetadataOptions) { + *out = *in + if in.HTTPEndpoint != nil { + in, out := &in.HTTPEndpoint, &out.HTTPEndpoint + *out = new(string) + **out = **in + } + if in.HTTPProtocolIPv6 != nil { + in, out := &in.HTTPProtocolIPv6, &out.HTTPProtocolIPv6 + *out = new(string) + **out = **in + } + if in.HTTPPutResponseHopLimit != nil { + in, out := &in.HTTPPutResponseHopLimit, &out.HTTPPutResponseHopLimit + *out = new(int64) + **out = **in + } + if in.HTTPTokens != nil { + in, out := &in.HTTPTokens, &out.HTTPTokens + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetadataOptions. +func (in *MetadataOptions) DeepCopy() *MetadataOptions { + if in == nil { + return nil + } + out := new(MetadataOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityGroup) DeepCopyInto(out *SecurityGroup) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityGroup. +func (in *SecurityGroup) DeepCopy() *SecurityGroup { + if in == nil { + return nil + } + out := new(SecurityGroup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityGroupSelectorTerm) DeepCopyInto(out *SecurityGroupSelectorTerm) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityGroupSelectorTerm. +func (in *SecurityGroupSelectorTerm) DeepCopy() *SecurityGroupSelectorTerm { + if in == nil { + return nil + } + out := new(SecurityGroupSelectorTerm) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Subnet) DeepCopyInto(out *Subnet) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Subnet. +func (in *Subnet) DeepCopy() *Subnet { + if in == nil { + return nil + } + out := new(Subnet) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetSelectorTerm) DeepCopyInto(out *SubnetSelectorTerm) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetSelectorTerm. +func (in *SubnetSelectorTerm) DeepCopy() *SubnetSelectorTerm { + if in == nil { + return nil + } + out := new(SubnetSelectorTerm) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/apis/v1beta1/ec2nodeclass.go b/pkg/apis/v1beta1/ec2nodeclass.go index 66a2926b2ced..47e0bf516b65 100644 --- a/pkg/apis/v1beta1/ec2nodeclass.go +++ b/pkg/apis/v1beta1/ec2nodeclass.go @@ -318,6 +318,7 @@ const ( // EC2NodeClass is the Schema for the EC2NodeClass API // +kubebuilder:object:root=true +// +kubebuilder:storageversion // +kubebuilder:resource:path=ec2nodeclasses,scope=Cluster,categories=karpenter,shortName={ec2nc,ec2ncs} // +kubebuilder:subresource:status type EC2NodeClass struct {