From 09d4c4a7ce72eac198f11130c07b61cf91c8d06b Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Tue, 8 Aug 2023 18:10:32 -0700 Subject: [PATCH] Create v1beta1 APIs --- Makefile | 4 +- charts/karpenter/crds | 1 - .../karpenter.k8s.aws_awsnodetemplates.yaml | 1 + .../karpenter/crds/karpenter.sh_machines.yaml | 1 + .../crds/karpenter.sh_provisioners.yaml | 1 + pkg/apis/apis.go | 7 +- .../crds/compute.k8s.aws_nodeclasses.yaml | 357 ++++++++++++++ pkg/apis/crds/karpenter.sh_nodeclaims.yaml | 359 ++++++++++++++ pkg/apis/crds/karpenter.sh_nodepools.yaml | 374 +++++++++++++++ pkg/apis/v1beta1/doc.go | 19 + pkg/apis/v1beta1/nodeclass.go | 273 +++++++++++ pkg/apis/v1beta1/nodeclass_defaults.go | 22 + pkg/apis/v1beta1/nodeclass_status.go | 66 +++ pkg/apis/v1beta1/nodeclass_validation.go | 257 ++++++++++ pkg/apis/v1beta1/register.go | 152 ++++++ pkg/apis/v1beta1/suite_test.go | 201 ++++++++ pkg/apis/v1beta1/zz_generated.deepcopy.go | 438 ++++++++++++++++++ 17 files changed, 2528 insertions(+), 5 deletions(-) delete mode 120000 charts/karpenter/crds create mode 120000 charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml create mode 120000 charts/karpenter/crds/karpenter.sh_machines.yaml create mode 120000 charts/karpenter/crds/karpenter.sh_provisioners.yaml create mode 100644 pkg/apis/crds/compute.k8s.aws_nodeclasses.yaml create mode 100644 pkg/apis/crds/karpenter.sh_nodeclaims.yaml create mode 100644 pkg/apis/crds/karpenter.sh_nodepools.yaml create mode 100644 pkg/apis/v1beta1/doc.go create mode 100644 pkg/apis/v1beta1/nodeclass.go create mode 100644 pkg/apis/v1beta1/nodeclass_defaults.go create mode 100644 pkg/apis/v1beta1/nodeclass_status.go create mode 100644 pkg/apis/v1beta1/nodeclass_validation.go create mode 100644 pkg/apis/v1beta1/register.go create mode 100644 pkg/apis/v1beta1/suite_test.go create mode 100644 pkg/apis/v1beta1/zz_generated.deepcopy.go diff --git a/Makefile b/Makefile index 221c3e671481..0eea2816ae43 100644 --- a/Makefile +++ b/Makefile @@ -112,9 +112,7 @@ coverage: verify: tidy download ## Verify code. Includes dependencies, linting, formatting, etc go generate ./... hack/boilerplate.sh - cp $(KARPENTER_CORE_DIR)/pkg/apis/crds/karpenter.sh_machines.yaml \ - $(KARPENTER_CORE_DIR)/pkg/apis/crds/karpenter.sh_provisioners.yaml \ - pkg/apis/crds + cp $(KARPENTER_CORE_DIR)/pkg/apis/crds/* pkg/apis/crds $(foreach dir,$(MOD_DIRS),cd $(dir) && golangci-lint run $(newline)) @git diff --quiet ||\ { echo "New file modification detected in the Git working tree. Please check in before commit."; git --no-pager diff --name-only | uniq | awk '{print " - " $$0}'; \ diff --git a/charts/karpenter/crds b/charts/karpenter/crds deleted file mode 120000 index e33998437775..000000000000 --- a/charts/karpenter/crds +++ /dev/null @@ -1 +0,0 @@ -../../pkg/apis/crds \ No newline at end of file diff --git a/charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml b/charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml new file mode 120000 index 000000000000..da7cc7630001 --- /dev/null +++ b/charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml @@ -0,0 +1 @@ +../../../pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml \ No newline at end of file diff --git a/charts/karpenter/crds/karpenter.sh_machines.yaml b/charts/karpenter/crds/karpenter.sh_machines.yaml new file mode 120000 index 000000000000..f08680c944a0 --- /dev/null +++ b/charts/karpenter/crds/karpenter.sh_machines.yaml @@ -0,0 +1 @@ +../../../pkg/apis/crds/karpenter.sh_machines.yaml \ No newline at end of file diff --git a/charts/karpenter/crds/karpenter.sh_provisioners.yaml b/charts/karpenter/crds/karpenter.sh_provisioners.yaml new file mode 120000 index 000000000000..921d1456ec3f --- /dev/null +++ b/charts/karpenter/crds/karpenter.sh_provisioners.yaml @@ -0,0 +1 @@ +../../../pkg/apis/crds/karpenter.sh_provisioners.yaml \ No newline at end of file diff --git a/pkg/apis/apis.go b/pkg/apis/apis.go index 44ee7abcdddd..349139566e0c 100644 --- a/pkg/apis/apis.go +++ b/pkg/apis/apis.go @@ -46,7 +46,12 @@ var ( var ( //go:embed crds/karpenter.k8s.aws_awsnodetemplates.yaml AWSNodeTemplateCRD []byte - CRDs = append(apis.CRDs, lo.Must(functional.Unmarshal[v1.CustomResourceDefinition](AWSNodeTemplateCRD))) + //go:embed crds/compute.k8s.aws_nodeclasses.yaml + NodeClassCRD []byte + CRDs = append(apis.CRDs, + lo.Must(functional.Unmarshal[v1.CustomResourceDefinition](AWSNodeTemplateCRD)), + lo.Must(functional.Unmarshal[v1.CustomResourceDefinition](NodeClassCRD)), + ) ) func init() { diff --git a/pkg/apis/crds/compute.k8s.aws_nodeclasses.yaml b/pkg/apis/crds/compute.k8s.aws_nodeclasses.yaml new file mode 100644 index 000000000000..444e7a74c527 --- /dev/null +++ b/pkg/apis/crds/compute.k8s.aws_nodeclasses.yaml @@ -0,0 +1,357 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.1 + name: nodeclasses.compute.k8s.aws +spec: + group: compute.k8s.aws + names: + categories: + - karpenter + kind: NodeClass + listKind: NodeClassList + plural: nodeclasses + singular: nodeclass + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: NodeClass is the Schema for the NodeClass API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodeClassSpec is the top level specification for the AWS + Karpenter Provider. This will contain configuration necessary to launch + instances in AWS. + properties: + amiFamily: + description: AMIFamily is the AMI family that instances use. + type: string + amiSelectorTerms: + description: AMISelectorTerms is a list of or ami selector terms. + The terms are ORed. + items: + description: AMISelectorTerm defines selection logic for an ami + used by Karpenter to launch nodes. If multiple fields are used + for selection, the requirements are ANDed. + properties: + id: + description: ID is the ami id in EC2 + pattern: ami-[0-9a-z]+ + type: string + name: + description: Name is the ami name in EC2. This value is thename + field, which is different from the name tag. + type: string + owner: + description: Owner is the owner for the ami. You can specify + a combination of AWS account IDs, "self", "amazon",and "aws-marketplace" + type: string + ssm: + description: SSM is the ssm alias for an ami. + type: string + tags: + additionalProperties: + type: string + description: Tags is a map of key/value tags used to select + subnets Specifying '*' for a value selects all values for + a given tag key. + type: object + type: object + type: array + blockDeviceMappings: + description: BlockDeviceMappings to be applied to provisioned nodes. + items: + properties: + deviceName: + description: The device name (for example, /dev/sdh or xvdh). + type: string + ebs: + description: EBS contains parameters used to automatically set + up EBS volumes when an instance is launched. + properties: + deleteOnTermination: + description: DeleteOnTermination indicates whether the EBS + volume is deleted on instance termination. + type: boolean + encrypted: + description: Encrypted indicates whether the EBS volume + is encrypted. Encrypted volumes can only be attached to + instances that support Amazon EBS encryption. If you are + creating a volume from a snapshot, you can't specify an + encryption value. + type: boolean + iops: + description: "IOPS is the number of I/O operations per second + (IOPS). For gp3, io1, and io2 volumes, this represents + the number of IOPS that are provisioned for the volume. + For gp2 volumes, this represents the baseline performance + of the volume and the rate at which the volume accumulates + I/O credits for bursting. \n The following are the supported + values for each volume type: \n * gp3: 3,000-16,000 IOPS + \n * io1: 100-64,000 IOPS \n * io2: 100-64,000 IOPS \n + For io1 and io2 volumes, we guarantee 64,000 IOPS only + for Instances built on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 + IOPS. \n This parameter is supported for io1, io2, and + gp3 volumes only. This parameter is not supported for + gp2, st1, sc1, or standard volumes." + format: int64 + type: integer + kmsKeyID: + description: KMSKeyID (ARN) of the symmetric Key Management + Service (KMS) CMK used for encryption. + type: string + snapshotID: + description: SnapshotID is the ID of an EBS snapshot + type: string + throughput: + description: 'Throughput to provision for a gp3 volume, + with a maximum of 1,000 MiB/s. Valid Range: Minimum value + of 125. Maximum value of 1000.' + format: int64 + type: integer + volumeSize: + anyOf: + - type: integer + - type: string + description: "VolumeSize in GiBs. You must specify either + a snapshot ID or a volume size. The following are the + supported volumes sizes for each volume type: \n * gp2 + and gp3: 1-16,384 \n * io1 and io2: 4-16,384 \n * st1 + and sc1: 125-16,384 \n * standard: 1-1,024" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + volumeType: + description: VolumeType of the block device. For more information, + see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + in the Amazon Elastic Compute Cloud User Guide. + type: string + type: object + type: object + type: array + context: + description: Context is a Reserved field in EC2 APIs https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + type: string + detailedMonitoring: + description: DetailedMonitoring controls if detailed monitoring is + enabled for instances that are launched + type: boolean + metadataOptions: + description: "MetadataOptions for the generated launch template of + provisioned nodes. \n This specifies the exposure of the Instance + Metadata Service to provisioned EC2 nodes. For more information, + see Instance Metadata and User Data (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. \n Refer to recommended, + security best practices (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + for limiting exposure of Instance Metadata and User Data to pods. + If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + disabled, with httpPutResponseLimit of 2, and with httpTokens required." + properties: + httpEndpoint: + description: "HTTPEndpoint enables or disables the HTTP metadata + endpoint on provisioned nodes. If metadata options is non-nil, + but this parameter is not specified, the default state is \"enabled\". + \n If you specify a value of \"disabled\", instance metadata + will not be accessible on the node." + type: string + httpProtocolIPv6: + description: HTTPProtocolIPv6 enables or disables the IPv6 endpoint + for the instance metadata service on provisioned nodes. If metadata + options is non-nil, but this parameter is not specified, the + default state is "disabled". + type: string + httpPutResponseHopLimit: + description: HTTPPutResponseHopLimit is the desired HTTP PUT response + hop limit for instance metadata requests. The larger the number, + the further instance metadata requests can travel. Possible + values are integers from 1 to 64. If metadata options is non-nil, + but this parameter is not specified, the default value is 1. + format: int64 + type: integer + httpTokens: + description: "HTTPTokens determines the state of token usage for + instance metadata requests. If metadata options is non-nil, + but this parameter is not specified, the default state is \"optional\". + \n If the state is optional, one can choose to retrieve instance + metadata with or without a signed token header on the request. + If one retrieves the IAM role credentials without a token, the + version 1.0 role credentials are returned. If one retrieves + the IAM role credentials using a valid signed token, the version + 2.0 role credentials are returned. \n If the state is \"required\", + one must send a signed token header with any instance metadata + retrieval requests. In this state, retrieving the IAM role credentials + always returns the version 2.0 credentials; the version 1.0 + credentials are not available." + type: string + type: object + role: + description: Role is the AWS identity that nodes use. + type: string + securityGroupSelectorTerms: + description: SecurityGroupSelectorTerms is a list of or security group + selector terms. The terms are ORed. + items: + description: SecurityGroupSelectorTerm defines selection logic for + a security group used by Karpenter to launch nodes. If multiple + fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the security group id in EC2 + pattern: sg-[0-9a-z]+ + type: string + name: + description: Name is the security group name in EC2. This value + is the name field, which is different from the name tag. + type: string + tags: + additionalProperties: + type: string + description: Tags is a map of key/value tags used to select + subnets Specifying '*' for a value selects all values for + a given tag key. + type: object + type: object + type: array + subnetSelectorTerms: + description: SubnetSelectorTerms is a list of or subnet selector terms. + The terms are ORed. + items: + description: SubnetSelectorTerm defines selection logic for a subnet + used by Karpenter to launch nodes. If multiple fields are used + for selection, the requirements are ANDed. + properties: + id: + description: ID is the subnet id in EC2 + pattern: subnet-[0-9a-z]+ + type: string + tags: + additionalProperties: + type: string + description: Tags is a map of key/value tags used to select + subnets Specifying '*' for a value selects all values for + a given tag key. + type: object + type: object + type: array + tags: + additionalProperties: + type: string + description: Tags to be applied on ec2 resources like instances and + launch templates. + type: object + userData: + description: UserData to be applied to the provisioned nodes. It must + be in the appropriate format based on the AMIFamily in use. Karpenter + will merge certain fields into this UserData to ensure nodes are + being provisioned with the correct configuration. + type: string + type: object + status: + description: NodeClassStatus contains the resolved state of the NodeClass + properties: + amis: + description: AMI contains the current AMI values that are available + to the cluster under the AMI selectors. + items: + description: AMI contains resolved AMI selector values utilized + for node launch + properties: + id: + description: ID of the AMI + type: string + name: + description: Name of the AMI + type: string + requirements: + description: Requirements of the AMI to be utilized on an instance + type + items: + description: A node selector requirement is a selector that + contains values, a key, and an operator that relates the + key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set + of values. Valid operators are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, the + values array must have a single element, which will + be interpreted as an integer. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + required: + - id + - requirements + type: object + type: array + securityGroups: + description: SecurityGroups contains the current Security Groups values + that are available to the cluster under the SecurityGroups selectors. + items: + description: SecurityGroup contains resolved SecurityGroup selector + values utilized for node launch + properties: + id: + description: ID of the security group + type: string + name: + description: Name of the security group + type: string + required: + - id + type: object + type: array + subnets: + description: Subnets contains the current Subnet values that are available + to the cluster under the subnet selectors. + items: + description: Subnet contains resolved Subnet selector values utilized + for node launch + properties: + id: + description: ID of the subnet + type: string + zone: + description: The associated availability zone + type: string + required: + - id + - zone + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml new file mode 100644 index 000000000000..eb89aeeb0725 --- /dev/null +++ b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml @@ -0,0 +1,359 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.1 + name: nodeclaims.karpenter.sh +spec: + group: karpenter.sh + names: + categories: + - karpenter + kind: NodeClaim + listKind: NodeClaimList + plural: nodeclaims + shortNames: + - nc + - ncs + singular: nodeclaim + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type + name: Type + type: string + - jsonPath: .metadata.labels.topology\.kubernetes\.io/zone + name: Zone + type: string + - jsonPath: .status.nodeName + name: Node + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .metadata.labels.karpenter\.sh/capacity-type + name: Capacity + priority: 1 + type: string + - jsonPath: .metadata.labels.karpenter\.sh/nodepool + name: NodePool + priority: 1 + type: string + - jsonPath: .spec.nodeClass.name + name: NodeClass + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: NodeClaim is the Schema for the NodeClaims API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodeClaimSpec describes the desired state of the NodeClaim + properties: + kubeletConfiguration: + description: KubeletConfiguration are options passed to the kubelet + when provisioning nodes + properties: + clusterDNS: + description: clusterDNS is a list of IP addresses for the cluster + DNS server. Note that not all providers may use all addresses. + items: + type: string + type: array + containerRuntime: + description: ContainerRuntime is the container runtime to be used + with your worker nodes. + type: string + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement for + containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + description: EvictionHard is the map of signal names to quantities + that define hard eviction thresholds + type: object + evictionMaxPodGracePeriod: + description: EvictionMaxPodGracePeriod is the maximum allowed + grace period (in seconds) to use when terminating pods in response + to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + description: EvictionSoft is the map of signal names to quantities + that define soft eviction thresholds + type: object + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal names + to quantities that define grace periods for each eviction signal + type: object + imageGCHighThresholdPercent: + description: ImageGCHighThresholdPercent is the percent of disk + usage after which image garbage collection is always run. The + percent is calculated by dividing this field value by 100, so + this field must be between 0 and 100, inclusive. When specified, + the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: ImageGCLowThresholdPercent is the percent of disk + usage before which image garbage collection is never run. Lowest + disk usage to garbage collect to. The percent is calculated + by dividing this field value by 100, so the field value must + be between 0 and 100, inclusive. When specified, the value must + be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: KubeReserved contains resources reserved for Kubernetes + system components. + type: object + maxPods: + description: MaxPods is an override for the maximum number of + pods that can run on a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: PodsPerCore is an override for the number of pods + that can run on a worker node instance based on the number of + cpu cores. This value cannot exceed MaxPods, so, if MaxPods + is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: SystemReserved contains resources reserved for OS + system daemons and kernel memory. + type: object + type: object + nodeClass: + description: NodeClass is a reference to an object that defines provider + specific configuration + properties: + apiVersion: + description: API version of the referent + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to + every node. + items: + description: A node selector requirement is a selector that contains + values, a key, and an operator that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and + Lt. + type: string + values: + description: An array of string values. If the operator is In + or NotIn, the values array must be non-empty. If the operator + is Exists or DoesNotExist, the values array must be empty. + If the operator is Gt or Lt, the values array must have a + single element, which will be interpreted as an integer. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + resources: + description: Resources models the resource requirements for the NodeClaim + to launch + properties: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources + for the NodeClaim to launch + type: object + type: object + startupTaints: + description: StartupTaints are taints that are applied to nodes upon + startup which are expected to be removed automatically within a + short period of time, typically by a DaemonSet that tolerates the + taint. These are commonly used by daemonsets to allow initialization + and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint + in order to have nodes provisioned for them. + items: + description: The node this Taint is attached to has the "effect" + on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods that + do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule + and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the taint + was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: The node this Taint is attached to has the "effect" + on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods that + do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule + and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the taint + was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + type: array + required: + - nodeClass + type: object + status: + description: NodeClaimStatus defines the observed state of NodeClaim + properties: + allocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocatable is the estimated allocatable capacity of + the node + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Capacity is the estimated full capacity of the node + type: object + conditions: + description: Conditions contains signals for health and readiness + items: + description: 'Condition defines a readiness condition for a Knative + resource. See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties' + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition + transitioned from one status to another. We use VolatileTime + in place of metav1.Time to exclude this from creating equality.Semantic + differences (all other things held constant). + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + severity: + description: Severity with which to treat failures of this type + of condition. When this is not specified, it defaults to Error. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - status + - type + type: object + type: array + nodeName: + description: NodeName is the name of the corresponding node object + type: string + providerID: + description: ProviderID of the corresponding node object + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/apis/crds/karpenter.sh_nodepools.yaml b/pkg/apis/crds/karpenter.sh_nodepools.yaml new file mode 100644 index 000000000000..4cc6ac296d5b --- /dev/null +++ b/pkg/apis/crds/karpenter.sh_nodepools.yaml @@ -0,0 +1,374 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.1 + name: nodepools.karpenter.sh +spec: + group: karpenter.sh + names: + categories: + - karpenter + kind: NodePool + listKind: NodePoolList + plural: nodepools + shortNames: + - np + - nps + singular: nodepool + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.template.spec.nodeClass.name + name: NodeClass + type: string + - jsonPath: .spec.weight + name: Weight + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: NodePool is the Schema for the Provisioners API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodePoolSpec is the top level provisioner specification. + Provisioners launch nodes in response to pods that are unschedulable. + A single provisioner is capable of managing a diverse set of nodes. + Node properties are determined from a combination of provisioner and + pod scheduling constraints. + properties: + deprovisioning: + default: + consolidationPolicy: WhenUnderutilized + consolidationTTL: 15s + expirationTTL: 90d + description: Deprovisioning contains the parameters that relate to + Karpenter's deprovisioning logic + properties: + consolidationPolicy: + default: WhenUnderutilized + description: ConsolidationPolicy describes which nodes Karpenter + can deprovision through its consolidation algorithm. This policy + defaults to "WhenUnderutilized" if not specified + enum: + - Never + - WhenEmpty + - WhenUnderutilized + type: string + consolidationTTL: + default: 15s + description: ConsolidationTTL is the duration the controller will + wait before attempting to terminate nodes that are underutilized. + Refer to ConsolidationPolicy for how underutilization is considered. + type: string + expirationTTL: + default: 90d + description: ExpirationTTL is the duration the controller will + wait before terminating a node, measured from when the node + is created. This is useful to implement features like eventually + consistent node upgrade, memory leak protection, and disruption + testing. + type: string + type: object + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Limits define a set of bounds for provisioning capacity. + type: object + template: + description: Template contains the template of possibilities for the + provisioning logic to launch a NodeClaim with. NodeClaims launched + from this NodePool will often be further constrained than the template + specifies. + properties: + metadata: + type: object + spec: + description: NodeClaimSpec describes the desired state of the + NodeClaim + properties: + kubeletConfiguration: + description: KubeletConfiguration are options passed to the + kubelet when provisioning nodes + properties: + clusterDNS: + description: clusterDNS is a list of IP addresses for + the cluster DNS server. Note that not all providers + may use all addresses. + items: + type: string + type: array + containerRuntime: + description: ContainerRuntime is the container runtime + to be used with your worker nodes. + type: string + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement + for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + description: EvictionHard is the map of signal names to + quantities that define hard eviction thresholds + type: object + evictionMaxPodGracePeriod: + description: EvictionMaxPodGracePeriod is the maximum + allowed grace period (in seconds) to use when terminating + pods in response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + description: EvictionSoft is the map of signal names to + quantities that define soft eviction thresholds + type: object + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal + names to quantities that define grace periods for each + eviction signal + type: object + imageGCHighThresholdPercent: + description: ImageGCHighThresholdPercent is the percent + of disk usage after which image garbage collection is + always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 + and 100, inclusive. When specified, the value must be + greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: ImageGCLowThresholdPercent is the percent + of disk usage before which image garbage collection + is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value + by 100, so the field value must be between 0 and 100, + inclusive. When specified, the value must be less than + imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: KubeReserved contains resources reserved + for Kubernetes system components. + type: object + maxPods: + description: MaxPods is an override for the maximum number + of pods that can run on a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: PodsPerCore is an override for the number + of pods that can run on a worker node instance based + on the number of cpu cores. This value cannot exceed + MaxPods, so, if MaxPods is a lower value, that value + will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: SystemReserved contains resources reserved + for OS system daemons and kernel memory. + type: object + type: object + nodeClass: + description: NodeClass is a reference to an object that defines + provider specific configuration + properties: + apiVersion: + description: API version of the referent + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied + to every node. + items: + description: A node selector requirement is a selector that + contains values, a key, and an operator that relates the + key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to a set + of values. Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, + the values array must have a single element, which + will be interpreted as an integer. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + resources: + description: Resources models the resource requirements for + the NodeClaim to launch + properties: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources + for the NodeClaim to launch + type: object + type: object + startupTaints: + description: StartupTaints are taints that are applied to + nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet + that tolerates the taint. These are commonly used by daemonsets + to allow initialization and enforce startup ordering. StartupTaints + are ignored for provisioning purposes in that pods are not + required to tolerate a StartupTaint in order to have nodes + provisioned for them. + items: + description: The node this Taint is attached to has the + "effect" on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods + that do not tolerate the taint. Valid effects are + NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to + a node. + type: string + timeAdded: + description: TimeAdded represents the time at which + the taint was added. It is only written for NoExecute + taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint + key. + type: string + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: The node this Taint is attached to has the + "effect" on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods + that do not tolerate the taint. Valid effects are + NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to + a node. + type: string + timeAdded: + description: TimeAdded represents the time at which + the taint was added. It is only written for NoExecute + taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint + key. + type: string + required: + - effect + - key + type: object + type: array + required: + - nodeClass + type: object + type: object + weight: + description: Weight is the priority given to the provisioner during + scheduling. A higher numerical weight indicates that this provisioner + will be ordered ahead of other provisioners with lower weights. + A provisioner with no weight will be treated as if it is a provisioner + with a weight of 0. + format: int32 + maximum: 100 + minimum: 1 + type: integer + type: object + status: + description: NodePoolStatus defines the observed state of NodePool + properties: + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is the list of resources that have been provisioned. + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/apis/v1beta1/doc.go b/pkg/apis/v1beta1/doc.go new file mode 100644 index 000000000000..cf107803b309 --- /dev/null +++ b/pkg/apis/v1beta1/doc.go @@ -0,0 +1,19 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen=package,register +// +k8s:defaulter-gen=TypeMeta +// +groupName=compute.k8s.aws +package v1beta1 // doc.go is discovered by codegen diff --git a/pkg/apis/v1beta1/nodeclass.go b/pkg/apis/v1beta1/nodeclass.go new file mode 100644 index 000000000000..88a09bfefc26 --- /dev/null +++ b/pkg/apis/v1beta1/nodeclass.go @@ -0,0 +1,273 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "fmt" + + "github.com/mitchellh/hashstructure/v2" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NodeClassSpec is the top level specification for the AWS Karpenter Provider. +// This will contain configuration necessary to launch instances in AWS. +type NodeClassSpec struct { + // AMIFamily is the AMI family that instances use. + // +optional + AMIFamily *string `json:"amiFamily,omitempty"` + // UserData to be applied to the provisioned nodes. + // It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + // this UserData to ensure nodes are being provisioned with the correct configuration. + // +optional + UserData *string `json:"userData,omitempty"` + // Role is the AWS identity that nodes use. + // +optional + Role *string `json:"role,omitempty"` + // SubnetSelectorTerms is a list of or subnet selector terms. The terms are ORed. + // +optional + SubnetSelectorTerms []SubnetSelectorTerm `json:"subnetSelectorTerms" hash:"ignore"` + // SecurityGroupSelectorTerms is a list of or security group selector terms. The terms are ORed. + // +optional + SecurityGroupSelectorTerms []SecurityGroupSelectorTerm `json:"securityGroupSelectorTerms" hash:"ignore"` + // AMISelectorTerms is a list of or ami selector terms. The terms are ORed. + // +optional + AMISelectorTerms []AMISelectorTerm `json:"amiSelectorTerms,omitempty" hash:"ignore"` + // Context is a Reserved field in EC2 APIs + // https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + // +optional + Context *string `json:"context,omitempty"` + // Tags to be applied on ec2 resources like instances and launch templates. + // +optional + Tags map[string]string `json:"tags,omitempty"` + // DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched + // +optional + DetailedMonitoring *bool `json:"detailedMonitoring,omitempty"` + // MetadataOptions for the generated launch template of provisioned nodes. + // + // This specifies the exposure of the Instance Metadata Service to + // provisioned EC2 nodes. For more information, + // see Instance Metadata and User Data + // (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + // in the Amazon Elastic Compute Cloud User Guide. + // + // Refer to recommended, security best practices + // (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + // for limiting exposure of Instance Metadata and User Data to pods. + // If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + // disabled, with httpPutResponseLimit of 2, and with httpTokens + // required. + // +optional + MetadataOptions *MetadataOptions `json:"metadataOptions,omitempty"` + // BlockDeviceMappings to be applied to provisioned nodes. + // +optional + BlockDeviceMappings []*BlockDeviceMapping `json:"blockDeviceMappings,omitempty"` + // LaunchTemplateName for the node. If not specified, a launch template will be generated. + // NOTE: This field is for specifying a custom launch template and is exposed in the Spec + // as `launchTemplate` for backwards compatibility. + // +optional + LaunchTemplateName *string `json:"-" hash:"ignore"` +} + +// SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. +// If multiple fields are used for selection, the requirements are ANDed. +type SubnetSelectorTerm struct { + // Tags is a map of key/value tags used to select subnets + // Specifying '*' for a value selects all values for a given tag key. + // +optional + Tags map[string]string `json:"tags,omitempty"` + // ID is the subnet id in EC2 + // +kubebuilder:validation:Pattern="subnet-[0-9a-z]+" + // +optional + ID string `json:"id,omitempty"` +} + +// SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. +// If multiple fields are used for selection, the requirements are ANDed. +type SecurityGroupSelectorTerm struct { + // Tags is a map of key/value tags used to select subnets + // Specifying '*' for a value selects all values for a given tag key. + // +optional + Tags map[string]string `json:"tags,omitempty"` + // ID is the security group id in EC2 + // +kubebuilder:validation:Pattern:="sg-[0-9a-z]+" + // +optional + ID string `json:"id,omitempty"` + // Name is the security group name in EC2. + // This value is the name field, which is different from the name tag. + Name string `json:"name,omitempty"` +} + +// AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. +// If multiple fields are used for selection, the requirements are ANDed. +type AMISelectorTerm struct { + // Tags is a map of key/value tags used to select subnets + // Specifying '*' for a value selects all values for a given tag key. + // +optional + Tags map[string]string `json:"tags,omitempty"` + // ID is the ami id in EC2 + // +kubebuilder:validation:Pattern:="ami-[0-9a-z]+" + // +optional + ID string `json:"id,omitempty"` + // Name is the ami name in EC2. + // This value is thename field, which is different from the name tag. + // +optional + Name string `json:"name,omitempty"` + // Owner is the owner for the ami. + // You can specify a combination of AWS account IDs, "self", "amazon",and "aws-marketplace" + Owner string `json:"owner,omitempty"` + // SSM is the ssm alias for an ami. + SSM string `json:"ssm,omitempty"` +} + +// MetadataOptions contains parameters for specifying the exposure of the +// Instance Metadata Service to provisioned EC2 nodes. +type MetadataOptions struct { + // HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + // nodes. If metadata options is non-nil, but this parameter is not specified, + // the default state is "enabled". + // + // If you specify a value of "disabled", instance metadata will not be accessible + // on the node. + // +optional + HTTPEndpoint *string `json:"httpEndpoint,omitempty"` + // HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + // service on provisioned nodes. If metadata options is non-nil, but this parameter + // is not specified, the default state is "disabled". + // +optional + HTTPProtocolIPv6 *string `json:"httpProtocolIPv6,omitempty"` + // HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + // instance metadata requests. The larger the number, the further instance + // metadata requests can travel. Possible values are integers from 1 to 64. + // If metadata options is non-nil, but this parameter is not specified, the + // default value is 1. + // +optional + HTTPPutResponseHopLimit *int64 `json:"httpPutResponseHopLimit,omitempty"` + // HTTPTokens determines the state of token usage for instance metadata + // requests. If metadata options is non-nil, but this parameter is not + // specified, the default state is "optional". + // + // If the state is optional, one can choose to retrieve instance metadata with + // or without a signed token header on the request. If one retrieves the IAM + // role credentials without a token, the version 1.0 role credentials are + // returned. If one retrieves the IAM role credentials using a valid signed + // token, the version 2.0 role credentials are returned. + // + // If the state is "required", one must send a signed token header with any + // instance metadata retrieval requests. In this state, retrieving the IAM + // role credentials always returns the version 2.0 credentials; the version + // 1.0 credentials are not available. + // +optional + HTTPTokens *string `json:"httpTokens,omitempty"` +} + +type BlockDeviceMapping struct { + // The device name (for example, /dev/sdh or xvdh). + // +optional + DeviceName *string `json:"deviceName,omitempty"` + // EBS contains parameters used to automatically set up EBS volumes when an instance is launched. + // +optional + EBS *BlockDevice `json:"ebs,omitempty"` +} + +type BlockDevice struct { + // DeleteOnTermination indicates whether the EBS volume is deleted on instance termination. + // +optional + DeleteOnTermination *bool `json:"deleteOnTermination,omitempty"` + // Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + // be attached to instances that support Amazon EBS encryption. If you are creating + // a volume from a snapshot, you can't specify an encryption value. + // +optional + Encrypted *bool `json:"encrypted,omitempty"` + // IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + // this represents the number of IOPS that are provisioned for the volume. For + // gp2 volumes, this represents the baseline performance of the volume and the + // rate at which the volume accumulates I/O credits for bursting. + // + // The following are the supported values for each volume type: + // + // * gp3: 3,000-16,000 IOPS + // + // * io1: 100-64,000 IOPS + // + // * io2: 100-64,000 IOPS + // + // For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + // on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + // Other instance families guarantee performance up to 32,000 IOPS. + // + // This parameter is supported for io1, io2, and gp3 volumes only. This parameter + // is not supported for gp2, st1, sc1, or standard volumes. + // +optional + IOPS *int64 `json:"iops,omitempty"` + // KMSKeyID (ARN) of the symmetric Key Management Service (KMS) CMK used for encryption. + // +optional + KMSKeyID *string `json:"kmsKeyID,omitempty"` + // SnapshotID is the ID of an EBS snapshot + // +optional + SnapshotID *string `json:"snapshotID,omitempty"` + // Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + // Valid Range: Minimum value of 125. Maximum value of 1000. + // +optional + Throughput *int64 `json:"throughput,omitempty"` + // VolumeSize in GiBs. You must specify either a snapshot ID or + // a volume size. The following are the supported volumes sizes for each volume + // type: + // + // * gp2 and gp3: 1-16,384 + // + // * io1 and io2: 4-16,384 + // + // * st1 and sc1: 125-16,384 + // + // * standard: 1-1,024 + // +optional + VolumeSize *resource.Quantity `json:"volumeSize,omitempty" hash:"string"` + // VolumeType of the block device. + // For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + // in the Amazon Elastic Compute Cloud User Guide. + // +optional + VolumeType *string `json:"volumeType,omitempty"` +} + +// NodeClass is the Schema for the NodeClass API +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=nodeclasses,scope=Cluster,categories=karpenter +// +kubebuilder:subresource:status +type NodeClass struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NodeClassSpec `json:"spec,omitempty"` + Status NodeClassStatus `json:"status,omitempty"` +} + +func (a *NodeClass) Hash() string { + hash, _ := hashstructure.Hash(a.Spec, hashstructure.FormatV2, &hashstructure.HashOptions{ + SlicesAsSets: true, + IgnoreZeroValue: true, + ZeroNil: true, + }) + + return fmt.Sprint(hash) +} + +// NodeClassList contains a list of NodeClass +// +kubebuilder:object:root=true +type NodeClassList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NodeClass `json:"items"` +} diff --git a/pkg/apis/v1beta1/nodeclass_defaults.go b/pkg/apis/v1beta1/nodeclass_defaults.go new file mode 100644 index 000000000000..5f980997e6cb --- /dev/null +++ b/pkg/apis/v1beta1/nodeclass_defaults.go @@ -0,0 +1,22 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" +) + +// SetDefaults for the NodeClass +func (a *NodeClass) SetDefaults(_ context.Context) {} diff --git a/pkg/apis/v1beta1/nodeclass_status.go b/pkg/apis/v1beta1/nodeclass_status.go new file mode 100644 index 000000000000..f2c1854bcf52 --- /dev/null +++ b/pkg/apis/v1beta1/nodeclass_status.go @@ -0,0 +1,66 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import v1 "k8s.io/api/core/v1" + +// Subnet contains resolved Subnet selector values utilized for node launch +type Subnet struct { + // ID of the subnet + // +required + ID string `json:"id"` + // The associated availability zone + // +required + Zone string `json:"zone"` +} + +// SecurityGroup contains resolved SecurityGroup selector values utilized for node launch +type SecurityGroup struct { + // ID of the security group + // +required + ID string `json:"id"` + // Name of the security group + // +optional + Name string `json:"name,omitempty"` +} + +// AMI contains resolved AMI selector values utilized for node launch +type AMI struct { + // ID of the AMI + // +required + ID string `json:"id"` + // Name of the AMI + // +optional + Name string `json:"name,omitempty"` + // Requirements of the AMI to be utilized on an instance type + // +required + Requirements []v1.NodeSelectorRequirement `json:"requirements"` +} + +// NodeClassStatus contains the resolved state of the NodeClass +type NodeClassStatus struct { + // Subnets contains the current Subnet values that are available to the + // cluster under the subnet selectors. + // +optional + Subnets []Subnet `json:"subnets,omitempty"` + // SecurityGroups contains the current Security Groups values that are available to the + // cluster under the SecurityGroups selectors. + // +optional + SecurityGroups []SecurityGroup `json:"securityGroups,omitempty"` + // AMI contains the current AMI values that are available to the + // cluster under the AMI selectors. + // +optional + AMIs []AMI `json:"amis,omitempty"` +} diff --git a/pkg/apis/v1beta1/nodeclass_validation.go b/pkg/apis/v1beta1/nodeclass_validation.go new file mode 100644 index 000000000000..d3ea74fa2637 --- /dev/null +++ b/pkg/apis/v1beta1/nodeclass_validation.go @@ -0,0 +1,257 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/samber/lo" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + "k8s.io/apimachinery/pkg/api/resource" + "knative.dev/pkg/apis" +) + +const ( + userDataPath = "userData" + subnetSelectorTermsPath = "subnetSelectorTerms" + securityGroupSelectorTermsPath = "securityGroupSelectorTerms" + amiSelectorTermsPath = "amiSelectorTerms" + amiFamilyPath = "amiFamily" + tagsPath = "tags" + metadataOptionsPath = "metadataOptions" + blockDeviceMappingsPath = "blockDeviceMappings" +) + +var ( + minVolumeSize = *resource.NewScaledQuantity(1, resource.Giga) + maxVolumeSize = *resource.NewScaledQuantity(64, resource.Tera) +) + +func (a *NodeClass) SupportedVerbs() []admissionregistrationv1.OperationType { + return []admissionregistrationv1.OperationType{ + admissionregistrationv1.Create, + admissionregistrationv1.Update, + } +} + +func (a *NodeClass) Validate(ctx context.Context) (errs *apis.FieldError) { + return errs.Also( + apis.ValidateObjectMetadata(a).ViaField("metadata"), + a.Spec.validate(ctx).ViaField("spec"), + ) +} + +func (in *NodeClassSpec) validate(_ context.Context) (errs *apis.FieldError) { + return errs.Also( + in.validateSubnetSelectorTerms().ViaField(subnetSelectorTermsPath), + in.validateSecurityGroupSelectorTerms().ViaField(securityGroupSelectorTermsPath), + in.validateAMISelectorTerms().ViaField(amiSelectorTermsPath), + in.validateMetadataOptions().ViaField(metadataOptionsPath), + in.validateAMIFamily().ViaField(amiFamilyPath), + in.validateBlockDeviceMappings().ViaField(blockDeviceMappingsPath), + in.validateUserData().ViaField(userDataPath), + in.validateTags().ViaField(tagsPath), + ) +} + +func (in *NodeClassSpec) validateSubnetSelectorTerms() (errs *apis.FieldError) { + for i, term := range in.SubnetSelectorTerms { + errs = errs.Also(term.validate()).ViaIndex(i) + } + return errs +} + +func (in *SubnetSelectorTerm) validate() (errs *apis.FieldError) { + return errs.Also(validateTags(in.Tags).ViaField("tags")) +} + +func (in *NodeClassSpec) validateSecurityGroupSelectorTerms() (errs *apis.FieldError) { + for _, term := range in.SecurityGroupSelectorTerms { + errs = errs.Also(term.validate()) + } + return errs +} + +func (in *SecurityGroupSelectorTerm) validate() (errs *apis.FieldError) { + return errs.Also(validateTags(in.Tags).ViaField("tags")) +} + +func (in *NodeClassSpec) validateAMISelectorTerms() (errs *apis.FieldError) { + for _, term := range in.AMISelectorTerms { + errs = errs.Also(term.validate()) + } + return errs +} + +func (in *AMISelectorTerm) validate() (errs *apis.FieldError) { + return errs.Also(validateTags(in.Tags).ViaField("tags")) +} + +func validateTags(m map[string]string) (errs *apis.FieldError) { + for k, v := range m { + if k == "" { + errs = errs.Also(apis.ErrInvalidKeyName(`""`, "")) + } + if v == "" { + errs = errs.Also(apis.ErrInvalidValue(`""`, k)) + } + } + return errs +} + +func (in *NodeClassSpec) validateMetadataOptions() (errs *apis.FieldError) { + if in.MetadataOptions == nil { + return nil + } + return errs.Also( + in.validateHTTPEndpoint(), + in.validateHTTPProtocolIpv6(), + in.validateHTTPPutResponseHopLimit(), + in.validateHTTPTokens(), + ) +} + +func (in *NodeClassSpec) validateHTTPEndpoint() *apis.FieldError { + if in.MetadataOptions.HTTPEndpoint == nil { + return nil + } + return in.validateStringEnum(*in.MetadataOptions.HTTPEndpoint, "httpEndpoint", ec2.LaunchTemplateInstanceMetadataEndpointState_Values()) +} + +func (in *NodeClassSpec) validateHTTPProtocolIpv6() *apis.FieldError { + if in.MetadataOptions.HTTPProtocolIPv6 == nil { + return nil + } + return in.validateStringEnum(*in.MetadataOptions.HTTPProtocolIPv6, "httpProtocolIPv6", ec2.LaunchTemplateInstanceMetadataProtocolIpv6_Values()) +} + +func (in *NodeClassSpec) validateHTTPPutResponseHopLimit() *apis.FieldError { + if in.MetadataOptions.HTTPPutResponseHopLimit == nil { + return nil + } + limit := *in.MetadataOptions.HTTPPutResponseHopLimit + if limit < 1 || limit > 64 { + return apis.ErrOutOfBoundsValue(limit, 1, 64, "httpPutResponseHopLimit") + } + return nil +} + +func (in *NodeClassSpec) validateHTTPTokens() *apis.FieldError { + if in.MetadataOptions.HTTPTokens == nil { + return nil + } + return in.validateStringEnum(*in.MetadataOptions.HTTPTokens, "httpTokens", ec2.LaunchTemplateHttpTokensState_Values()) +} + +func (in *NodeClassSpec) validateStringEnum(value, field string, validValues []string) *apis.FieldError { + for _, validValue := range validValues { + if value == validValue { + return nil + } + } + return apis.ErrInvalidValue(fmt.Sprintf("%s not in %v", value, strings.Join(validValues, ", ")), field) +} + +func (in *NodeClassSpec) validateBlockDeviceMappings() (errs *apis.FieldError) { + for i, blockDeviceMapping := range in.BlockDeviceMappings { + if err := in.validateBlockDeviceMapping(blockDeviceMapping); err != nil { + errs = errs.Also(err.ViaFieldIndex(blockDeviceMappingsPath, i)) + } + } + return errs +} + +func (in *NodeClassSpec) validateBlockDeviceMapping(blockDeviceMapping *BlockDeviceMapping) (errs *apis.FieldError) { + return errs.Also(in.validateDeviceName(blockDeviceMapping), in.validateEBS(blockDeviceMapping)) +} + +func (in *NodeClassSpec) validateDeviceName(blockDeviceMapping *BlockDeviceMapping) *apis.FieldError { + if blockDeviceMapping.DeviceName == nil { + return apis.ErrMissingField("deviceName") + } + return nil +} + +func (in *NodeClassSpec) validateEBS(blockDeviceMapping *BlockDeviceMapping) (errs *apis.FieldError) { + if blockDeviceMapping.EBS == nil { + return apis.ErrMissingField("ebs") + } + for _, err := range []*apis.FieldError{ + in.validateVolumeType(blockDeviceMapping), + in.validateVolumeSize(blockDeviceMapping), + } { + if err != nil { + errs = errs.Also(err.ViaField("ebs")) + } + } + return errs +} + +func (in *NodeClassSpec) validateVolumeType(blockDeviceMapping *BlockDeviceMapping) *apis.FieldError { + if blockDeviceMapping.EBS.VolumeType != nil { + return in.validateStringEnum(*blockDeviceMapping.EBS.VolumeType, "volumeType", ec2.VolumeType_Values()) + } + return nil +} + +func (in *NodeClassSpec) validateVolumeSize(blockDeviceMapping *BlockDeviceMapping) *apis.FieldError { + // If an EBS mapping is present, one of volumeSize or snapshotID must be present + if blockDeviceMapping.EBS.SnapshotID != nil && blockDeviceMapping.EBS.VolumeSize == nil { + return nil + } else if blockDeviceMapping.EBS.VolumeSize == nil { + return apis.ErrMissingField("volumeSize") + } else if blockDeviceMapping.EBS.VolumeSize.Cmp(minVolumeSize) == -1 || blockDeviceMapping.EBS.VolumeSize.Cmp(maxVolumeSize) == 1 { + return apis.ErrOutOfBoundsValue(blockDeviceMapping.EBS.VolumeSize.String(), minVolumeSize.String(), maxVolumeSize.String(), "volumeSize") + } + return nil +} + +func (in *NodeClassSpec) validateUserData() (errs *apis.FieldError) { + if in.UserData == nil { + return nil + } + if lo.FromPtr(in.AMIFamily) == AMIFamilyWindows2019 || lo.FromPtr(in.AMIFamily) == AMIFamilyWindows2022 { + errs = errs.Also(apis.ErrGeneric(fmt.Sprintf("%s AMIFamily is not currently supported with custom userData", lo.FromPtr(in.AMIFamily)), userDataPath)) + } + return errs +} + +func (in *NodeClassSpec) validateAMIFamily() (errs *apis.FieldError) { + if in.AMIFamily == nil { + return nil + } + if *in.AMIFamily == AMIFamilyCustom && len(in.AMISelectorTerms) == 0 { + errs = errs.Also(apis.ErrMissingField(amiSelectorTermsPath)) + } + return errs.Also(in.validateStringEnum(*in.AMIFamily, amiFamilyPath, SupportedAMIFamilies)) +} + +func (in *NodeClassSpec) validateTags() (errs *apis.FieldError) { + for k, v := range in.Tags { + if k == "" { + errs = errs.Also(apis.ErrInvalidValue(fmt.Sprintf( + "the tag with key : '' and value : '%s' is invalid because empty tag keys aren't supported", v), "tags")) + } + for _, pattern := range RestrictedTagPatterns { + if pattern.MatchString(k) { + errs = errs.Also(apis.ErrInvalidKeyName(k, "tags", fmt.Sprintf("tag contains in restricted tag matching %q", pattern.String()))) + } + } + } + return errs +} diff --git a/pkg/apis/v1beta1/register.go b/pkg/apis/v1beta1/register.go new file mode 100644 index 000000000000..df536ce6cf40 --- /dev/null +++ b/pkg/apis/v1beta1/register.go @@ -0,0 +1,152 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "fmt" + "regexp" + + "github.com/aws/aws-sdk-go/service/ec2" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + "github.com/aws/karpenter-core/pkg/apis/v1beta1" +) + +const Group = "compute.k8s.aws" + +type AcceleratorManufacturer string + +var ( + CapacityTypeSpot = ec2.DefaultTargetCapacityTypeSpot + CapacityTypeOnDemand = ec2.DefaultTargetCapacityTypeOnDemand + AWSToKubeArchitectures = map[string]string{ + "x86_64": v1beta1.ArchitectureAmd64, + v1beta1.ArchitectureArm64: v1beta1.ArchitectureArm64, + } + WellKnownArchitectures = sets.NewString( + v1beta1.ArchitectureAmd64, + v1beta1.ArchitectureArm64, + ) + RestrictedLabelDomains = []string{ + Group, + } + RestrictedTagPatterns = []*regexp.Regexp{ + // Adheres to cluster name pattern matching as specified in the API spec + // https://docs.aws.amazon.com/eks/latest/APIReference/API_CreateCluster.html + regexp.MustCompile(`^kubernetes\.io/cluster/[0-9A-Za-z][A-Za-z0-9\-_]*$`), + regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(v1alpha5.ProvisionerNameLabelKey))), + regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(v1alpha5.MachineManagedByAnnotationKey))), + regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(v1beta1.NodePoolLabelKey))), + regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(v1beta1.ManagedByAnnotationKey))), + } + AMIFamilyBottlerocket = "Bottlerocket" + AMIFamilyAL2 = "AL2" + AMIFamilyUbuntu = "Ubuntu" + AMIFamilyWindows2019 = "Windows2019" + AMIFamilyWindows2022 = "Windows2022" + AMIFamilyCustom = "Custom" + SupportedAMIFamilies = []string{ + AMIFamilyBottlerocket, + AMIFamilyAL2, + AMIFamilyUbuntu, + AMIFamilyWindows2019, + AMIFamilyWindows2022, + AMIFamilyCustom, + } + SupportedContainerRuntimesByAMIFamily = map[string]sets.Set[string]{ + AMIFamilyBottlerocket: sets.New("containerd"), + AMIFamilyAL2: sets.New("dockerd", "containerd"), + AMIFamilyUbuntu: sets.New("dockerd", "containerd"), + AMIFamilyWindows2019: sets.New("dockerd", "containerd"), + AMIFamilyWindows2022: sets.New("dockerd", "containerd"), + } + + Windows2019 = "2019" + Windows2022 = "2022" + WindowsCore = "Core" + Windows2019Build = "10.0.17763" + Windows2022Build = "10.0.20348" + ResourceNVIDIAGPU v1.ResourceName = "nvidia.com/gpu" + ResourceAMDGPU v1.ResourceName = "amd.com/gpu" + ResourceAWSNeuron v1.ResourceName = "aws.amazon.com/neuron" + ResourceHabanaGaudi v1.ResourceName = "habana.ai/gaudi" + ResourceAWSPodENI v1.ResourceName = "vpc.amazonaws.com/pod-eni" + ResourcePrivateIPv4Address v1.ResourceName = "vpc.amazonaws.com/PrivateIPv4Address" + NVIDIAacceleratorManufacturer AcceleratorManufacturer = "nvidia" + AWSAcceleratorManufacturer AcceleratorManufacturer = "aws" + + LabelInstanceHypervisor = Group + "/instance-hypervisor" + LabelInstanceEncryptionInTransitSupported = Group + "/instance-encryption-in-transit-supported" + LabelInstanceCategory = Group + "/instance-category" + LabelInstanceFamily = Group + "/instance-family" + LabelInstanceGeneration = Group + "/instance-generation" + LabelInstanceLocalNVME = Group + "/instance-local-nvme" + LabelInstanceSize = Group + "/instance-size" + LabelInstanceCPU = Group + "/instance-cpu" + LabelInstanceMemory = Group + "/instance-memory" + LabelInstanceNetworkBandwidth = Group + "/instance-network-bandwidth" + LabelInstancePods = Group + "/instance-pods" + LabelInstanceGPUName = Group + "/instance-gpu-name" + LabelInstanceGPUManufacturer = Group + "/instance-gpu-manufacturer" + LabelInstanceGPUCount = Group + "/instance-gpu-count" + LabelInstanceGPUMemory = Group + "/instance-gpu-memory" + LabelInstanceAMIID = Group + "/instance-ami-id" + LabelInstanceAcceleratorName = Group + "/instance-accelerator-name" + LabelInstanceAcceleratorManufacturer = Group + "/instance-accelerator-manufacturer" + LabelInstanceAcceleratorCount = Group + "/instance-accelerator-count" + AnnotationNodeTemplateHash = Group + "/nodetemplate-hash" +) + +var ( + SchemeGroupVersion = schema.GroupVersion{Group: Group, Version: "v1beta1"} + SchemeBuilder = runtime.NewSchemeBuilder(func(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &NodeClass{}, + &NodeClassList{}, + ) + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil + }) +) + +func init() { + v1beta1.RestrictedLabelDomains = v1beta1.RestrictedLabelDomains.Insert(RestrictedLabelDomains...) + v1beta1.WellKnownLabels = v1beta1.WellKnownLabels.Insert( + LabelInstanceHypervisor, + LabelInstanceEncryptionInTransitSupported, + LabelInstanceCategory, + LabelInstanceFamily, + LabelInstanceGeneration, + LabelInstanceSize, + LabelInstanceLocalNVME, + LabelInstanceCPU, + LabelInstanceMemory, + LabelInstanceNetworkBandwidth, + LabelInstancePods, + LabelInstanceGPUName, + LabelInstanceGPUManufacturer, + LabelInstanceGPUCount, + LabelInstanceGPUMemory, + LabelInstanceAcceleratorName, + LabelInstanceAcceleratorManufacturer, + LabelInstanceAcceleratorCount, + v1.LabelWindowsBuild, + ) +} diff --git a/pkg/apis/v1beta1/suite_test.go b/pkg/apis/v1beta1/suite_test.go new file mode 100644 index 000000000000..4d9bcd9e5c04 --- /dev/null +++ b/pkg/apis/v1beta1/suite_test.go @@ -0,0 +1,201 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1_test + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/Pallinder/go-randomdata" + "github.com/mitchellh/hashstructure/v2" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "knative.dev/pkg/logging/testing" + "knative.dev/pkg/ptr" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/aws/aws-sdk-go/aws" + + "github.com/aws/karpenter/pkg/apis/v1alpha1" + "github.com/aws/karpenter/pkg/test" +) + +var ctx context.Context + +func TestAPIs(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "Validation") +} + +var _ = Describe("Validation", func() { + var ant *v1alpha1.AWSNodeTemplate + + BeforeEach(func() { + ant = &v1alpha1.AWSNodeTemplate{ + ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, + Spec: v1alpha1.AWSNodeTemplateSpec{ + AWS: v1alpha1.AWS{ + SubnetSelector: map[string]string{"foo": "bar"}, + SecurityGroupSelector: map[string]string{"foo": "bar"}, + }, + }, + } + }) + + Context("UserData", func() { + It("should succeed if user data is empty", func() { + Expect(ant.Validate(ctx)).To(Succeed()) + }) + It("should fail if launch template is also specified", func() { + ant.Spec.LaunchTemplateName = ptr.String("someLaunchTemplate") + ant.Spec.UserData = ptr.String("someUserData") + Expect(ant.Validate(ctx)).To(Not(Succeed())) + }) + It("should fail if Windows2019 AMIFamily is specified", func() { + ant.Spec.AMIFamily = &v1alpha1.AMIFamilyWindows2019 + ant.Spec.UserData = ptr.String("someUserData") + Expect(ant.Validate(ctx)).To(Not(Succeed())) + }) + It("should fail if Windows2022 AMIFamily is specified", func() { + ant.Spec.AMIFamily = &v1alpha1.AMIFamilyWindows2022 + ant.Spec.UserData = ptr.String("someUserData") + Expect(ant.Validate(ctx)).To(Not(Succeed())) + }) + }) + Context("Tags", func() { + It("should succeed when tags are empty", func() { + ant.Spec.Tags = map[string]string{} + Expect(ant.Validate(ctx)).To(Succeed()) + }) + It("should succeed if tags aren't in restricted tag keys", func() { + ant.Spec.Tags = map[string]string{ + "karpenter.sh/custom-key": "value", + "karpenter.sh/managed": "true", + "kubernetes.io/role/key": "value", + } + Expect(ant.Validate(ctx)).To(Succeed()) + }) + It("should succeed by validating that regex is properly escaped", func() { + ant.Spec.Tags = map[string]string{ + "karpenterzsh/provisioner-name": "value", + } + Expect(ant.Validate(ctx)).To(Succeed()) + ant.Spec.Tags = map[string]string{ + "kubernetesbio/cluster/test": "value", + } + Expect(ant.Validate(ctx)).To(Succeed()) + ant.Spec.Tags = map[string]string{ + "karpenterzsh/managed-by": "test", + } + Expect(ant.Validate(ctx)).To(Succeed()) + }) + It("should fail if tags contain a restricted domain key", func() { + ant.Spec.Tags = map[string]string{ + "karpenter.sh/provisioner-name": "value", + } + Expect(ant.Validate(ctx)).To(Not(Succeed())) + ant.Spec.Tags = map[string]string{ + "kubernetes.io/cluster/test": "value", + } + Expect(ant.Validate(ctx)).To(Not(Succeed())) + ant.Spec.Tags = map[string]string{ + "karpenter.sh/managed-by": "test", + } + Expect(ant.Validate(ctx)).To(Not(Succeed())) + }) + }) + var _ = Describe("NodeClass Hash", func() { + var awsnodetemplatespec v1alpha1.AWSNodeTemplateSpec + var awsnodetemplate *v1alpha1.AWSNodeTemplate + BeforeEach(func() { + awsnodetemplatespec = v1alpha1.AWSNodeTemplateSpec{ + AWS: v1alpha1.AWS{ + AMIFamily: aws.String(v1alpha1.AMIFamilyAL2), + Context: aws.String("context-1"), + InstanceProfile: aws.String("profile-1"), + Tags: map[string]string{ + "keyTag-1": "valueTag-1", + "keyTag-2": "valueTag-2", + }, + LaunchTemplate: v1alpha1.LaunchTemplate{ + MetadataOptions: &v1alpha1.MetadataOptions{ + HTTPEndpoint: aws.String("test-metadata-1"), + }, + BlockDeviceMappings: []*v1alpha1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + }, + { + DeviceName: aws.String("map-device-2"), + }, + }, + }, + }, + UserData: aws.String("userdata-test-1"), + DetailedMonitoring: aws.Bool(false), + } + awsnodetemplate = test.AWSNodeTemplate(awsnodetemplatespec) + }) + DescribeTable("should change hash when static fields are updated", func(awsnodetemplatespec v1alpha1.AWSNodeTemplateSpec) { + expectedHash := awsnodetemplate.Hash() + updatedAWSNodeTemplate := test.AWSNodeTemplate(*awsnodetemplatespec.DeepCopy(), awsnodetemplatespec) + actualHash := updatedAWSNodeTemplate.Hash() + Expect(actualHash).ToNot(Equal(fmt.Sprint(expectedHash))) + }, + Entry("InstanceProfile Drift", v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{InstanceProfile: aws.String("profile-2")}}), + Entry("UserData Drift", v1alpha1.AWSNodeTemplateSpec{UserData: aws.String("userdata-test-2")}), + Entry("Tags Drift", v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), + Entry("MetadataOptions Drift", v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{LaunchTemplate: v1alpha1.LaunchTemplate{MetadataOptions: &v1alpha1.MetadataOptions{HTTPEndpoint: aws.String("test-metadata-2")}}}}), + Entry("BlockDeviceMappings Drift", v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{LaunchTemplate: v1alpha1.LaunchTemplate{BlockDeviceMappings: []*v1alpha1.BlockDeviceMapping{{DeviceName: aws.String("map-device-test-3")}}}}}), + Entry("Context Drift", v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{Context: aws.String("context-2")}}), + Entry("DetailedMonitoring Drift", v1alpha1.AWSNodeTemplateSpec{DetailedMonitoring: aws.Bool(true)}), + Entry("AMIFamily Drift", v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{AMIFamily: aws.String(v1alpha1.AMIFamilyBottlerocket)}}), + Entry("Reorder Tags", v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{Tags: map[string]string{"keyTag-2": "valueTag-2", "keyTag-1": "valueTag-1"}}}), + Entry("Reorder BlockDeviceMapping", v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{LaunchTemplate: v1alpha1.LaunchTemplate{BlockDeviceMappings: []*v1alpha1.BlockDeviceMapping{{DeviceName: aws.String("map-device-2")}, {DeviceName: aws.String("map-device-1")}}}}}), + ) + It("should not change hash when behavior/dynamic fields are updated", func() { + actualHash := awsnodetemplate.Hash() + + expectedHash, err := hashstructure.Hash(awsnodetemplate.Spec, hashstructure.FormatV2, &hashstructure.HashOptions{ + SlicesAsSets: true, + IgnoreZeroValue: true, + ZeroNil: true, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(actualHash).To(Equal(fmt.Sprint(expectedHash))) + + // Update a behavior/dynamic field + awsnodetemplate.Spec.SubnetSelector = map[string]string{"subnet-test-key": "subnet-test-value"} + awsnodetemplate.Spec.SecurityGroupSelector = map[string]string{"sg-test-key": "sg-test-value"} + awsnodetemplate.Spec.AMISelector = map[string]string{"ami-test-key": "ami-test-value"} + + actualHash = awsnodetemplate.Hash() + Expect(err).ToNot(HaveOccurred()) + Expect(actualHash).To(Equal(fmt.Sprint(expectedHash))) + }) + It("should expect two provisioner with the same spec to have the same provisioner hash", func() { + awsnodetemplateTwo := &v1alpha1.AWSNodeTemplate{ + ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, + } + awsnodetemplateTwo.Spec = awsnodetemplatespec + + Expect(awsnodetemplate.Hash()).To(Equal(awsnodetemplateTwo.Hash())) + }) + }) +}) diff --git a/pkg/apis/v1beta1/zz_generated.deepcopy.go b/pkg/apis/v1beta1/zz_generated.deepcopy.go new file mode 100644 index 000000000000..7f5b153cdbb1 --- /dev/null +++ b/pkg/apis/v1beta1/zz_generated.deepcopy.go @@ -0,0 +1,438 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1beta1 + +import ( + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AMI) DeepCopyInto(out *AMI) { + *out = *in + if in.Requirements != nil { + in, out := &in.Requirements, &out.Requirements + *out = make([]v1.NodeSelectorRequirement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AMI. +func (in *AMI) DeepCopy() *AMI { + if in == nil { + return nil + } + out := new(AMI) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AMISelectorTerm) DeepCopyInto(out *AMISelectorTerm) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AMISelectorTerm. +func (in *AMISelectorTerm) DeepCopy() *AMISelectorTerm { + if in == nil { + return nil + } + out := new(AMISelectorTerm) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BlockDevice) DeepCopyInto(out *BlockDevice) { + *out = *in + if in.DeleteOnTermination != nil { + in, out := &in.DeleteOnTermination, &out.DeleteOnTermination + *out = new(bool) + **out = **in + } + if in.Encrypted != nil { + in, out := &in.Encrypted, &out.Encrypted + *out = new(bool) + **out = **in + } + if in.IOPS != nil { + in, out := &in.IOPS, &out.IOPS + *out = new(int64) + **out = **in + } + if in.KMSKeyID != nil { + in, out := &in.KMSKeyID, &out.KMSKeyID + *out = new(string) + **out = **in + } + if in.SnapshotID != nil { + in, out := &in.SnapshotID, &out.SnapshotID + *out = new(string) + **out = **in + } + if in.Throughput != nil { + in, out := &in.Throughput, &out.Throughput + *out = new(int64) + **out = **in + } + if in.VolumeSize != nil { + in, out := &in.VolumeSize, &out.VolumeSize + x := (*in).DeepCopy() + *out = &x + } + if in.VolumeType != nil { + in, out := &in.VolumeType, &out.VolumeType + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlockDevice. +func (in *BlockDevice) DeepCopy() *BlockDevice { + if in == nil { + return nil + } + out := new(BlockDevice) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BlockDeviceMapping) DeepCopyInto(out *BlockDeviceMapping) { + *out = *in + if in.DeviceName != nil { + in, out := &in.DeviceName, &out.DeviceName + *out = new(string) + **out = **in + } + if in.EBS != nil { + in, out := &in.EBS, &out.EBS + *out = new(BlockDevice) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlockDeviceMapping. +func (in *BlockDeviceMapping) DeepCopy() *BlockDeviceMapping { + if in == nil { + return nil + } + out := new(BlockDeviceMapping) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetadataOptions) DeepCopyInto(out *MetadataOptions) { + *out = *in + if in.HTTPEndpoint != nil { + in, out := &in.HTTPEndpoint, &out.HTTPEndpoint + *out = new(string) + **out = **in + } + if in.HTTPProtocolIPv6 != nil { + in, out := &in.HTTPProtocolIPv6, &out.HTTPProtocolIPv6 + *out = new(string) + **out = **in + } + if in.HTTPPutResponseHopLimit != nil { + in, out := &in.HTTPPutResponseHopLimit, &out.HTTPPutResponseHopLimit + *out = new(int64) + **out = **in + } + if in.HTTPTokens != nil { + in, out := &in.HTTPTokens, &out.HTTPTokens + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetadataOptions. +func (in *MetadataOptions) DeepCopy() *MetadataOptions { + if in == nil { + return nil + } + out := new(MetadataOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeClass) DeepCopyInto(out *NodeClass) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeClass. +func (in *NodeClass) DeepCopy() *NodeClass { + if in == nil { + return nil + } + out := new(NodeClass) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NodeClass) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeClassList) DeepCopyInto(out *NodeClassList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NodeClass, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeClassList. +func (in *NodeClassList) DeepCopy() *NodeClassList { + if in == nil { + return nil + } + out := new(NodeClassList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NodeClassList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeClassSpec) DeepCopyInto(out *NodeClassSpec) { + *out = *in + if in.AMIFamily != nil { + in, out := &in.AMIFamily, &out.AMIFamily + *out = new(string) + **out = **in + } + if in.UserData != nil { + in, out := &in.UserData, &out.UserData + *out = new(string) + **out = **in + } + if in.Role != nil { + in, out := &in.Role, &out.Role + *out = new(string) + **out = **in + } + if in.SubnetSelectorTerms != nil { + in, out := &in.SubnetSelectorTerms, &out.SubnetSelectorTerms + *out = make([]SubnetSelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.SecurityGroupSelectorTerms != nil { + in, out := &in.SecurityGroupSelectorTerms, &out.SecurityGroupSelectorTerms + *out = make([]SecurityGroupSelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.AMISelectorTerms != nil { + in, out := &in.AMISelectorTerms, &out.AMISelectorTerms + *out = make([]AMISelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Context != nil { + in, out := &in.Context, &out.Context + *out = new(string) + **out = **in + } + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.DetailedMonitoring != nil { + in, out := &in.DetailedMonitoring, &out.DetailedMonitoring + *out = new(bool) + **out = **in + } + if in.MetadataOptions != nil { + in, out := &in.MetadataOptions, &out.MetadataOptions + *out = new(MetadataOptions) + (*in).DeepCopyInto(*out) + } + if in.BlockDeviceMappings != nil { + in, out := &in.BlockDeviceMappings, &out.BlockDeviceMappings + *out = make([]*BlockDeviceMapping, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(BlockDeviceMapping) + (*in).DeepCopyInto(*out) + } + } + } + if in.LaunchTemplateName != nil { + in, out := &in.LaunchTemplateName, &out.LaunchTemplateName + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeClassSpec. +func (in *NodeClassSpec) DeepCopy() *NodeClassSpec { + if in == nil { + return nil + } + out := new(NodeClassSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeClassStatus) DeepCopyInto(out *NodeClassStatus) { + *out = *in + if in.Subnets != nil { + in, out := &in.Subnets, &out.Subnets + *out = make([]Subnet, len(*in)) + copy(*out, *in) + } + if in.SecurityGroups != nil { + in, out := &in.SecurityGroups, &out.SecurityGroups + *out = make([]SecurityGroup, len(*in)) + copy(*out, *in) + } + if in.AMIs != nil { + in, out := &in.AMIs, &out.AMIs + *out = make([]AMI, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeClassStatus. +func (in *NodeClassStatus) DeepCopy() *NodeClassStatus { + if in == nil { + return nil + } + out := new(NodeClassStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityGroup) DeepCopyInto(out *SecurityGroup) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityGroup. +func (in *SecurityGroup) DeepCopy() *SecurityGroup { + if in == nil { + return nil + } + out := new(SecurityGroup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityGroupSelectorTerm) DeepCopyInto(out *SecurityGroupSelectorTerm) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityGroupSelectorTerm. +func (in *SecurityGroupSelectorTerm) DeepCopy() *SecurityGroupSelectorTerm { + if in == nil { + return nil + } + out := new(SecurityGroupSelectorTerm) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Subnet) DeepCopyInto(out *Subnet) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Subnet. +func (in *Subnet) DeepCopy() *Subnet { + if in == nil { + return nil + } + out := new(Subnet) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetSelectorTerm) DeepCopyInto(out *SubnetSelectorTerm) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetSelectorTerm. +func (in *SubnetSelectorTerm) DeepCopy() *SubnetSelectorTerm { + if in == nil { + return nil + } + out := new(SubnetSelectorTerm) + in.DeepCopyInto(out) + return out +}