feat: dedicated hosts selectors

cmd/controller/main.go

@@ -66,6 +66,9 @@ func main() {
 			op.InstanceProfileProvider,
 			op.PricingProvider,
 			op.AMIProvider,
+			op.LicenseProvider,
+			op.HostResourceGroupProvider,
+			op.PlacementGroupProvider,
 		)...).
 		WithWebhooks(ctx, webhooks.NewWebhooks()...).
 		Start(ctx)

designs/dedicatedhosts.md

@@ -0,0 +1,107 @@
+# Dedicated host support
+## Background
+
+As described in [3182](https://github.com/aws/karpenter/issues/3182) AWS provides the 
+ability to launch EC2 instances on [Dedicated Host hardware](https://docs.aws.amazon.com/license-manager/latest/userguide/host-resource-groups.html)
+Dedicated hosts are often used to pay for ec2 usage as capital expenditure (CapEx)
+rather than an operating expense (OpEx). This is often useful for publicly listed
+companies that want to manage their revenue to OpEx ratio.
+
+AWS allows the allocation of ec2 instances to Dedidcated Host Host Resource Groups (HRG)
+through the use of launchtemplates. 
+Detailed documentation for launchtemplates and host resource groups can be found here:
+- https://docs.aws.amazon.com/autoscaling/ec2/userguide/create-launch-template.html#advanced-settings-for-your-launch-template
+- https://docs.aws.amazon.com/license-manager/latest/userguide/host-resource-groups.html
+
+## Solutions
+
+aws-karpenter already supports a set of [LaunchTemplate configuration](https://github.com/aws/karpenter/blob/main/pkg/apis/v1alpha1/awsnodetemplate.go)
+with design document: https://github.com/aws/karpenter/blob/main/designs/aws-launch-templates-v2.md
+
+1. Implement the minimal fields from https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-launchtemplate-launchtemplatedata-placement.html
+specifically `placement.hostResourceGroupArn` and `licenseConfiguration`
+
+this would look like:
+
+```
+apiVersion: karpenter.k8s.aws/v1alpha1
+kind: AWSNodeTemplate
+metadata:
+  name: default
+spec:
+  licenseConfiguration: arn:aws:license-manager:eu-east-1:123456789012:license-configuration:lic-edf7f9e241f5e16f29996c842111f448 # optional, arn of the license configuration
+  placement:
+    hostResourceGroupArn: arn:aws:resource-groups:us-east-1:123456789012:group/my-hrg-name #optional, The ARN of the host resource group in which to launch the instances. If you specify a host resource group ARN, omit the Tenancy parameter or set it to host.
+
+```
+
+2. Implement the complete fields from AWS Launch Templates
+
+eg.
+```
+apiVersion: karpenter.k8s.aws/v1alpha1
+kind: AWSNodeTemplate
+metadata:
+  name: default
+spec:
+  licenseConfiguration: arn:aws:license-manager:eu-east-1:123456789012:license-configuration:lic-edf7f9e241f5e16f29996c842111f448 # optional, arn of the license configuration
+  placement:
+    affinity: #optional
+    availabilityZone: #optional
+    groupId: #optional, something to do with placement groups
+    groupName: #optional
+    hostId: #optional, ID of the dedicated host
+    hostResourceGroupArn: arn:aws:resource-groups:us-east-1:123456789012:group/my-hrg-name #optional, The ARN of the host resource group in which to launch the instances. If you specify a host resource group ARN, omit the Tenancy parameter or set it to host.
+    paritionNumber: #optional, The number of the partition the instance should launch in. Valid only if the placement group strategy is set to partition.
+    spreadDomain: #optional, reserved for future use
+    tenancy: dedicated #optional, The tenancy of the instance. An instance with a tenancy of dedicated runs on single-tenant hardware, one of dedicated | default | host
+
+```
+
+3. Implement a simplified API
+AWS Launch templates also include extra fields
+
+```
+apiVersion: karpenter.k8s.aws/v1alpha1
+kind: AWSNodeTemplate
+metadata:
+  name: default
+spec:
+  dedicatedHostConfig:
+    licenseConfiguration: arn:aws:license-manager:eu-east-1:123456789012:license-configuration:lic-edf7f9e241f5e16f29996c842111f448 # optional, arn of the license configuration
+    resourceHostGroup: arn:aws:resource-groups:us-east-1:123456789012:group/my-hrg-name #option, arn of the HRG 
+```
+
+4. Implement Selectors for all relevant fields
+
+```
+apiVersion: karpenter.k8s.aws/v1alpha1
+kind: AWSNodeTemplate
+metadata:
+  name: default
+spec:
+  licenseSelector:
+    name: "myLicense"
+  hostResourceGroupSelector:
+    name: "myHrg"
+```
+
+This would require inexpensive or filterable APIs to query for available license configurations and host resource groups.
+Today the recommended apis are:
+* `aws license-manager list-license-configurations`
+* `aws resource-groups list-groups`
+* `aws ec2 describe-placement-groups`
+
+
+## Recommendations
+
+1. Middle ground solution, stick both to the AWS Launch Template api, but implementing the smallest set of configuration practical, 
+limits the amount of checks required to be performed in Karpenter, eg. setting both hostId and HostResourceGroupArn
+
+2. Completely copies the AWS api, would allow the full set of possible configurations supported by AWS. As documented in launch-template-options.md also requires the most work to implement and support.
+
+3. Focuses entirely on the dedicated host feature, ignoring other configuration options. Reduces confusion
+
+## Decision from Working Group meeting
+
+Implement selectors for all relevant fields to improve portability of configuration between clusters / regions / accounts.

pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml

@@ -163,6 +163,33 @@ spec:
                 description: DetailedMonitoring controls if detailed monitoring is
                   enabled for instances that are launched
                 type: boolean
+              hostResourceGroupSelectorTerms:
+                description: HostResourceGroupSelectorTerms is a list of HostResourceGroupSelectors.
+                  The terms are ORed.
+                items:
+                  description: HostResourceGroupSelectorTerm defines the selection
+                    logic for host Resource groups that are used to launch nodes.
+                    If multiple fields are used for selection, the requirements are
+                    ANDed
+                  properties:
+                    name:
+                      description: Name of the hrg to be selected
+                      type: string
+                  type: object
+                type: array
+              licenseSelectorTerms:
+                description: LicenseSelectorTerms is a list of LicenseSelectors. The
+                  terms are ORed.
+                items:
+                  description: LicenseSelectorTerm defines selection logic for a licenseConfigurationSpecification
+                    used to launch nodes If multiple fields are used for selection,
+                    the requirements are ANDed.
+                  properties:
+                    name:
+                      description: Name of the license to be selected.
+                      type: string
+                  type: object
+                type: array
               metadataOptions:
                 default:
                   httpEndpoint: enabled
@@ -220,6 +247,19 @@ spec:
                       credentials are not available."
                     type: string
                 type: object
+              placementGroupSelectorTerms:
+                description: PlacementGroupSelectorTerms is a list of PlacementGroupSelector.
+                  The terms are ORed.
+                items:
+                  description: PlacementGroupSelectorTerm defines the selection logic
+                    for ec2 placement groups that are used to launch nodes. If multiple
+                    fields are used for selection, the requirements are ANDed
+                  properties:
+                    name:
+                      description: Name of the placement group to be selected
+                      type: string
+                  type: object
+                type: array
               role:
                 description: Role is the AWS identity that nodes use. This field is
                   immutable. Marking this field as immutable avoids concerns around
@@ -345,10 +385,30 @@ spec:
                   - requirements
                   type: object
                 type: array
+              hostResourceGroup:
+                description: HostResourceGroups contains the HRG arns
+                properties:
+                  arn:
+                    description: Arn of the HRG
+                    type: string
+                  name:
+                    description: Name of the HRG
+                    type: string
+                type: object
               instanceProfile:
                 description: InstanceProfile contains the resolved instance profile
                   for the role
                 type: string
+              licenses:
+                description: Licenses contains the license arns
+                items:
+                  type: string
+                type: array
+              placementGroups:
+                description: PlacementGroups contains the ec2 placement group arns
+                items:
+                  type: string
+                type: array
               securityGroups:
                 description: SecurityGroups contains the current Security Groups values
                   that are available to the cluster under the SecurityGroups selectors.

pkg/apis/v1beta1/ec2nodeclass.go

@@ -39,6 +39,15 @@ type EC2NodeClassSpec struct {
 	// +kubebuilder:validation:Enum:={AL2,Bottlerocket,Ubuntu,Custom,Windows2019,Windows2022}
 	// +required
 	AMIFamily *string `json:"amiFamily"`
+	// LicenseSelectorTerms is a list of LicenseSelectors. The terms are ORed.
+	// +optional
+	LicenseSelectorTerms []LicenseSelectorTerm `json:"licenseSelectorTerms,omitempty" hash:"ignore"`
+	// HostResourceGroupSelectorTerms is a list of HostResourceGroupSelectors. The terms are ORed.
+	// +optional
+	HostResourceGroupSelectorTerms []HostResourceGroupSelectorTerm `json:"hostResourceGroupSelectorTerms,omitempty" hash:"ignore"`
+	// PlacementGroupSelectorTerms is a list of PlacementGroupSelector. The terms are ORed.
+	// +optional
+	PlacementGroupSelectorTerms []PlacementGroupSelectorTerm `json:"placementGroupSelectorTerms,omitempty" hash:"ignore"`
 	// UserData to be applied to the provisioned nodes.
 	// It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into
 	// this UserData to ensure nodes are being provisioned with the correct configuration.
@@ -157,6 +166,30 @@ type AMISelectorTerm struct {
 	Owner string `json:"owner,omitempty"`
 }
 
+// LicenseSelectorTerm defines selection logic for a licenseConfigurationSpecification used to launch nodes
+// If multiple fields are used for selection, the requirements are ANDed.
+type LicenseSelectorTerm struct {
+	// Name of the license to be selected.
+	// +optional
+	Name string `json:"name,omitempty"`
+}
+
+// HostResourceGroupSelectorTerm defines the selection logic for host Resource groups
+// that are used to launch nodes. If multiple fields are used for selection, the requirements are ANDed
+type HostResourceGroupSelectorTerm struct {
+	// Name of the hrg to be selected
+	// +optional
+	Name string `json:"name,omitempty"`
+}
+
+// PlacementGroupSelectorTerm defines the selection logic for ec2 placement groups
+// that are used to launch nodes. If multiple fields are used for selection, the requirements are ANDed
+type PlacementGroupSelectorTerm struct {
+	// Name of the placement group to be selected
+	// +optional
+	Name string `json:"name,omitempty"`
+}
+
 // MetadataOptions contains parameters for specifying the exposure of the
 // Instance Metadata Service to provisioned EC2 nodes.
 type MetadataOptions struct {

pkg/apis/v1beta1/ec2nodeclass_status.go

@@ -49,6 +49,16 @@ type AMI struct {
 	Requirements []v1.NodeSelectorRequirement `json:"requirements"`
 }
 
+// HostResourceGroup contains the resolved host resource group name and arn for node launch
+type HostResourceGroup struct {
+	// Name of the HRG
+	// +optional
+	Name string `json:"name,omitempty"`
+	// Arn of the HRG
+	// +optional
+	ARN string `json:"arn,omitempty"`
+}
+
 // EC2NodeClassStatus contains the resolved state of the EC2NodeClass
 type EC2NodeClassStatus struct {
 	// Subnets contains the current Subnet values that are available to the
@@ -63,6 +73,15 @@ type EC2NodeClassStatus struct {
 	// cluster under the AMI selectors.
 	// +optional
 	AMIs []AMI `json:"amis,omitempty"`
+	// Licenses contains the license arns
+	// +optional
+	Licenses []string `json:"licenses,omitempty"`
+	// HostResourceGroups contains the HRG arns
+	// +optional
+	HostResourceGroup *HostResourceGroup `json:"hostResourceGroup,omitempty"`
+	// PlacementGroups contains the ec2 placement group arns
+	// +optional
+	PlacementGroups []string `json:"placementGroups,omitempty"`
 	// InstanceProfile contains the resolved instance profile for the role
 	// +optional
 	InstanceProfile string `json:"instanceProfile,omitempty"`