Add initial implementation to add capacityReservationSelectorTerms an…

…d update status when found

cmd/controller/main.go

@@ -62,6 +62,7 @@ func main() {
 			cloudProvider,
 			op.SubnetProvider,
 			op.SecurityGroupProvider,
+			op.CapacityReservationProvider,
 			op.InstanceProfileProvider,
 			op.InstanceProvider,
 			op.PricingProvider,

pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml

@@ -216,6 +216,53 @@ spec:
                 - message: must have only one blockDeviceMappings with rootVolume
                   rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size()
                     <= 1
+              capacityReservationSelectorTerms:
+                description: CapacityReservationSelectorTerms is a list of or Capacity
+                  Reservation selector terms. The terms are ORed.
+                items:
+                  description: |-
+                    CapacityReservationSelectorTerm defines selection logic for a Capacity Reservation used by Karpenter to launch nodes.
+                    If multiple fields are used for selection, the requirements are ANDed.
+                  properties:
+                    availabilityZone:
+                      description: The Availability Zone of the Capacity Reservation
+                      type: string
+                    id:
+                      description: The platform of operating system for which the
+                        Capacity Reservation reserves capacity
+                      type: string
+                    instancePlatform:
+                      description: |-
+                        Indicates the tenancy of the Capacity Reservation.
+                        A Capacity Reservation can have one of the following tenancy 'default' or 'dedicated'
+                        default - The Capacity Reservation is created on hardware that is shared with other Amazon Web Services accounts.
+                        dedicated - The Capacity Reservation is created on single-tenant hardware that is dedicated to a single Amazon Web Services account.
+                      type: string
+                    instanceType:
+                      description: The type of operating system for which the Capacity
+                        Reservation reserves capacity
+                      type: string
+                    ownerId:
+                      description: The ID of the Amazon Web Services account that
+                        owns the Capacity Reservation
+                      type: string
+                    tags:
+                      additionalProperties:
+                        type: string
+                      description: |-
+                        Tags is a map of key/value tags used to select subnets
+                        Specifying '*' for a value selects all values for a given tag key.
+                      maxProperties: 20
+                      type: object
+                      x-kubernetes-validations:
+                      - message: empty tag keys or values aren't supported
+                        rule: self.all(k, k != '' && self[k] != '')
+                    tenancy:
+                      description: ID is the Capacity Reservation id in EC2
+                      pattern: cr-[0-9a-z]+
+                      type: string
+                  type: object
+                type: array
               context:
                 description: |-
                   Context is a Reserved field in EC2 APIs
@@ -516,6 +563,76 @@ spec:
                   - requirements
                   type: object
                 type: array
+              capacityReservations:
+                description: |-
+                  CapacityReservations contains the current Capacity Reservations values that are available to the
+                  cluster under the CapacityReservations selectors.
+                items:
+                  description: CapacityReservation contains resolved Capacity Reservation
+                    selector values utilized for node launch
+                  properties:
+                    availabilityZone:
+                      description: AvailabilityZone of the Capacity Reservation
+                      type: string
+                    availableInstanceCount:
+                      description: Available Instance Count of the Capacity Reservation
+                      type: integer
+                    id:
+                      description: ID of the Capacity Reservation
+                      type: string
+                    instanceType:
+                      description: InstanceType of the Capacity Reservation
+                      type: string
+                    requirements:
+                      description: Requirements of the Capacity Reservation to be
+                        utilized on an instance type
+                      items:
+                        description: |-
+                          A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values
+                          and minValues that represent the requirement to have at least that many values.
+                        properties:
+                          key:
+                            description: The label key that the selector applies to.
+                            type: string
+                          minValues:
+                            description: |-
+                              This field is ALPHA and can be dropped or replaced at any time
+                              MinValues is the minimum number of unique values required to define the flexibility of the specific requirement.
+                            maximum: 50
+                            minimum: 1
+                            type: integer
+                          operator:
+                            description: |-
+                              Represents a key's relationship to a set of values.
+                              Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+                            type: string
+                          values:
+                            description: |-
+                              An array of string values. If the operator is In or NotIn,
+                              the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                              the values array must be empty. If the operator is Gt or Lt, the values
+                              array must have a single element, which will be interpreted as an integer.
+                              This array is replaced during a strategic merge patch.
+                            items:
+                              type: string
+                            type: array
+                        required:
+                        - key
+                        - operator
+                        type: object
+                      type: array
+                    totalInstanceCount:
+                      description: Total Instance Count of the Capacity Reservation
+                      type: integer
+                  required:
+                  - availabilityZone
+                  - availableInstanceCount
+                  - id
+                  - instanceType
+                  - requirements
+                  - totalInstanceCount
+                  type: object
+                type: array
               instanceProfile:
                 description: InstanceProfile contains the resolved instance profile
                   for the role

pkg/apis/v1beta1/ec2nodeclass.go

@@ -55,6 +55,9 @@ type EC2NodeClassSpec struct {
 	// +kubebuilder:validation:Enum:={AL2,AL2023,Bottlerocket,Ubuntu,Custom,Windows2019,Windows2022}
 	// +required
 	AMIFamily *string `json:"amiFamily"`
+	// CapacityReservationSelectorTerms is a list of or Capacity Reservation selector terms. The terms are ORed.
+	// +required
+	CapacityReservationSelectorTerms []CapacityReservationSelectorTerm `json:"capacityReservationSelectorTerms,omitempty" hash:"ignore"`
 	// UserData to be applied to the provisioned nodes.
 	// It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into
 	// this UserData to ensure nodes are being provisioned with the correct configuration.
@@ -175,6 +178,39 @@ type AMISelectorTerm struct {
 	Owner string `json:"owner,omitempty"`
 }
 
+// CapacityReservationSelectorTerm defines selection logic for a Capacity Reservation used by Karpenter to launch nodes.
+// If multiple fields are used for selection, the requirements are ANDed.
+type CapacityReservationSelectorTerm struct {
+	// The Availability Zone of the Capacity Reservation
+	// +optional
+	AvailabilityZone string `json:"availabilityZone,omitempty"`
+	// The platform of operating system for which the Capacity Reservation reserves capacity
+	// +optional
+	ID string `json:"id,omitempty"`
+	// Tags is a map of key/value tags used to select subnets
+	// Specifying '*' for a value selects all values for a given tag key.
+	// +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')"
+	// +kubebuilder:validation:MaxProperties:=20
+	// +optional
+	Tags map[string]string `json:"tags,omitempty"`
+	// ID is the Capacity Reservation id in EC2
+	// +kubebuilder:validation:Pattern:="cr-[0-9a-z]+"
+	// +optional
+	Tenancy string `json:"tenancy,omitempty"`
+	// Indicates the tenancy of the Capacity Reservation.
+	// A Capacity Reservation can have one of the following tenancy 'default' or 'dedicated'
+	// default - The Capacity Reservation is created on hardware that is shared with other Amazon Web Services accounts.
+	// dedicated - The Capacity Reservation is created on single-tenant hardware that is dedicated to a single Amazon Web Services account.
+	// +optional
+	InstancePlatform string `json:"instancePlatform,omitempty"`
+	// The type of operating system for which the Capacity Reservation reserves capacity
+	// +optional
+	InstanceType string `json:"instanceType,omitempty"`
+	// The ID of the Amazon Web Services account that owns the Capacity Reservation
+	// +optional
+	OwnerId string `json:"ownerId,omitempty"`
+}
+
 // MetadataOptions contains parameters for specifying the exposure of the
 // Instance Metadata Service to provisioned EC2 nodes.
 type MetadataOptions struct {

pkg/apis/v1beta1/ec2nodeclass_status.go

@@ -38,6 +38,28 @@ type SecurityGroup struct {
 	Name string `json:"name,omitempty"`
 }
 
+// CapacityReservation contains resolved Capacity Reservation selector values utilized for node launch
+type CapacityReservation struct {
+	// ID of the Capacity Reservation
+	// +required
+	ID string `json:"id"`
+	// AvailabilityZone of the Capacity Reservation
+	// +required
+	AvailabilityZone string `json:"availabilityZone"`
+	// Available Instance Count of the Capacity Reservation
+	// +required
+	AvailableInstanceCount int `json:"availableInstanceCount"`
+	// InstanceType of the Capacity Reservation
+	// +required
+	InstanceType string `json:"instanceType"`
+	// Requirements of the Capacity Reservation to be utilized on an instance type
+	// +required
+	Requirements []corev1beta1.NodeSelectorRequirementWithMinValues `json:"requirements"`
+	// Total Instance Count of the Capacity Reservation
+	// +required
+	TotalInstanceCount int `json:"totalInstanceCount"`
+}
+
 // AMI contains resolved AMI selector values utilized for node launch
 type AMI struct {
 	// ID of the AMI
@@ -53,6 +75,10 @@ type AMI struct {
 
 // EC2NodeClassStatus contains the resolved state of the EC2NodeClass
 type EC2NodeClassStatus struct {
+	// CapacityReservations contains the current Capacity Reservations values that are available to the
+	// cluster under the CapacityReservations selectors.
+	// +optional
+	CapacityReservations []CapacityReservation `json:"capacityReservations,omitempty"`
 	// Subnets contains the current Subnet values that are available to the
 	// cluster under the subnet selectors.
 	// +optional

pkg/cloudprovider/cloudprovider.go

@@ -96,6 +96,9 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *corev1beta1.NodeC
 	}
 	instance, err := c.instanceProvider.Create(ctx, nodeClass, nodeClaim, instanceTypes)
 	if err != nil {
+		if cloudprovider.IsInsufficientCapacityError(err) {
+			return nil, cloudprovider.NewInsufficientCapacityError(fmt.Errorf("creating instance, %w", err))
+		}
 		return nil, fmt.Errorf("creating instance, %w", err)
 	}
 	instanceType, _ := lo.Find(instanceTypes, func(i *cloudprovider.InstanceType) bool {

pkg/controllers/controllers.go

@@ -24,6 +24,7 @@ import (
 	nodeclasstermination "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/termination"
 	controllersinstancetype "github.com/aws/karpenter-provider-aws/pkg/controllers/providers/instancetype"
 	controllerspricing "github.com/aws/karpenter-provider-aws/pkg/controllers/providers/pricing"
+	"github.com/aws/karpenter-provider-aws/pkg/providers/capacityreservation"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate"
 
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -52,12 +53,12 @@ import (
 
 func NewControllers(ctx context.Context, sess *session.Session, clk clock.Clock, kubeClient client.Client, recorder events.Recorder,
 	unavailableOfferings *cache.UnavailableOfferings, cloudProvider cloudprovider.CloudProvider, subnetProvider subnet.Provider,
-	securityGroupProvider securitygroup.Provider, instanceProfileProvider instanceprofile.Provider, instanceProvider instance.Provider,
+	securityGroupProvider securitygroup.Provider, capacityReservationProvider capacityreservation.Provider, instanceProfileProvider instanceprofile.Provider, instanceProvider instance.Provider,
 	pricingProvider pricing.Provider, amiProvider amifamily.Provider, launchTemplateProvider launchtemplate.Provider, instanceTypeProvider instancetype.Provider) []controller.Controller {
 
 	controllers := []controller.Controller{
 		nodeclasshash.NewController(kubeClient),
-		nodeclassstatus.NewController(kubeClient, subnetProvider, securityGroupProvider, amiProvider, instanceProfileProvider, launchTemplateProvider),
+		nodeclassstatus.NewController(kubeClient, subnetProvider, securityGroupProvider, capacityReservationProvider, amiProvider, instanceProfileProvider, launchTemplateProvider),
 		nodeclasstermination.NewController(kubeClient, recorder, instanceProfileProvider, launchTemplateProvider),
 		nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider),
 		nodeclaimtagging.NewController(kubeClient, instanceProvider),

pkg/controllers/nodeclass/status/controller.go

@@ -34,6 +34,7 @@ import (
 
 	"github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/amifamily"
+	"github.com/aws/karpenter-provider-aws/pkg/providers/capacityreservation"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/securitygroup"
@@ -49,23 +50,25 @@ type nodeClassStatusReconciler interface {
 type Controller struct {
 	kubeClient client.Client
 
-	ami             *AMI
-	instanceprofile *InstanceProfile
-	subnet          *Subnet
-	securitygroup   *SecurityGroup
-	launchtemplate  *LaunchTemplate
+	ami                 *AMI
+	instanceprofile     *InstanceProfile
+	subnet              *Subnet
+	securitygroup       *SecurityGroup
+	capacityreservation *CapacityReservation
+	launchtemplate      *LaunchTemplate
 }
 
-func NewController(kubeClient client.Client, subnetProvider subnet.Provider, securityGroupProvider securitygroup.Provider,
+func NewController(kubeClient client.Client, subnetProvider subnet.Provider, securityGroupProvider securitygroup.Provider, capacityReservationProvider capacityreservation.Provider,
 	amiProvider amifamily.Provider, instanceProfileProvider instanceprofile.Provider, launchTemplateProvider launchtemplate.Provider) corecontroller.Controller {
 	return corecontroller.Typed[*v1beta1.EC2NodeClass](kubeClient, &Controller{
 		kubeClient: kubeClient,
 
-		ami:             &AMI{amiProvider: amiProvider},
-		subnet:          &Subnet{subnetProvider: subnetProvider},
-		securitygroup:   &SecurityGroup{securityGroupProvider: securityGroupProvider},
-		instanceprofile: &InstanceProfile{instanceProfileProvider: instanceProfileProvider},
-		launchtemplate:  &LaunchTemplate{launchTemplateProvider: launchTemplateProvider},
+		ami:                 &AMI{amiProvider: amiProvider},
+		subnet:              &Subnet{subnetProvider: subnetProvider},
+		securitygroup:       &SecurityGroup{securityGroupProvider: securityGroupProvider},
+		capacityreservation: &CapacityReservation{capacityReservationProvider: capacityReservationProvider},
+		instanceprofile:     &InstanceProfile{instanceProfileProvider: instanceProfileProvider},
+		launchtemplate:      &LaunchTemplate{launchTemplateProvider: launchTemplateProvider},
 	})
 }
 
@@ -87,6 +90,7 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeCl
 		c.securitygroup,
 		c.instanceprofile,
 		c.launchtemplate,
+		c.capacityreservation,
 	} {
 		res, err := reconciler.Reconcile(ctx, nodeClass)
 		errs = multierr.Append(errs, err)