From 6867ca800e182e596d70f42b92051cd038082355 Mon Sep 17 00:00:00 2001 From: Thorben von Hacht Date: Mon, 6 May 2024 00:42:24 -0700 Subject: [PATCH] Add initial implementation to add capacityReservationSelectorTerms and update status when found --- cmd/controller/main.go | 1 + .../karpenter.k8s.aws_ec2nodeclasses.yaml | 133 ++++++++++++++++++ pkg/apis/v1beta1/ec2nodeclass.go | 49 +++++++ pkg/apis/v1beta1/ec2nodeclass_status.go | 26 ++++ pkg/apis/v1beta1/zz_generated.deepcopy.go | 58 ++++++++ pkg/cloudprovider/cloudprovider.go | 3 + pkg/controllers/controllers.go | 5 +- .../nodeclass/status/controller.go | 26 ++-- .../nodeclass/status/suite_test.go | 1 + pkg/errors/errors.go | 1 + pkg/operator/operator.go | 58 ++++---- pkg/providers/amifamily/resolver.go | 77 +++++++++- pkg/providers/instance/instance.go | 50 ++++--- pkg/providers/instance/types.go | 5 +- pkg/providers/instancetype/instancetype.go | 73 +++++++++- .../launchtemplate/launchtemplate.go | 47 +++++-- pkg/test/environment.go | 47 ++++--- 17 files changed, 556 insertions(+), 104 deletions(-) diff --git a/cmd/controller/main.go b/cmd/controller/main.go index fc09b917beef..af2cd28e5d7b 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -62,6 +62,7 @@ func main() { cloudProvider, op.SubnetProvider, op.SecurityGroupProvider, + op.CapacityReservationProvider, op.InstanceProfileProvider, op.InstanceProvider, op.PricingProvider, diff --git a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml index 66ec659e7747..ce815ddd44ef 100644 --- a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -216,6 +216,67 @@ spec: - message: must have only one blockDeviceMappings with rootVolume rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1 + capacityReservationSelectorTerms: + description: CapacityReservationSelectorTerms is a list of or Capacity + Reservation selector terms. The terms are ORed. + items: + description: |- + CapacityReservationSelectorTerm defines selection logic for a Capacity Reservation used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + availabilityZone: + description: The Availability Zone of the Capacity Reservation + type: string + id: + description: The platform of operating system for which the + Capacity Reservation reserves capacity + type: string + instanceMatchCriteria: + description: |- + Indicates the type of instance launches that the Capacity Reservation accepts. The options include 'open' and 'targeted'. + open - The Capacity Reservation accepts all instances that have + matching attributes (instance type, platform, and Availability + Zone). Instances that have matching attributes launch into the + Capacity Reservation automatically without specifying any + additional parameters. + targeted - The Capacity Reservation only accepts instances that + have matching attributes (instance type, platform, and + Availability Zone), and explicitly target the Capacity + Reservation. This ensures that only permitted instances can use + the reserved capacity. + type: string + instancePlatform: + description: |- + Indicates the tenancy of the Capacity Reservation. + A Capacity Reservation can have one of the following tenancy 'default' or 'dedicated' + default - The Capacity Reservation is created on hardware that is shared with other Amazon Web Services accounts. + dedicated - The Capacity Reservation is created on single-tenant hardware that is dedicated to a single Amazon Web Services account. + type: string + instanceType: + description: The type of operating system for which the Capacity + Reservation reserves capacity + type: string + ownerId: + description: The ID of the Amazon Web Services account that + owns the Capacity Reservation + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + tenancy: + description: ID is the Capacity Reservation id in EC2 + pattern: cr-[0-9a-z]+ + type: string + type: object + type: array context: description: |- Context is a Reserved field in EC2 APIs @@ -506,14 +567,86 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + required: + - id + - requirements + type: object + type: array + capacityReservations: + description: |- + CapacityReservations contains the current Capacity Reservations values that are available to the + cluster under the CapacityReservations selectors. + items: + description: CapacityReservation contains resolved Capacity Reservation + selector values utilized for node launch + properties: + availabilityZone: + description: AvailabilityZone of the Capacity Reservation + type: string + availableInstanceCount: + description: Available Instance Count of the Capacity Reservation + type: integer + id: + description: ID of the Capacity Reservation + type: string + instanceType: + description: InstanceType of the Capacity Reservation + type: string + requirements: + description: Requirements of the Capacity Reservation to be + utilized on an instance type + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + totalInstanceCount: + description: Total Instance Count of the Capacity Reservation + type: integer required: + - availabilityZone + - availableInstanceCount - id + - instanceType - requirements + - totalInstanceCount type: object type: array instanceProfile: diff --git a/pkg/apis/v1beta1/ec2nodeclass.go b/pkg/apis/v1beta1/ec2nodeclass.go index 66a2926b2ced..c51ee12fb3b1 100644 --- a/pkg/apis/v1beta1/ec2nodeclass.go +++ b/pkg/apis/v1beta1/ec2nodeclass.go @@ -55,6 +55,9 @@ type EC2NodeClassSpec struct { // +kubebuilder:validation:Enum:={AL2,AL2023,Bottlerocket,Ubuntu,Custom,Windows2019,Windows2022} // +required AMIFamily *string `json:"amiFamily"` + // CapacityReservationSelectorTerms is a list of or Capacity Reservation selector terms. The terms are ORed. + // +required + CapacityReservationSelectorTerms []CapacityReservationSelectorTerm `json:"capacityReservationSelectorTerms,omitempty" hash:"ignore"` // UserData to be applied to the provisioned nodes. // It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into // this UserData to ensure nodes are being provisioned with the correct configuration. @@ -175,6 +178,52 @@ type AMISelectorTerm struct { Owner string `json:"owner,omitempty"` } +// CapacityReservationSelectorTerm defines selection logic for a Capacity Reservation used by Karpenter to launch nodes. +// If multiple fields are used for selection, the requirements are ANDed. +type CapacityReservationSelectorTerm struct { + // The Availability Zone of the Capacity Reservation + // +optional + AvailabilityZone string `json:"availabilityZone,omitempty"` + // The platform of operating system for which the Capacity Reservation reserves capacity + // +optional + ID string `json:"id,omitempty"` + // Tags is a map of key/value tags used to select subnets + // Specifying '*' for a value selects all values for a given tag key. + // +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')" + // +kubebuilder:validation:MaxProperties:=20 + // +optional + Tags map[string]string `json:"tags,omitempty"` + // ID is the Capacity Reservation id in EC2 + // +kubebuilder:validation:Pattern:="cr-[0-9a-z]+" + // +optional + Tenancy string `json:"tenancy,omitempty"` + // Indicates the type of instance launches that the Capacity Reservation accepts. The options include 'open' and 'targeted'. + // open - The Capacity Reservation accepts all instances that have + // matching attributes (instance type, platform, and Availability + // Zone). Instances that have matching attributes launch into the + // Capacity Reservation automatically without specifying any + // additional parameters. + // targeted - The Capacity Reservation only accepts instances that + // have matching attributes (instance type, platform, and + // Availability Zone), and explicitly target the Capacity + // Reservation. This ensures that only permitted instances can use + // the reserved capacity. + // +optional + InstanceMatchCriteria string `json:"instanceMatchCriteria,omitempty"` + // Indicates the tenancy of the Capacity Reservation. + // A Capacity Reservation can have one of the following tenancy 'default' or 'dedicated' + // default - The Capacity Reservation is created on hardware that is shared with other Amazon Web Services accounts. + // dedicated - The Capacity Reservation is created on single-tenant hardware that is dedicated to a single Amazon Web Services account. + // +optional + InstancePlatform string `json:"instancePlatform,omitempty"` + // The type of operating system for which the Capacity Reservation reserves capacity + // +optional + InstanceType string `json:"instanceType,omitempty"` + // The ID of the Amazon Web Services account that owns the Capacity Reservation + // +optional + OwnerId string `json:"ownerId,omitempty"` +} + // MetadataOptions contains parameters for specifying the exposure of the // Instance Metadata Service to provisioned EC2 nodes. type MetadataOptions struct { diff --git a/pkg/apis/v1beta1/ec2nodeclass_status.go b/pkg/apis/v1beta1/ec2nodeclass_status.go index 611e94d62117..2af9766a8d21 100644 --- a/pkg/apis/v1beta1/ec2nodeclass_status.go +++ b/pkg/apis/v1beta1/ec2nodeclass_status.go @@ -38,6 +38,28 @@ type SecurityGroup struct { Name string `json:"name,omitempty"` } +// CapacityReservation contains resolved Capacity Reservation selector values utilized for node launch +type CapacityReservation struct { + // ID of the Capacity Reservation + // +required + ID string `json:"id"` + // AvailabilityZone of the Capacity Reservation + // +required + AvailabilityZone string `json:"availabilityZone"` + // Available Instance Count of the Capacity Reservation + // +required + AvailableInstanceCount int `json:"availableInstanceCount"` + // InstanceType of the Capacity Reservation + // +required + InstanceType string `json:"instanceType"` + // Requirements of the Capacity Reservation to be utilized on an instance type + // +required + Requirements []corev1beta1.NodeSelectorRequirementWithMinValues `json:"requirements"` + // Total Instance Count of the Capacity Reservation + // +required + TotalInstanceCount int `json:"totalInstanceCount"` +} + // AMI contains resolved AMI selector values utilized for node launch type AMI struct { // ID of the AMI @@ -53,6 +75,10 @@ type AMI struct { // EC2NodeClassStatus contains the resolved state of the EC2NodeClass type EC2NodeClassStatus struct { + // CapacityReservations contains the current Capacity Reservations values that are available to the + // cluster under the CapacityReservations selectors. + // +optional + CapacityReservations []CapacityReservation `json:"capacityReservations,omitempty"` // Subnets contains the current Subnet values that are available to the // cluster under the subnet selectors. // +optional diff --git a/pkg/apis/v1beta1/zz_generated.deepcopy.go b/pkg/apis/v1beta1/zz_generated.deepcopy.go index 781d88c876c8..6901f4019f07 100644 --- a/pkg/apis/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/v1beta1/zz_generated.deepcopy.go @@ -147,6 +147,50 @@ func (in *BlockDeviceMapping) DeepCopy() *BlockDeviceMapping { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CapacityReservation) DeepCopyInto(out *CapacityReservation) { + *out = *in + if in.Requirements != nil { + in, out := &in.Requirements, &out.Requirements + *out = make([]apisv1beta1.NodeSelectorRequirementWithMinValues, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CapacityReservation. +func (in *CapacityReservation) DeepCopy() *CapacityReservation { + if in == nil { + return nil + } + out := new(CapacityReservation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CapacityReservationSelectorTerm) DeepCopyInto(out *CapacityReservationSelectorTerm) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CapacityReservationSelectorTerm. +func (in *CapacityReservationSelectorTerm) DeepCopy() *CapacityReservationSelectorTerm { + if in == nil { + return nil + } + out := new(CapacityReservationSelectorTerm) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EC2NodeClass) DeepCopyInto(out *EC2NodeClass) { *out = *in @@ -240,6 +284,13 @@ func (in *EC2NodeClassSpec) DeepCopyInto(out *EC2NodeClassSpec) { *out = new(string) **out = **in } + if in.CapacityReservationSelectorTerms != nil { + in, out := &in.CapacityReservationSelectorTerms, &out.CapacityReservationSelectorTerms + *out = make([]CapacityReservationSelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.UserData != nil { in, out := &in.UserData, &out.UserData *out = new(string) @@ -303,6 +354,13 @@ func (in *EC2NodeClassSpec) DeepCopy() *EC2NodeClassSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EC2NodeClassStatus) DeepCopyInto(out *EC2NodeClassStatus) { *out = *in + if in.CapacityReservations != nil { + in, out := &in.CapacityReservations, &out.CapacityReservations + *out = make([]CapacityReservation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Subnets != nil { in, out := &in.Subnets, &out.Subnets *out = make([]Subnet, len(*in)) diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index acea3187e4d1..35923589d55c 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -96,6 +96,9 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *corev1beta1.NodeC } instance, err := c.instanceProvider.Create(ctx, nodeClass, nodeClaim, instanceTypes) if err != nil { + if cloudprovider.IsInsufficientCapacityError(err) { + return nil, cloudprovider.NewInsufficientCapacityError(fmt.Errorf("creating instance, %w", err)) + } return nil, fmt.Errorf("creating instance, %w", err) } instanceType, _ := lo.Find(instanceTypes, func(i *cloudprovider.InstanceType) bool { diff --git a/pkg/controllers/controllers.go b/pkg/controllers/controllers.go index 6510a522f870..ab23945200a3 100644 --- a/pkg/controllers/controllers.go +++ b/pkg/controllers/controllers.go @@ -24,6 +24,7 @@ import ( nodeclasstermination "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/termination" controllersinstancetype "github.com/aws/karpenter-provider-aws/pkg/controllers/providers/instancetype" controllerspricing "github.com/aws/karpenter-provider-aws/pkg/controllers/providers/pricing" + "github.com/aws/karpenter-provider-aws/pkg/providers/capacityreservation" "github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate" "github.com/aws/aws-sdk-go/aws/session" @@ -52,12 +53,12 @@ import ( func NewControllers(ctx context.Context, sess *session.Session, clk clock.Clock, kubeClient client.Client, recorder events.Recorder, unavailableOfferings *cache.UnavailableOfferings, cloudProvider cloudprovider.CloudProvider, subnetProvider subnet.Provider, - securityGroupProvider securitygroup.Provider, instanceProfileProvider instanceprofile.Provider, instanceProvider instance.Provider, + securityGroupProvider securitygroup.Provider, capacityReservationProvider capacityreservation.Provider, instanceProfileProvider instanceprofile.Provider, instanceProvider instance.Provider, pricingProvider pricing.Provider, amiProvider amifamily.Provider, launchTemplateProvider launchtemplate.Provider, instanceTypeProvider instancetype.Provider) []controller.Controller { controllers := []controller.Controller{ nodeclasshash.NewController(kubeClient), - nodeclassstatus.NewController(kubeClient, subnetProvider, securityGroupProvider, amiProvider, instanceProfileProvider, launchTemplateProvider), + nodeclassstatus.NewController(kubeClient, subnetProvider, securityGroupProvider, capacityReservationProvider, amiProvider, instanceProfileProvider, launchTemplateProvider), nodeclasstermination.NewController(kubeClient, recorder, instanceProfileProvider, launchTemplateProvider), nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider), nodeclaimtagging.NewController(kubeClient, instanceProvider), diff --git a/pkg/controllers/nodeclass/status/controller.go b/pkg/controllers/nodeclass/status/controller.go index da4aeca04dc0..37b7598322ae 100644 --- a/pkg/controllers/nodeclass/status/controller.go +++ b/pkg/controllers/nodeclass/status/controller.go @@ -34,6 +34,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" + "github.com/aws/karpenter-provider-aws/pkg/providers/capacityreservation" "github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile" "github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate" "github.com/aws/karpenter-provider-aws/pkg/providers/securitygroup" @@ -49,23 +50,25 @@ type nodeClassStatusReconciler interface { type Controller struct { kubeClient client.Client - ami *AMI - instanceprofile *InstanceProfile - subnet *Subnet - securitygroup *SecurityGroup - launchtemplate *LaunchTemplate + ami *AMI + instanceprofile *InstanceProfile + subnet *Subnet + securitygroup *SecurityGroup + capacityreservation *CapacityReservation + launchtemplate *LaunchTemplate } -func NewController(kubeClient client.Client, subnetProvider subnet.Provider, securityGroupProvider securitygroup.Provider, +func NewController(kubeClient client.Client, subnetProvider subnet.Provider, securityGroupProvider securitygroup.Provider, capacityReservationProvider capacityreservation.Provider, amiProvider amifamily.Provider, instanceProfileProvider instanceprofile.Provider, launchTemplateProvider launchtemplate.Provider) corecontroller.Controller { return corecontroller.Typed[*v1beta1.EC2NodeClass](kubeClient, &Controller{ kubeClient: kubeClient, - ami: &AMI{amiProvider: amiProvider}, - subnet: &Subnet{subnetProvider: subnetProvider}, - securitygroup: &SecurityGroup{securityGroupProvider: securityGroupProvider}, - instanceprofile: &InstanceProfile{instanceProfileProvider: instanceProfileProvider}, - launchtemplate: &LaunchTemplate{launchTemplateProvider: launchTemplateProvider}, + ami: &AMI{amiProvider: amiProvider}, + subnet: &Subnet{subnetProvider: subnetProvider}, + securitygroup: &SecurityGroup{securityGroupProvider: securityGroupProvider}, + capacityreservation: &CapacityReservation{capacityReservationProvider: capacityReservationProvider}, + instanceprofile: &InstanceProfile{instanceProfileProvider: instanceProfileProvider}, + launchtemplate: &LaunchTemplate{launchTemplateProvider: launchTemplateProvider}, }) } @@ -87,6 +90,7 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeCl c.securitygroup, c.instanceprofile, c.launchtemplate, + c.capacityreservation, } { res, err := reconciler.Reconcile(ctx, nodeClass) errs = multierr.Append(errs, err) diff --git a/pkg/controllers/nodeclass/status/suite_test.go b/pkg/controllers/nodeclass/status/suite_test.go index 545c9ba6cfd5..ac6e75848639 100644 --- a/pkg/controllers/nodeclass/status/suite_test.go +++ b/pkg/controllers/nodeclass/status/suite_test.go @@ -59,6 +59,7 @@ var _ = BeforeSuite(func() { env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, + awsEnv.CapacityReservationProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider, diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index 5267aad4672d..26de0dbb4837 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -48,6 +48,7 @@ var ( "UnfulfillableCapacity", "Unsupported", "InsufficientFreeAddressesInSubnet", + "ReservationCapacityExceeded", ) ) diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 113b863f9732..12ed5c5f06db 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -54,6 +54,7 @@ import ( awscache "github.com/aws/karpenter-provider-aws/pkg/cache" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" + "github.com/aws/karpenter-provider-aws/pkg/providers/capacityreservation" "github.com/aws/karpenter-provider-aws/pkg/providers/instance" "github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile" "github.com/aws/karpenter-provider-aws/pkg/providers/instancetype" @@ -73,19 +74,20 @@ func init() { type Operator struct { *operator.Operator - Session *session.Session - UnavailableOfferingsCache *awscache.UnavailableOfferings - EC2API ec2iface.EC2API - SubnetProvider subnet.Provider - SecurityGroupProvider securitygroup.Provider - InstanceProfileProvider instanceprofile.Provider - AMIProvider amifamily.Provider - AMIResolver *amifamily.Resolver - LaunchTemplateProvider launchtemplate.Provider - PricingProvider pricing.Provider - VersionProvider version.Provider - InstanceTypesProvider instancetype.Provider - InstanceProvider instance.Provider + Session *session.Session + UnavailableOfferingsCache *awscache.UnavailableOfferings + EC2API ec2iface.EC2API + SubnetProvider subnet.Provider + SecurityGroupProvider securitygroup.Provider + InstanceProfileProvider instanceprofile.Provider + AMIProvider amifamily.Provider + AMIResolver *amifamily.Resolver + LaunchTemplateProvider launchtemplate.Provider + PricingProvider pricing.Provider + VersionProvider version.Provider + InstanceTypesProvider instancetype.Provider + InstanceProvider instance.Provider + CapacityReservationProvider capacityreservation.Provider } func NewOperator(ctx context.Context, operator *operator.Operator) (context.Context, *Operator) { @@ -174,22 +176,24 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont subnetProvider, launchTemplateProvider, ) + capacityReservationProvider := capacityreservation.NewDefaultProvider(ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) return ctx, &Operator{ - Operator: operator, - Session: sess, - UnavailableOfferingsCache: unavailableOfferingsCache, - EC2API: ec2api, - SubnetProvider: subnetProvider, - SecurityGroupProvider: securityGroupProvider, - InstanceProfileProvider: instanceProfileProvider, - AMIProvider: amiProvider, - AMIResolver: amiResolver, - VersionProvider: versionProvider, - LaunchTemplateProvider: launchTemplateProvider, - PricingProvider: pricingProvider, - InstanceTypesProvider: instanceTypeProvider, - InstanceProvider: instanceProvider, + Operator: operator, + Session: sess, + UnavailableOfferingsCache: unavailableOfferingsCache, + EC2API: ec2api, + SubnetProvider: subnetProvider, + SecurityGroupProvider: securityGroupProvider, + InstanceProfileProvider: instanceProfileProvider, + AMIProvider: amiProvider, + AMIResolver: amiResolver, + VersionProvider: versionProvider, + LaunchTemplateProvider: launchTemplateProvider, + PricingProvider: pricingProvider, + InstanceTypesProvider: instanceTypeProvider, + InstanceProvider: instanceProvider, + CapacityReservationProvider: capacityReservationProvider, } } diff --git a/pkg/providers/amifamily/resolver.go b/pkg/providers/amifamily/resolver.go index 65adec3a9abb..65e1ed9a7f20 100644 --- a/pkg/providers/amifamily/resolver.go +++ b/pkg/providers/amifamily/resolver.go @@ -24,6 +24,7 @@ import ( "github.com/imdario/mergo" "github.com/samber/lo" core "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" @@ -56,6 +57,7 @@ type Options struct { InstanceStorePolicy *v1beta1.InstanceStorePolicy // Level-triggered fields that may change out of sync. SecurityGroups []v1beta1.SecurityGroup + CapacityReservations []v1beta1.CapacityReservation Tags map[string]string Labels map[string]string `hash:"ignore"` KubeDNSIP net.IP @@ -67,6 +69,7 @@ type Options struct { type LaunchTemplate struct { *Options UserData bootstrap.Bootstrapper + CapacityReservation *v1beta1.CapacityReservation BlockDeviceMappings []*v1beta1.BlockDeviceMapping MetadataOptions *v1beta1.MetadataOptions AMIID string @@ -154,12 +157,63 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, maxPods: int(instanceType.Capacity.Pods().Value()), } }) + + zones := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...).Get(v1.LabelTopologyZone) + capacityReservations := []v1beta1.CapacityReservation{} + if capacityType == "capacity-reservation" { + for _, capacityReservation := range nodeClass.Status.CapacityReservations { + if capacityReservation.AvailableInstanceCount == 0 { + continue + } + if !zones.Has(capacityReservation.AvailabilityZone) { + continue + } + capacityReservations = append(capacityReservations, capacityReservation) + } + if len(capacityReservations) == 0 { + return nil, cloudprovider.NewInsufficientCapacityError(fmt.Errorf("trying to resolve capacity-reservation but no available capacity reservations available")) + } + } + for params, instanceTypes := range paramsToInstanceTypes { - resolved, err := r.resolveLaunchTemplate(nodeClass, nodeClaim, instanceTypes, capacityType, amiFamily, amiID, params.maxPods, params.efaCount, options) - if err != nil { - return nil, err + + if len(capacityReservations) > 0 { + for _, capacityReservation := range capacityReservations { + resolved, err := r.resolveLaunchTemplate( + nodeClass, + nodeClaim, + &capacityReservation, + instanceTypes, + "on-demand", // capacity-type + amiFamily, + amiID, + params.maxPods, + params.efaCount, + options, + ) + if err != nil { + return nil, err + } + resolvedTemplates = append(resolvedTemplates, resolved) + } + } else { + resolved, err := r.resolveLaunchTemplate( + nodeClass, + nodeClaim, + nil, + instanceTypes, + capacityType, + amiFamily, + amiID, + params.maxPods, + params.efaCount, + options, + ) + if err != nil { + return nil, err + } + resolvedTemplates = append(resolvedTemplates, resolved) } - resolvedTemplates = append(resolvedTemplates, resolved) } } return resolvedTemplates, nil @@ -210,8 +264,18 @@ func (r Resolver) defaultClusterDNS(opts *Options, kubeletConfig *corev1beta1.Ku return newKubeletConfig } -func (r Resolver) resolveLaunchTemplate(nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, capacityType string, - amiFamily AMIFamily, amiID string, maxPods int, efaCount int, options *Options) (*LaunchTemplate, error) { +func (r Resolver) resolveLaunchTemplate( + nodeClass *v1beta1.EC2NodeClass, + nodeClaim *corev1beta1.NodeClaim, + capacityReservation *v1beta1.CapacityReservation, + instanceTypes []*cloudprovider.InstanceType, + capacityType string, + amiFamily AMIFamily, + amiID string, + maxPods int, + efaCount int, + options *Options, +) (*LaunchTemplate, error) { kubeletConfig := &corev1beta1.KubeletConfiguration{} if nodeClaim.Spec.Kubelet != nil { if err := mergo.Merge(kubeletConfig, nodeClaim.Spec.Kubelet); err != nil { @@ -232,6 +296,7 @@ func (r Resolver) resolveLaunchTemplate(nodeClass *v1beta1.EC2NodeClass, nodeCla nodeClass.Spec.UserData, options.InstanceStorePolicy, ), + CapacityReservation: capacityReservation, BlockDeviceMappings: nodeClass.Spec.BlockDeviceMappings, MetadataOptions: nodeClass.Spec.MetadataOptions, DetailedMonitoring: aws.BoolValue(nodeClass.Spec.DetailedMonitoring), diff --git a/pkg/providers/instance/instance.go b/pkg/providers/instance/instance.go index f228815563a7..7aaab3952704 100644 --- a/pkg/providers/instance/instance.go +++ b/pkg/providers/instance/instance.go @@ -96,17 +96,17 @@ func (p *DefaultProvider) Create(ctx context.Context, nodeClass *v1beta1.EC2Node instanceTypes = p.filterInstanceTypes(nodeClaim, instanceTypes) } tags := getTags(ctx, nodeClass, nodeClaim) - fleetInstance, err := p.launchInstance(ctx, nodeClass, nodeClaim, instanceTypes, tags) + fleetInstance, capacityType, err := p.launchInstance(ctx, nodeClass, nodeClaim, instanceTypes, tags) if awserrors.IsLaunchTemplateNotFound(err) { // retry once if launch template is not found. This allows karpenter to generate a new LT if the // cache was out-of-sync on the first try - fleetInstance, err = p.launchInstance(ctx, nodeClass, nodeClaim, instanceTypes, tags) + fleetInstance, capacityType, err = p.launchInstance(ctx, nodeClass, nodeClaim, instanceTypes, tags) } if err != nil { return nil, err } efaEnabled := lo.Contains(lo.Keys(nodeClaim.Spec.Resources.Requests), v1beta1.ResourceEFA) - return NewInstanceFromFleet(fleetInstance, tags, efaEnabled), nil + return NewInstanceFromFleet(fleetInstance, tags, efaEnabled, capacityType), nil } func (p *DefaultProvider) Get(ctx context.Context, id string) (*Instance, error) { @@ -193,17 +193,22 @@ func (p *DefaultProvider) CreateTags(ctx context.Context, id string, tags map[st return nil } -func (p *DefaultProvider) launchInstance(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, tags map[string]string) (*ec2.CreateFleetInstance, error) { +// workaround until capacity-reservation natively supported by EC2 API to return capacity type +func (p *DefaultProvider) launchInstance(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, tags map[string]string) (*ec2.CreateFleetInstance, string, error) { capacityType := p.getCapacityType(nodeClaim, instanceTypes) + zonalSubnets, err := p.subnetProvider.ZonalSubnetsForLaunch(ctx, nodeClass, instanceTypes, capacityType) if err != nil { - return nil, fmt.Errorf("getting subnets, %w", err) + return nil, "", fmt.Errorf("getting subnets, %w", err) } // Get Launch Template Configs, which may differ due to GPU or Architecture requirements launchTemplateConfigs, err := p.getLaunchTemplateConfigs(ctx, nodeClass, nodeClaim, instanceTypes, zonalSubnets, capacityType, tags) if err != nil { - return nil, fmt.Errorf("getting launch template configs, %w", err) + if cloudprovider.IsInsufficientCapacityError(err) { + return nil, "", cloudprovider.NewInsufficientCapacityError(fmt.Errorf("getting launch template configs, %w", err)) + } + return nil, "", fmt.Errorf("getting launch template configs, %w", err) } if err := p.checkODFallback(nodeClaim, instanceTypes, launchTemplateConfigs); err != nil { logging.FromContext(ctx).Warn(err.Error()) @@ -214,7 +219,8 @@ func (p *DefaultProvider) launchInstance(ctx context.Context, nodeClass *v1beta1 Context: nodeClass.Spec.Context, LaunchTemplateConfigs: launchTemplateConfigs, TargetCapacitySpecification: &ec2.TargetCapacitySpecificationRequest{ - DefaultTargetCapacityType: aws.String(capacityType), + // workaround until capacity-reservation natively supported by EC2 API + DefaultTargetCapacityType: lo.Ternary(capacityType == "capacity-reservation", aws.String("on-demand"), aws.String(capacityType)), TotalTargetCapacity: aws.Int64(1), }, TagSpecifications: []*ec2.TagSpecification{ @@ -226,6 +232,7 @@ func (p *DefaultProvider) launchInstance(ctx context.Context, nodeClass *v1beta1 if capacityType == corev1beta1.CapacityTypeSpot { createFleetInput.SpotOptions = &ec2.SpotOptionsRequest{AllocationStrategy: aws.String(ec2.SpotAllocationStrategyPriceCapacityOptimized)} } else { + // will handle capacity-reservation and on-demand createFleetInput.OnDemandOptions = &ec2.OnDemandOptionsRequest{AllocationStrategy: aws.String(ec2.FleetOnDemandAllocationStrategyLowestPrice)} } @@ -236,19 +243,19 @@ func (p *DefaultProvider) launchInstance(ctx context.Context, nodeClass *v1beta1 for _, lt := range launchTemplateConfigs { p.launchTemplateProvider.InvalidateCache(ctx, aws.StringValue(lt.LaunchTemplateSpecification.LaunchTemplateName), aws.StringValue(lt.LaunchTemplateSpecification.LaunchTemplateId)) } - return nil, fmt.Errorf("creating fleet %w", err) + return nil, "", fmt.Errorf("creating fleet %w", err) } var reqFailure awserr.RequestFailure if errors.As(err, &reqFailure) { - return nil, fmt.Errorf("creating fleet %w (%s)", err, reqFailure.RequestID()) + return nil, "", fmt.Errorf("creating fleet %w (%s)", err, reqFailure.RequestID()) } - return nil, fmt.Errorf("creating fleet %w", err) + return nil, "", fmt.Errorf("creating fleet %w", err) } p.updateUnavailableOfferingsCache(ctx, createFleetOutput.Errors, capacityType) if len(createFleetOutput.Instances) == 0 || len(createFleetOutput.Instances[0].InstanceIds) == 0 { - return nil, combineFleetErrors(createFleetOutput.Errors) + return nil, "", combineFleetErrors(createFleetOutput.Errors) } - return createFleetOutput.Instances[0], nil + return createFleetOutput.Instances[0], capacityType, nil } func getTags(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim) map[string]string { @@ -289,6 +296,9 @@ func (p *DefaultProvider) getLaunchTemplateConfigs(ctx context.Context, nodeClas var launchTemplateConfigs []*ec2.FleetLaunchTemplateConfigRequest launchTemplates, err := p.launchTemplateProvider.EnsureAll(ctx, nodeClass, nodeClaim, instanceTypes, capacityType, tags) if err != nil { + if cloudprovider.IsInsufficientCapacityError(err) { + return nil, cloudprovider.NewInsufficientCapacityError(fmt.Errorf("getting launch templates, %w", err)) + } return nil, fmt.Errorf("getting launch templates, %w", err) } for _, launchTemplate := range launchTemplates { @@ -361,12 +371,20 @@ func (p *DefaultProvider) updateUnavailableOfferingsCache(ctx context.Context, e } } -// getCapacityType selects spot if both constraints are flexible and there is an -// available offering. The AWS Cloud Provider defaults to [ on-demand ], so spot +// getCapacityType selects capacity-reservation or spot if both constraints are flexible and there is an +// available offering. The AWS Cloud Provider defaults to [ on-demand ], so capacity-reservation or spot // must be explicitly included in capacity type requirements. func (p *DefaultProvider) getCapacityType(nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType) string { - requirements := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim. - Spec.Requirements...) + requirements := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...) + if requirements.Get(corev1beta1.CapacityTypeLabelKey).Has("capacity-reservation") { + for _, instanceType := range instanceTypes { + for _, offering := range instanceType.Offerings.Available() { + if requirements.Get(v1.LabelTopologyZone).Has(offering.Zone) && offering.CapacityType == "capacity-reservation" { + return "capacity-reservation" + } + } + } + } if requirements.Get(corev1beta1.CapacityTypeLabelKey).Has(corev1beta1.CapacityTypeSpot) { for _, instanceType := range instanceTypes { for _, offering := range instanceType.Offerings.Available() { diff --git a/pkg/providers/instance/types.go b/pkg/providers/instance/types.go index 5f3804f2d004..6cb86273e477 100644 --- a/pkg/providers/instance/types.go +++ b/pkg/providers/instance/types.go @@ -61,7 +61,8 @@ func NewInstance(out *ec2.Instance) *Instance { } -func NewInstanceFromFleet(out *ec2.CreateFleetInstance, tags map[string]string, efaEnabled bool) *Instance { +// workaround until capacity-reservation natively supported by EC2 API to return capacity type in out.Lifecycle +func NewInstanceFromFleet(out *ec2.CreateFleetInstance, tags map[string]string, efaEnabled bool, capacityType string) *Instance { return &Instance{ LaunchTime: time.Now(), // estimate the launch time since we just launched State: ec2.StatePending, @@ -69,7 +70,7 @@ func NewInstanceFromFleet(out *ec2.CreateFleetInstance, tags map[string]string, ImageID: aws.StringValue(out.LaunchTemplateAndOverrides.Overrides.ImageId), Type: aws.StringValue(out.InstanceType), Zone: aws.StringValue(out.LaunchTemplateAndOverrides.Overrides.AvailabilityZone), - CapacityType: aws.StringValue(out.Lifecycle), + CapacityType: capacityType, SubnetID: aws.StringValue(out.LaunchTemplateAndOverrides.Overrides.SubnetId), Tags: tags, EFAEnabled: efaEnabled, diff --git a/pkg/providers/instancetype/instancetype.go b/pkg/providers/instancetype/instancetype.go index 0d86dd941e8a..178859853101 100644 --- a/pkg/providers/instancetype/instancetype.go +++ b/pkg/providers/instancetype/instancetype.go @@ -164,10 +164,28 @@ func (p *DefaultProvider) List(ctx context.Context, kc *corev1beta1.KubeletConfi // Any changes to the values passed into the NewInstanceType method will require making updates to the cache key // so that Karpenter is able to cache the set of InstanceTypes based on values that alter the set of instance types // !!! Important !!! - return NewInstanceType(ctx, i, p.region, - nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, - kc.MaxPods, kc.PodsPerCore, kc.KubeReserved, kc.SystemReserved, kc.EvictionHard, kc.EvictionSoft, - amiFamily, p.createOfferings(ctx, i, p.instanceTypeOfferings[aws.StringValue(i.InstanceType)], allZones, subnetZones)) + return NewInstanceType( + ctx, + i, + p.region, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + kc.MaxPods, + kc.PodsPerCore, + kc.KubeReserved, + kc.SystemReserved, + kc.EvictionHard, + kc.EvictionSoft, + amiFamily, + p.createOfferings( + ctx, + i, + p.instanceTypeOfferings[aws.StringValue(i.InstanceType)], + allZones, + subnetZones, + nodeClass.Status.CapacityReservations, + ), + ) }) p.instanceTypesCache.SetDefault(key, result) return result, nil @@ -251,11 +269,22 @@ func (p *DefaultProvider) UpdateInstanceTypeOfferings(ctx context.Context) error return nil } -func (p *DefaultProvider) createOfferings(ctx context.Context, instanceType *ec2.InstanceTypeInfo, instanceTypeZones, zones, subnetZones sets.Set[string]) []cloudprovider.Offering { +func (p *DefaultProvider) createOfferings( + ctx context.Context, + instanceType *ec2.InstanceTypeInfo, + instanceTypeZones, + zones, + subnetZones sets.Set[string], + capacityReservations []v1beta1.CapacityReservation, +) []cloudprovider.Offering { var offerings []cloudprovider.Offering + + // workaround until ec2 supports "capacity-reservation" as a supported class natively + supportedUsageClasses := sets.NewString(append(aws.StringValueSlice(instanceType.SupportedUsageClasses), "capacity-reservation")...) + for zone := range zones { // while usage classes should be a distinct set, there's no guarantee of that - for capacityType := range sets.NewString(aws.StringValueSlice(instanceType.SupportedUsageClasses)...) { + for capacityType := range supportedUsageClasses { // exclude any offerings that have recently seen an insufficient capacity error from EC2 isUnavailable := p.unavailableOfferings.IsUnavailable(*instanceType.InstanceType, zone, capacityType) var price float64 @@ -265,6 +294,14 @@ func (p *DefaultProvider) createOfferings(ctx context.Context, instanceType *ec2 price, ok = p.pricingProvider.SpotPrice(*instanceType.InstanceType, zone) case ec2.UsageClassTypeOnDemand: price, ok = p.pricingProvider.OnDemandPrice(*instanceType.InstanceType) + case "capacity-reservation": + // logging.FromContext(ctx).Debugf("Creating offering for capacity reservation instance type %s and zone %s", *instanceType.InstanceType, zone) + price = 0.0 + ok = hasCapacityReservation( + capacityReservations, + instanceType, + zone, + ) case "capacity-block": // ignore since karpenter doesn't support it yet, but do not log an unknown capacity type error continue @@ -279,6 +316,7 @@ func (p *DefaultProvider) createOfferings(ctx context.Context, instanceType *ec2 Price: price, Available: available, }) + // TODO: Do we want to set this for capacityType capacity-reservation? instanceTypeOfferingAvailable.With(prometheus.Labels{ instanceTypeLabel: *instanceType.InstanceType, capacityTypeLabel: capacityType, @@ -294,6 +332,29 @@ func (p *DefaultProvider) createOfferings(ctx context.Context, instanceType *ec2 return offerings } +func hasCapacityReservation( + capacityReservations []v1beta1.CapacityReservation, + instanceType *ec2.InstanceTypeInfo, + zone string, +) bool { + for _, capacityReservation := range capacityReservations { + if capacityReservation.AvailableInstanceCount == 0 { + continue + } + + if capacityReservation.AvailabilityZone != zone { + continue + } + + if capacityReservation.InstanceType != aws.StringValue(instanceType.InstanceType) { + continue + } + + return true + } + return false +} + func (p *DefaultProvider) Reset() { p.instanceTypesInfo = []*ec2.InstanceTypeInfo{} p.instanceTypeOfferings = map[string]sets.Set[string]{} diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index b90949530f6f..1f2dc7a55ec9 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -174,17 +174,18 @@ func (p *DefaultProvider) createAMIOptions(ctx context.Context, nodeClass *v1bet return nil, fmt.Errorf("no security groups are present in the status") } options := &amifamily.Options{ - ClusterName: options.FromContext(ctx).ClusterName, - ClusterEndpoint: p.ClusterEndpoint, - ClusterCIDR: p.ClusterCIDR.Load(), - InstanceProfile: instanceProfile, - InstanceStorePolicy: nodeClass.Spec.InstanceStorePolicy, - SecurityGroups: nodeClass.Status.SecurityGroups, - Tags: tags, - Labels: labels, - CABundle: p.CABundle, - KubeDNSIP: p.KubeDNSIP, - NodeClassName: nodeClass.Name, + ClusterName: options.FromContext(ctx).ClusterName, + ClusterEndpoint: p.ClusterEndpoint, + ClusterCIDR: p.ClusterCIDR.Load(), + InstanceProfile: instanceProfile, + InstanceStorePolicy: nodeClass.Spec.InstanceStorePolicy, + SecurityGroups: nodeClass.Status.SecurityGroups, + CapacityReservations: nodeClass.Status.CapacityReservations, + Tags: tags, + Labels: labels, + CABundle: p.CABundle, + KubeDNSIP: p.KubeDNSIP, + NodeClassName: nodeClass.Name, } if nodeClass.Spec.AssociatePublicIPAddress != nil { options.AssociatePublicIPAddress = nodeClass.Spec.AssociatePublicIPAddress @@ -247,6 +248,7 @@ func (p *DefaultProvider) createLaunchTemplate(ctx context.Context, options *ami launchTemplateDataTags = append(launchTemplateDataTags, &ec2.LaunchTemplateTagSpecificationRequest{ResourceType: aws.String(ec2.ResourceTypeSpotInstancesRequest), Tags: utils.MergeTags(options.Tags)}) } networkInterfaces := p.generateNetworkInterfaces(options) + capacityReservationSpecification := p.generateCapacityReservationSpecification(options) output, err := p.ec2api.CreateLaunchTemplateWithContext(ctx, &ec2.CreateLaunchTemplateInput{ LaunchTemplateName: aws.String(LaunchTemplateName(options)), LaunchTemplateData: &ec2.RequestLaunchTemplateData{ @@ -258,9 +260,10 @@ func (p *DefaultProvider) createLaunchTemplate(ctx context.Context, options *ami Enabled: aws.Bool(options.DetailedMonitoring), }, // If the network interface is defined, the security groups are defined within it - SecurityGroupIds: lo.Ternary(networkInterfaces != nil, nil, lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) })), - UserData: aws.String(userData), - ImageId: aws.String(options.AMIID), + SecurityGroupIds: lo.Ternary(networkInterfaces != nil, nil, lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) })), + CapacityReservationSpecification: capacityReservationSpecification, + UserData: aws.String(userData), + ImageId: aws.String(options.AMIID), MetadataOptions: &ec2.LaunchTemplateInstanceMetadataOptionsRequest{ HttpEndpoint: options.MetadataOptions.HTTPEndpoint, HttpProtocolIpv6: options.MetadataOptions.HTTPProtocolIPv6, @@ -313,6 +316,22 @@ func (p *DefaultProvider) generateNetworkInterfaces(options *amifamily.LaunchTem return nil } +func (p *DefaultProvider) generateCapacityReservationSpecification(options *amifamily.LaunchTemplate) *ec2.LaunchTemplateCapacityReservationSpecificationRequest { + if options == nil { + return nil + } + + if options.CapacityReservation == nil { + return nil + } + + return &ec2.LaunchTemplateCapacityReservationSpecificationRequest{ + CapacityReservationTarget: &ec2.CapacityReservationTarget{ + CapacityReservationId: &options.CapacityReservation.ID, + }, + } +} + func (p *DefaultProvider) blockDeviceMappings(blockDeviceMappings []*v1beta1.BlockDeviceMapping) []*ec2.LaunchTemplateBlockDeviceMappingRequest { if len(blockDeviceMappings) == 0 { // The EC2 API fails with empty slices and expects nil. diff --git a/pkg/test/environment.go b/pkg/test/environment.go index 81aa70575470..cfa2a4af74c1 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -30,6 +30,7 @@ import ( awscache "github.com/aws/karpenter-provider-aws/pkg/cache" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" + "github.com/aws/karpenter-provider-aws/pkg/providers/capacityreservation" "github.com/aws/karpenter-provider-aws/pkg/providers/instance" "github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile" "github.com/aws/karpenter-provider-aws/pkg/providers/instancetype" @@ -67,19 +68,21 @@ type Environment struct { AvailableIPAdressCache *cache.Cache AssociatePublicIPAddressCache *cache.Cache SecurityGroupCache *cache.Cache + CapacityReservationCache *cache.Cache InstanceProfileCache *cache.Cache // Providers - InstanceTypesProvider *instancetype.DefaultProvider - InstanceProvider *instance.DefaultProvider - SubnetProvider *subnet.DefaultProvider - SecurityGroupProvider *securitygroup.DefaultProvider - InstanceProfileProvider *instanceprofile.DefaultProvider - PricingProvider *pricing.DefaultProvider - AMIProvider *amifamily.DefaultProvider - AMIResolver *amifamily.Resolver - VersionProvider *version.DefaultProvider - LaunchTemplateProvider *launchtemplate.DefaultProvider + InstanceTypesProvider *instancetype.DefaultProvider + InstanceProvider *instance.DefaultProvider + SubnetProvider *subnet.DefaultProvider + SecurityGroupProvider *securitygroup.DefaultProvider + CapacityReservationProvider *capacityreservation.DefaultProvider + InstanceProfileProvider *instanceprofile.DefaultProvider + PricingProvider *pricing.DefaultProvider + AMIProvider *amifamily.DefaultProvider + AMIResolver *amifamily.Resolver + VersionProvider *version.DefaultProvider + LaunchTemplateProvider *launchtemplate.DefaultProvider } func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment { @@ -99,6 +102,7 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment availableIPAdressCache := cache.New(awscache.AvailableIPAddressTTL, awscache.DefaultCleanupInterval) associatePublicIPAddressCache := cache.New(awscache.AssociatePublicIPAddressTTL, awscache.DefaultCleanupInterval) securityGroupCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + capacityReservationCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) instanceProfileCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) fakePricingAPI := &fake.PricingAPI{} @@ -106,6 +110,7 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment pricingProvider := pricing.NewDefaultProvider(ctx, fakePricingAPI, ec2api, fake.DefaultRegion) subnetProvider := subnet.NewDefaultProvider(ec2api, subnetCache, availableIPAdressCache, associatePublicIPAddressCache) securityGroupProvider := securitygroup.NewDefaultProvider(ec2api, securityGroupCache) + capacityReservationProvider := capacityreservation.NewDefaultProvider(ec2api, capacityReservationCache) versionProvider := version.NewDefaultProvider(env.KubernetesInterface, kubernetesVersionCache) instanceProfileProvider := instanceprofile.NewDefaultProvider(fake.DefaultRegion, iamapi, instanceProfileCache) amiProvider := amifamily.NewDefaultProvider(versionProvider, ssmapi, ec2api, ec2Cache) @@ -149,19 +154,21 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment AvailableIPAdressCache: availableIPAdressCache, AssociatePublicIPAddressCache: associatePublicIPAddressCache, SecurityGroupCache: securityGroupCache, + CapacityReservationCache: capacityReservationCache, InstanceProfileCache: instanceProfileCache, UnavailableOfferingsCache: unavailableOfferingsCache, - InstanceTypesProvider: instanceTypesProvider, - InstanceProvider: instanceProvider, - SubnetProvider: subnetProvider, - SecurityGroupProvider: securityGroupProvider, - LaunchTemplateProvider: launchTemplateProvider, - InstanceProfileProvider: instanceProfileProvider, - PricingProvider: pricingProvider, - AMIProvider: amiProvider, - AMIResolver: amiResolver, - VersionProvider: versionProvider, + InstanceTypesProvider: instanceTypesProvider, + InstanceProvider: instanceProvider, + SubnetProvider: subnetProvider, + SecurityGroupProvider: securityGroupProvider, + CapacityReservationProvider: capacityReservationProvider, + LaunchTemplateProvider: launchTemplateProvider, + InstanceProfileProvider: instanceProfileProvider, + PricingProvider: pricingProvider, + AMIProvider: amiProvider, + AMIResolver: amiResolver, + VersionProvider: versionProvider, } }