Add support for spot-instance-requests tagging for launched instances

pkg/providers/instance/instance.go

@@ -300,7 +300,7 @@ func (p *Provider) checkODFallback(nodeClaim *corev1beta1.NodeClaim, instanceTyp
 func (p *Provider) getLaunchTemplateConfigs(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim,
 	instanceTypes []*cloudprovider.InstanceType, zonalSubnets map[string]*ec2.Subnet, capacityType string, tags map[string]string) ([]*ec2.FleetLaunchTemplateConfigRequest, error) {
 	var launchTemplateConfigs []*ec2.FleetLaunchTemplateConfigRequest
-	launchTemplates, err := p.launchTemplateProvider.EnsureAll(ctx, nodeClass, nodeClaim, instanceTypes, map[string]string{corev1beta1.CapacityTypeLabelKey: capacityType}, tags)
+	launchTemplates, err := p.launchTemplateProvider.EnsureAll(ctx, nodeClass, nodeClaim, instanceTypes, capacityType, tags)
 	if err != nil {
 		return nil, fmt.Errorf("getting launch templates, %w", err)
 	}

pkg/providers/launchtemplate/launchtemplate.go

@@ -104,7 +104,7 @@ func NewProvider(ctx context.Context, cache *cache.Cache, ec2api ec2iface.EC2API
 }
 
 func (p *Provider) EnsureAll(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim,
-	instanceTypes []*cloudprovider.InstanceType, additionalLabels map[string]string, tags map[string]string) ([]*LaunchTemplate, error) {
+	instanceTypes []*cloudprovider.InstanceType, capacityType string, tags map[string]string) ([]*LaunchTemplate, error) {
 
 	p.Lock()
 	defer p.Unlock()
@@ -113,7 +113,7 @@ func (p *Provider) EnsureAll(ctx context.Context, nodeClass *v1beta1.EC2NodeClas
 		return []*LaunchTemplate{{Name: ptr.StringValue(nodeClass.Spec.LaunchTemplateName), InstanceTypes: instanceTypes}}, nil
 	}
 
-	options, err := p.createAMIOptions(ctx, nodeClass, lo.Assign(nodeClaim.Labels, additionalLabels), tags)
+	options, err := p.createAMIOptions(ctx, nodeClass, lo.Assign(nodeClaim.Labels, map[string]string{corev1beta1.CapacityTypeLabelKey: capacityType}), tags)
 	if err != nil {
 		return nil, err
 	}
@@ -124,7 +124,7 @@ func (p *Provider) EnsureAll(ctx context.Context, nodeClass *v1beta1.EC2NodeClas
 	var launchTemplates []*LaunchTemplate
 	for _, resolvedLaunchTemplate := range resolvedLaunchTemplates {
 		// Ensure the launch template exists, or create it
-		ec2LaunchTemplate, err := p.ensureLaunchTemplate(ctx, resolvedLaunchTemplate)
+		ec2LaunchTemplate, err := p.ensureLaunchTemplate(ctx, capacityType, resolvedLaunchTemplate)
 		if err != nil {
 			return nil, err
 		}
@@ -197,7 +197,7 @@ func (p *Provider) createAMIOptions(ctx context.Context, nodeClass *v1beta1.EC2N
 	return options, nil
 }
 
-func (p *Provider) ensureLaunchTemplate(ctx context.Context, options *amifamily.LaunchTemplate) (*ec2.LaunchTemplate, error) {
+func (p *Provider) ensureLaunchTemplate(ctx context.Context, capacityType string, options *amifamily.LaunchTemplate) (*ec2.LaunchTemplate, error) {
 	var launchTemplate *ec2.LaunchTemplate
 	name := launchTemplateName(options)
 	ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("launch-template-name", name))
@@ -212,7 +212,7 @@ func (p *Provider) ensureLaunchTemplate(ctx context.Context, options *amifamily.
 	})
 	// Create LT if one doesn't exist
 	if awserrors.IsNotFound(err) {
-		launchTemplate, err = p.createLaunchTemplate(ctx, options)
+		launchTemplate, err = p.createLaunchTemplate(ctx, capacityType, options)
 		if err != nil {
 			return nil, fmt.Errorf("creating launch template, %w", err)
 		}
@@ -230,11 +230,18 @@ func (p *Provider) ensureLaunchTemplate(ctx context.Context, options *amifamily.
 	return launchTemplate, nil
 }
 
-func (p *Provider) createLaunchTemplate(ctx context.Context, options *amifamily.LaunchTemplate) (*ec2.LaunchTemplate, error) {
+func (p *Provider) createLaunchTemplate(ctx context.Context, capacityType string, options *amifamily.LaunchTemplate) (*ec2.LaunchTemplate, error) {
 	userData, err := options.UserData.Script()
 	if err != nil {
 		return nil, err
 	}
+	launchTemplateDataTags := []*ec2.LaunchTemplateTagSpecificationRequest{
+		{ResourceType: aws.String(ec2.ResourceTypeNetworkInterface), Tags: utils.MergeTags(options.Tags)},
+	}
+	// Add the spot-instances-request tag if trying to launch spot capacity
+	if capacityType == corev1beta1.CapacityTypeSpot {
+		launchTemplateDataTags = append(launchTemplateDataTags, &ec2.LaunchTemplateTagSpecificationRequest{ResourceType: aws.String(ec2.ResourceTypeSpotInstancesRequest), Tags: utils.MergeTags(options.Tags)})
+	}
 	networkInterface := p.generateNetworkInterface(options)
 	output, err := p.ec2api.CreateLaunchTemplateWithContext(ctx, &ec2.CreateLaunchTemplateInput{
 		LaunchTemplateName: aws.String(launchTemplateName(options)),
@@ -257,9 +264,7 @@ func (p *Provider) createLaunchTemplate(ctx context.Context, options *amifamily.
 				HttpTokens:              options.MetadataOptions.HTTPTokens,
 			},
 			NetworkInterfaces: networkInterface,
-			TagSpecifications: []*ec2.LaunchTemplateTagSpecificationRequest{
-				{ResourceType: aws.String(ec2.ResourceTypeNetworkInterface), Tags: utils.MergeTags(options.Tags)},
-			},
+			TagSpecifications: launchTemplateDataTags,
 		},
 		TagSpecifications: []*ec2.TagSpecification{
 			{

pkg/providers/launchtemplate/suite_test.go

@@ -282,6 +282,33 @@ var _ = Describe("LaunchTemplates", func() {
 			Expect(*createFleetInput.TagSpecifications[2].ResourceType).To(Equal(ec2.ResourceTypeFleet))
 			ExpectTags(createFleetInput.TagSpecifications[2].Tags, nodeClass.Spec.Tags)
 		})
+		It("should request that tags be applied to both network interfaces and spot instance requests", func() {
+			nodeClass.Spec.Tags = map[string]string{
+				"tag1": "tag1value",
+				"tag2": "tag2value",
+			}
+			nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{
+				{
+					Key:      corev1beta1.CapacityTypeLabelKey,
+					Operator: v1.NodeSelectorOpIn,
+					Values:   []string{corev1beta1.CapacityTypeSpot},
+				},
+			}
+			ExpectApplied(ctx, env.Client, nodePool, nodeClass)
+			pod := coretest.UnschedulablePod()
+			ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod)
+			ExpectScheduled(ctx, env.Client, pod)
+			awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.ForEach(func(i *ec2.CreateLaunchTemplateInput) {
+				Expect(i.LaunchTemplateData.TagSpecifications).To(HaveLen(2))
+
+				// tags should be included in instance, volume, and fleet tag specification
+				Expect(*i.LaunchTemplateData.TagSpecifications[0].ResourceType).To(Equal(ec2.ResourceTypeNetworkInterface))
+				ExpectTags(i.LaunchTemplateData.TagSpecifications[0].Tags, nodeClass.Spec.Tags)
+
+				Expect(*i.LaunchTemplateData.TagSpecifications[1].ResourceType).To(Equal(ec2.ResourceTypeSpotInstancesRequest))
+				ExpectTags(i.LaunchTemplateData.TagSpecifications[1].Tags, nodeClass.Spec.Tags)
+			})
+		})
 		It("should override default tag names", func() {
 			// these tags are defaulted, so ensure users can override them
 			nodeClass.Spec.Tags = map[string]string{

test/pkg/environment/aws/expectations.go

@@ -172,12 +172,38 @@ func (env *Environment) GetInstanceByID(instanceID string) ec2.Instance {
 	return *instance.Reservations[0].Instances[0]
 }
 
-func (env *Environment) GetVolume(volumeID *string) ec2.Volume {
+func (env *Environment) GetVolume(id *string) *ec2.Volume {
+	volumes := env.GetVolumes(id)
+	Expect(volumes).To(HaveLen(1))
+	return volumes[0]
+}
+
+func (env *Environment) GetVolumes(ids ...*string) []*ec2.Volume {
+	GinkgoHelper()
+	dvo, err := env.EC2API.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: ids})
+	Expect(err).ToNot(HaveOccurred())
+	return dvo.Volumes
+}
+
+func (env *Environment) GetNetworkInterface(id *string) *ec2.NetworkInterface {
+	networkInterfaces := env.GetNetworkInterfaces(id)
+	Expect(networkInterfaces).To(HaveLen(1))
+	return networkInterfaces[0]
+}
+
+func (env *Environment) GetNetworkInterfaces(ids ...*string) []*ec2.NetworkInterface {
+	GinkgoHelper()
+	dnio, err := env.EC2API.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{NetworkInterfaceIds: ids})
+	Expect(err).ToNot(HaveOccurred())
+	return dnio.NetworkInterfaces
+}
+
+func (env *Environment) GetSpotInstanceRequest(id *string) *ec2.SpotInstanceRequest {
 	GinkgoHelper()
-	dvo, err := env.EC2API.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: []*string{volumeID}})
+	siro, err := env.EC2API.DescribeSpotInstanceRequests(&ec2.DescribeSpotInstanceRequestsInput{SpotInstanceRequestIds: []*string{id}})
 	Expect(err).ToNot(HaveOccurred())
-	Expect(len(dvo.Volumes)).To(Equal(1))
-	return *dvo.Volumes[0]
+	Expect(siro.SpotInstanceRequests).To(HaveLen(1))
+	return siro.SpotInstanceRequests[0]
 }
 
 // GetSubnets returns all subnets matching the label selector

test/suites/integration/tags_test.go

@@ -32,6 +32,8 @@ import (
 	"github.com/aws/karpenter/pkg/test"
 )
 
+const createdAtTag = "node.k8s.amazonaws.com/createdAt"
+
 var _ = Describe("Tags", func() {
 	Context("Static Tags", func() {
 		It("should tag all associated resources", func() {
@@ -42,11 +44,40 @@ var _ = Describe("Tags", func() {
 			env.EventuallyExpectHealthy(pod)
 			env.ExpectCreatedNodeCount("==", 1)
 			instance := env.GetInstance(pod.Spec.NodeName)
-			volumeTags := tagMap(env.GetVolume(instance.BlockDeviceMappings[0].Ebs.VolumeId).Tags)
-			instanceTags := tagMap(instance.Tags)
+			volumes := env.GetVolumes(lo.Map(instance.BlockDeviceMappings, func(bdm *ec2.InstanceBlockDeviceMapping, _ int) *string {
+				return bdm.Ebs.VolumeId
+			})...)
+			networkInterfaces := env.GetNetworkInterfaces(lo.Map(instance.NetworkInterfaces, func(ni *ec2.InstanceNetworkInterface, _ int) *string {
+				return ni.NetworkInterfaceId
+			})...)
+
+			Expect(tagMap(instance.Tags)).To(HaveKeyWithValue("TestTag", "TestVal"))
+			for _, volume := range volumes {
+				Expect(tagMap(volume.Tags)).To(HaveKeyWithValue("TestTag", "TestVal"))
+			}
+			for _, networkInterface := range networkInterfaces {
+				// Any ENI that contains this createdAt tag was created by the VPC CNI DaemonSet
+				if _, ok := tagMap(networkInterface.TagSet)[createdAtTag]; !ok {
+					Expect(tagMap(networkInterface.TagSet)).To(HaveKeyWithValue("TestTag", "TestVal"))
+				}
+			}
+		})
+		It("should tag spot instance requests when creating resources", func() {
+			coretest.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{
+				Key:      corev1beta1.CapacityTypeLabelKey,
+				Operator: v1.NodeSelectorOpIn,
+				Values:   []string{corev1beta1.CapacityTypeSpot},
+			})
+			nodeClass.Spec.Tags = map[string]string{"TestTag": "TestVal"}
+			pod := coretest.Pod()
 
-			Expect(instanceTags).To(HaveKeyWithValue("TestTag", "TestVal"))
-			Expect(volumeTags).To(HaveKeyWithValue("TestTag", "TestVal"))
+			env.ExpectCreated(pod, nodeClass, nodePool)
+			env.EventuallyExpectHealthy(pod)
+			env.ExpectCreatedNodeCount("==", 1)
+			instance := env.GetInstance(pod.Spec.NodeName)
+			Expect(instance.SpotInstanceRequestId).ToNot(BeNil())
+			spotInstanceRequest := env.GetSpotInstanceRequest(instance.SpotInstanceRequestId)
+			Expect(tagMap(spotInstanceRequest.Tags)).To(HaveKeyWithValue("TestTag", "TestVal"))
 		})
 	})
 

website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml

@@ -40,7 +40,6 @@ Resources:
               "Resource": [
                 "arn:${AWS::Partition}:ec2:${AWS::Region}::image/*",
                 "arn:${AWS::Partition}:ec2:${AWS::Region}::snapshot/*",
-                "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*",
                 "arn:${AWS::Partition}:ec2:${AWS::Region}:*:security-group/*",
                 "arn:${AWS::Partition}:ec2:${AWS::Region}:*:subnet/*",
                 "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*"
@@ -58,7 +57,8 @@ Resources:
                 "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*",
                 "arn:${AWS::Partition}:ec2:${AWS::Region}:*:volume/*",
                 "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*",
-                "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*"
+                "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*",
+                "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*"
               ],
               "Action": [
                 "ec2:RunInstances",
@@ -82,7 +82,8 @@ Resources:
                 "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*",
                 "arn:${AWS::Partition}:ec2:${AWS::Region}:*:volume/*",
                 "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*",
-                "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*"
+                "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*",
+                "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*"
               ],
               "Action": "ec2:CreateTags",
               "Condition": {