Plumbing for host resource group provider into launchtemplates

cmd/controller/main.go

@@ -67,6 +67,7 @@ func main() {
 			op.PricingProvider,
 			op.AMIProvider,
             op.LicenseProvider,
+            op.HostResourceGroupProvider,
 		)...).
 		WithWebhooks(ctx, webhooks.NewWebhooks()...).
 		Start(ctx)

pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml

@@ -129,7 +129,7 @@ spec:
                 description: DetailedMonitoring controls if detailed monitoring is
                   enabled for instances that are launched
                 type: boolean
-              hostResourceGroup:
+              hostResourceGroupSelector:
                 additionalProperties:
                   type: string
                 description: HostResourceGroups specific the hostResourceGroupArns
@@ -204,7 +204,7 @@ spec:
                       credentials are not available."
                     type: string
                 type: object
-              placementGroup:
+              placementGroupSelector:
                 additionalProperties:
                   type: string
                 description: PlacementGroup specifies the placement group to use for

pkg/apis/v1alpha1/provider.go

@@ -46,10 +46,10 @@ type AWS struct {
 	LicenseSelector map[string]string `json:"licenseSelector,omitempty" hash:"ignore"`
 	// HostResourceGroups specific the hostResourceGroupArns to use for ec2 placement
 	// +optional
-	HostResourceGroupSelector map[string]string `json:"hostResourceGroup,omitempty" hash:"ignore"`
+	HostResourceGroupSelector map[string]string `json:"hostResourceGroupSelector,omitempty" hash:"ignore"`
 	// PlacementGroup specifies the placement group to use for ec2 placement
 	// +optional
-	PlacementGroupSelector map[string]string `json:"placementGroup,omitempty" hash:"ignore"`
+	PlacementGroupSelector map[string]string `json:"placementGroupSelector,omitempty" hash:"ignore"`
 	// Tags to be applied on ec2 resources like instances and launch templates.
 	// +optional
 	Tags map[string]string `json:"tags,omitempty"`

pkg/apis/v1beta1/ec2nodeclass.go

@@ -107,10 +107,15 @@ type EC2NodeClassSpec struct {
 	// +optional
 	OriginalAMISelector map[string]string `json:"-" hash:"ignore"`
 	// TODO @joinnis: Remove this field when v1alpha5 is unsupported in a future version of Karpenter
-	// OriginalAMISelector is the original ami selector that was used by the v1alpha5 representation of this API.
+	// OriginalLicenseSelector is the original license selector that was used by the v1alpha5 representation of this API.
 	// DO NOT USE THIS VALUE when performing business logic in code
 	// +optional
 	OriginalLicenseSelector map[string]string `json:"-" hash:"ignore"`
+	// TODO @joinnis: Remove this field when v1alpha5 is unsupported in a future version of Karpenter
+	// OriginalHostResourceGroupSelector is the original hrg selector that was used by the v1alpha5 representation of this API.
+	// DO NOT USE THIS VALUE when performing business logic in code
+	// +optional
+	OriginalHostResourceGroupSelector map[string]string `json:"-" hash:"ignore"`
 }
 
 // SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes.

pkg/apis/v1beta1/ec2nodeclass_status.go

@@ -78,5 +78,5 @@ type EC2NodeClassStatus struct {
     Licenses []string `json:"licenses,omitempty"`
     // HostResourceGroups contains the HRG arns
     // +optional
-    HostResourceGroup HostResourceGroup `json:"hostResourceGroup,omitempty"`
+    HostResourceGroup *HostResourceGroup `json:"hostResourceGroup,omitempty"`
 }

pkg/controllers/controllers.go

@@ -32,6 +32,7 @@ import (
 	nodeclaimlink "github.com/aws/karpenter/pkg/controllers/nodeclaim/link"
 	"github.com/aws/karpenter/pkg/controllers/nodeclass"
 	"github.com/aws/karpenter/pkg/providers/amifamily"
+	"github.com/aws/karpenter/pkg/providers/hostresourcegroup"
 	"github.com/aws/karpenter/pkg/providers/license"
 	"github.com/aws/karpenter/pkg/providers/pricing"
 	"github.com/aws/karpenter/pkg/providers/securitygroup"
@@ -43,13 +44,13 @@ import (
 
 func NewControllers(ctx context.Context, sess *session.Session, clk clock.Clock, kubeClient client.Client, recorder events.Recorder,
 	unavailableOfferings *cache.UnavailableOfferings, cloudProvider *cloudprovider.CloudProvider, subnetProvider *subnet.Provider,
-	securityGroupProvider *securitygroup.Provider, pricingProvider *pricing.Provider, amiProvider *amifamily.Provider, licenseProvider *license.Provider) []controller.Controller {
+	securityGroupProvider *securitygroup.Provider, pricingProvider *pricing.Provider, amiProvider *amifamily.Provider, licenseProvider *license.Provider, hostResourceGroupProvider *hostresourcegroup.Provider) []controller.Controller {
 
 	logging.FromContext(ctx).With("version", project.Version).Debugf("discovered version")
 
 	linkController := nodeclaimlink.NewController(kubeClient, cloudProvider)
 	controllers := []controller.Controller{
-		nodeclass.NewNodeTemplateController(kubeClient, subnetProvider, securityGroupProvider, amiProvider, licenseProvider),
+		nodeclass.NewNodeTemplateController(kubeClient, subnetProvider, securityGroupProvider, amiProvider, licenseProvider, hostResourceGroupProvider),
 		linkController,
 		nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider, linkController),
 	}

pkg/controllers/nodeclass/controller.go

@@ -38,28 +38,31 @@ import (
 	"github.com/aws/karpenter/pkg/apis/v1alpha1"
 	"github.com/aws/karpenter/pkg/apis/v1beta1"
 	"github.com/aws/karpenter/pkg/providers/amifamily"
+	"github.com/aws/karpenter/pkg/providers/hostresourcegroup"
 	"github.com/aws/karpenter/pkg/providers/license"
 	"github.com/aws/karpenter/pkg/providers/securitygroup"
 	"github.com/aws/karpenter/pkg/providers/subnet"
 	nodeclassutil "github.com/aws/karpenter/pkg/utils/nodeclass"
 )
 
 type Controller struct {
-	kubeClient            client.Client
-	subnetProvider        *subnet.Provider
-	securityGroupProvider *securitygroup.Provider
-	amiProvider           *amifamily.Provider
-	licenseProvider       *license.Provider
+	kubeClient                client.Client
+	subnetProvider            *subnet.Provider
+	securityGroupProvider     *securitygroup.Provider
+	amiProvider               *amifamily.Provider
+	licenseProvider           *license.Provider
+	hostResourceGroupProvider *hostresourcegroup.Provider
 }
 
 func NewController(kubeClient client.Client, subnetProvider *subnet.Provider,
-	securityGroupProvider *securitygroup.Provider, amiProvider *amifamily.Provider, licenseProvider *license.Provider) *Controller {
+	securityGroupProvider *securitygroup.Provider, amiProvider *amifamily.Provider, licenseProvider *license.Provider, hostresourcegroupProvider *hostresourcegroup.Provider) *Controller {
 	return &Controller{
-		kubeClient:            kubeClient,
-		subnetProvider:        subnetProvider,
-		securityGroupProvider: securityGroupProvider,
-		amiProvider:           amiProvider,
-		licenseProvider:       licenseProvider,
+		kubeClient:                kubeClient,
+		subnetProvider:            subnetProvider,
+		securityGroupProvider:     securityGroupProvider,
+		amiProvider:               amiProvider,
+		licenseProvider:           licenseProvider,
+		hostResourceGroupProvider: hostresourcegroupProvider,
 	}
 }
 
@@ -71,6 +74,7 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeCl
 		c.resolveSecurityGroups(ctx, nodeClass),
 		c.resolveAMIs(ctx, nodeClass),
 		c.resolveLicenses(ctx, nodeClass),
+		c.resolveHostResourceGroups(ctx, nodeClass),
 	)
 	if !equality.Semantic.DeepEqual(stored, nodeClass) {
 		statusCopy := nodeClass.DeepCopy()
@@ -167,21 +171,26 @@ func (c *Controller) resolveLicenses(ctx context.Context, nodeClass *v1beta1.Nod
 
 }
 
+func (c *Controller) resolveHostResourceGroups(ctx context.Context, nodeClass *v1beta1.NodeClass) error {
+    result , err :=  c.hostResourceGroupProvider.Get(ctx, nodeClass)
+    if err != nil {
+        return err
+    }
+
+    nodeClass.Status.HostResourceGroup = result
+
+	return nil
+}
+
 //nolint:revive
 type NodeClassController struct {
 	*Controller
 }
 
 func NewNodeClassController(kubeClient client.Client, subnetProvider *subnet.Provider,
-<<<<<<< HEAD
-	securityGroupProvider *securitygroup.Provider, amiProvider *amifamily.Provider) corecontroller.Controller {
+	securityGroupProvider *securitygroup.Provider, amiProvider *amifamily.Provider, licenseProvider *license.Provider, hostresourcegroupProvider *hostresourcegroup.Provider) corecontroller.Controller {
 	return corecontroller.Typed[*v1beta1.EC2NodeClass](kubeClient, &NodeClassController{
-		Controller: NewController(kubeClient, subnetProvider, securityGroupProvider, amiProvider),
-=======
-	securityGroupProvider *securitygroup.Provider, amiProvider *amifamily.Provider, licenseProvider *license.Provider) corecontroller.Controller {
-	return corecontroller.Typed[*v1beta1.NodeClass](kubeClient, &NodeClassController{
-		Controller: NewController(kubeClient, subnetProvider, securityGroupProvider, amiProvider, licenseProvider),
->>>>>>> 7b9eb759 (More license provider scaffolding)
+		Controller: NewController(kubeClient, subnetProvider, securityGroupProvider, amiProvider, licenseProvider, hostresourcegroupProvider),
 	})
 }
 
@@ -209,9 +218,9 @@ type NodeTemplateController struct {
 }
 
 func NewNodeTemplateController(kubeClient client.Client, subnetProvider *subnet.Provider,
-	securityGroupProvider *securitygroup.Provider, amiProvider *amifamily.Provider, licenseProvider *license.Provider) corecontroller.Controller {
+	securityGroupProvider *securitygroup.Provider, amiProvider *amifamily.Provider, licenseProvider *license.Provider, hostresourcegroupProvider *hostresourcegroup.Provider) corecontroller.Controller {
 	return corecontroller.Typed[*v1alpha1.AWSNodeTemplate](kubeClient, &NodeTemplateController{
-		Controller: NewController(kubeClient, subnetProvider, securityGroupProvider, amiProvider, licenseProvider),
+		Controller: NewController(kubeClient, subnetProvider, securityGroupProvider, amiProvider, licenseProvider, hostresourcegroupProvider),
 	})
 }
 

pkg/operator/operator.go

@@ -30,11 +30,12 @@ import (
 	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/licensemanager"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
 	"github.com/aws/aws-sdk-go/service/eks"
 	"github.com/aws/aws-sdk-go/service/eks/eksiface"
+	"github.com/aws/aws-sdk-go/service/licensemanager"
+	"github.com/aws/aws-sdk-go/service/resourcegroups"
 	"github.com/aws/aws-sdk-go/service/ssm"
 	"github.com/patrickmn/go-cache"
 	"github.com/samber/lo"
@@ -49,6 +50,7 @@ import (
 	"github.com/aws/karpenter/pkg/apis/settings"
 	awscache "github.com/aws/karpenter/pkg/cache"
 	"github.com/aws/karpenter/pkg/providers/amifamily"
+	"github.com/aws/karpenter/pkg/providers/hostresourcegroup"
 	"github.com/aws/karpenter/pkg/providers/instance"
 	"github.com/aws/karpenter/pkg/providers/instancetype"
 	"github.com/aws/karpenter/pkg/providers/launchtemplate"
@@ -76,6 +78,7 @@ type Operator struct {
 	InstanceTypesProvider     *instancetype.Provider
 	InstanceProvider          *instance.Provider
 	LicenseProvider           *license.Provider
+	HostResourceGroupProvider *hostresourcegroup.Provider
 }
 
 func NewOperator(ctx context.Context, operator *operator.Operator) (context.Context, *Operator) {
@@ -132,16 +135,17 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont
 	)
 	versionProvider := version.NewProvider(operator.KubernetesInterface, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval))
 	amiProvider := amifamily.NewProvider(versionProvider, ssm.New(sess), ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval))
-	licenseProvider := license.NewProvider(licensemanager.New(sess) , cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval))
-	amiResolver := amifamily.New(amiProvider, licenseProvider)
+	licenseProvider := license.NewProvider(licensemanager.New(sess), cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval))
+    hostresourcegroupProvider := hostresourcegroup.NewProvider(resourcegroups.New(sess), cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval))
+	amiResolver := amifamily.New(amiProvider, licenseProvider, hostresourcegroupProvider)
 	launchTemplateProvider := launchtemplate.NewProvider(
 		ctx,
 		cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval),
 		ec2api,
 		amiResolver,
 		securityGroupProvider,
 		subnetProvider,
-        licenseProvider,
+		licenseProvider,
 		lo.Must(getCABundle(ctx, operator.GetConfig())),
 		operator.Elected(),
 		kubeDNSIP,
@@ -164,7 +168,6 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont
 		subnetProvider,
 		launchTemplateProvider,
 	)
-
 
 	return ctx, &Operator{
 		Operator:                  operator,
@@ -180,6 +183,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont
 		InstanceTypesProvider:     instanceTypeProvider,
 		InstanceProvider:          instanceProvider,
 		LicenseProvider:           licenseProvider,
+        HostResourceGroupProvider: hostresourcegroupProvider,
 	}
 }
 

pkg/providers/amifamily/resolver.go

@@ -30,6 +30,7 @@ import (
 	"github.com/aws/karpenter/pkg/apis/v1alpha1"
 	"github.com/aws/karpenter/pkg/apis/v1beta1"
 	"github.com/aws/karpenter/pkg/providers/amifamily/bootstrap"
+	"github.com/aws/karpenter/pkg/providers/hostresourcegroup"
 	"github.com/aws/karpenter/pkg/providers/license"
 
 	"github.com/aws/karpenter-core/pkg/cloudprovider"
@@ -44,8 +45,9 @@ var DefaultEBS = v1beta1.BlockDevice{
 
 // Resolver is able to fill-in dynamic launch template parameters
 type Resolver struct {
-	amiProvider     *Provider
-	licenseProvider *license.Provider
+	amiProvider               *Provider
+	licenseProvider           *license.Provider
+	hostResourceGroupProvider *hostresourcegroup.Provider
 }
 
 // Options define the static launch template parameters
@@ -73,6 +75,12 @@ type LaunchTemplate struct {
 	InstanceTypes       []*cloudprovider.InstanceType `hash:"ignore"`
 	DetailedMonitoring  bool
 	Licenses            []string
+	Placement           *Placement
+}
+
+// Placement holds the dynamically generated launch template placement parameters
+type Placement struct {
+	HostResourceGroup string
 }
 
 // AMIFamily can be implemented to override the default logic for generating dynamic launch template parameters
@@ -111,10 +119,11 @@ func (d DefaultFamily) FeatureFlags() FeatureFlags {
 }
 
 // New constructs a new launch template Resolver
-func New(amiProvider *Provider, licenseProvider *license.Provider) *Resolver {
+func New(amiProvider *Provider, licenseProvider *license.Provider, hostResourceGroupProvider *hostresourcegroup.Provider) *Resolver {
 	return &Resolver{
-		amiProvider:     amiProvider,
-		licenseProvider: licenseProvider,
+		amiProvider:               amiProvider,
+		licenseProvider:           licenseProvider,
+		hostResourceGroupProvider: hostResourceGroupProvider,
 	}
 }
 
@@ -137,6 +146,16 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1beta1.EC2NodeClass,
 	if err != nil {
 		return nil, err
 	}
+    var placement *Placement
+	hrg, err := r.hostResourceGroupProvider.Get(ctx, nodeClass)
+	if err != nil {
+		return nil, err
+	}
+    if hrg == nil {
+        placement = nil
+    } else {
+        placement = &Placement{HostResourceGroup: hrg.ARN}
+    }
 	var resolvedTemplates []*LaunchTemplate
 	for amiID, instanceTypes := range mappedAMIs {
 		maxPodsToInstanceTypes := lo.GroupBy(instanceTypes, func(instanceType *cloudprovider.InstanceType) int {
@@ -170,7 +189,8 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1beta1.EC2NodeClass,
 				DetailedMonitoring:  aws.BoolValue(nodeClass.Spec.DetailedMonitoring),
 				AMIID:               amiID,
 				InstanceTypes:       instanceTypes,
-                Licenses: licenses,
+				Licenses:            licenses,
+				Placement:           placement,
 			}
 			if len(resolved.BlockDeviceMappings) == 0 {
 				resolved.BlockDeviceMappings = amiFamily.DefaultBlockDeviceMappings()

pkg/providers/launchtemplate/launchtemplate.go

@@ -251,6 +251,7 @@ func (p *Provider) createLaunchTemplate(ctx context.Context, options *amifamily.
 				{ResourceType: aws.String(ec2.ResourceTypeNetworkInterface), Tags: utils.MergeTags(options.Tags)},
 			},
             LicenseSpecifications: generateLicenseSpecification(options.Licenses),
+            Placement: generatePlacement(options.Placement),
 		},
 		TagSpecifications: []*ec2.TagSpecification{
 			{
@@ -266,6 +267,16 @@ func (p *Provider) createLaunchTemplate(ctx context.Context, options *amifamily.
 	return output.LaunchTemplate, nil
 }
 
+func generatePlacement(placement *amifamily.Placement) *ec2.LaunchTemplatePlacementRequest {
+    if placement == nil {
+        return nil
+    }
+    return &ec2.LaunchTemplatePlacementRequest{
+        HostResourceGroupArn: aws.String(placement.HostResourceGroup),
+    }
+
+}
+
 func generateLicenseSpecification(licenses []string) []*ec2.LaunchTemplateLicenseConfigurationRequest {
     result := []*ec2.LaunchTemplateLicenseConfigurationRequest{}
     if len(licenses) == 0 {