Skip to content

Commit

Permalink
chore: Handle v1beta1 labels for requirements and tags (aws#4551)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jonathan-innis
jonathan-innis authored Sep 6, 2023
1 parent a784eed commit 99caf37
Show file tree
Hide file tree
Showing 15 changed files with 152 additions and 146 deletions.
10 changes: 7 additions & 3 deletions pkg/cloudprovider/cloudprovider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ import (

func init() {
v1alpha5.NormalizedLabels = lo.Assign(v1alpha5.NormalizedLabels, map[string]string{"topology.ebs.csi.aws.com/zone": v1.LabelTopologyZone})
corev1beta1.NormalizedLabels = lo.Assign(corev1beta1.NormalizedLabels, map[string]string{"topology.ebs.csi.aws.com/zone": v1.LabelTopologyZone})
coreapis.Settings = append(coreapis.Settings, apis.Settings...)
}

Expand Down Expand Up @@ -340,12 +341,15 @@ func (c *CloudProvider) instanceToMachine(i *instance.Instance, instanceType *cl
machine.Status.Allocatable = functional.FilterMap(instanceType.Allocatable(), func(_ v1.ResourceName, v resource.Quantity) bool { return !resources.IsZero(v) })
}
labels[v1.LabelTopologyZone] = i.Zone
labels[v1alpha5.LabelCapacityType] = i.CapacityType
labels[corev1beta1.CapacityTypeLabelKey] = i.CapacityType
if v, ok := i.Tags[v1alpha5.ProvisionerNameLabelKey]; ok {
labels[v1alpha5.ProvisionerNameLabelKey] = v
}
if v, ok := i.Tags[v1alpha5.MachineManagedByAnnotationKey]; ok {
annotations[v1alpha5.MachineManagedByAnnotationKey] = v
if v, ok := i.Tags[corev1beta1.NodePoolLabelKey]; ok {
labels[corev1beta1.NodePoolLabelKey] = v
}
if v, ok := i.Tags[corev1beta1.ManagedByAnnotationKey]; ok {
annotations[corev1beta1.ManagedByAnnotationKey] = v
}
machine.Labels = labels
machine.Annotations = annotations
Expand Down
26 changes: 13 additions & 13 deletions pkg/providers/amifamily/al2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ import (
"fmt"

"github.com/aws/aws-sdk-go/aws"
"github.com/samber/lo"
v1 "k8s.io/api/core/v1"

"github.com/aws/karpenter-core/pkg/apis/v1alpha5"
corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1"
"github.com/aws/karpenter-core/pkg/scheduling"
"github.com/aws/karpenter/pkg/apis/v1beta1"
Expand All @@ -36,36 +36,36 @@ type AL2 struct {
}

// DefaultAMIs returns the AMI name, and Requirements, with an SSM query
func (a AL2) DefaultAMIs(version string) []DefaultAMIOutput {
func (a AL2) DefaultAMIs(version string, isNodeTemplate bool) []DefaultAMIOutput {
return []DefaultAMIOutput{
{
Query: fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", version),
Requirements: scheduling.NewRequirements(
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, v1alpha5.ArchitectureAmd64),
scheduling.NewRequirement(v1alpha1.LabelInstanceGPUCount, v1.NodeSelectorOpDoesNotExist),
scheduling.NewRequirement(v1alpha1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpDoesNotExist),
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64),
scheduling.NewRequirement(lo.Ternary(isNodeTemplate, v1alpha1.LabelInstanceGPUCount, v1beta1.LabelInstanceGPUCount), v1.NodeSelectorOpDoesNotExist),
scheduling.NewRequirement(lo.Ternary(isNodeTemplate, v1alpha1.LabelInstanceAcceleratorCount, v1beta1.LabelInstanceAcceleratorCount), v1.NodeSelectorOpDoesNotExist),
),
},
{
Query: fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-gpu/recommended/image_id", version),
Requirements: scheduling.NewRequirements(
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, v1alpha5.ArchitectureAmd64),
scheduling.NewRequirement(v1alpha1.LabelInstanceGPUCount, v1.NodeSelectorOpExists),
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64),
scheduling.NewRequirement(lo.Ternary(isNodeTemplate, v1alpha1.LabelInstanceGPUCount, v1beta1.LabelInstanceGPUCount), v1.NodeSelectorOpExists),
),
},
{
Query: fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-gpu/recommended/image_id", version),
Requirements: scheduling.NewRequirements(
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, v1alpha5.ArchitectureAmd64),
scheduling.NewRequirement(v1alpha1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpExists),
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64),
scheduling.NewRequirement(lo.Ternary(isNodeTemplate, v1alpha1.LabelInstanceAcceleratorCount, v1beta1.LabelInstanceAcceleratorCount), v1.NodeSelectorOpExists),
),
},
{
Query: fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-%s/recommended/image_id", version, v1alpha5.ArchitectureArm64),
Query: fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-%s/recommended/image_id", version, corev1beta1.ArchitectureArm64),
Requirements: scheduling.NewRequirements(
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, v1alpha5.ArchitectureArm64),
scheduling.NewRequirement(v1alpha1.LabelInstanceGPUCount, v1.NodeSelectorOpDoesNotExist),
scheduling.NewRequirement(v1alpha1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpDoesNotExist),
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64),
scheduling.NewRequirement(lo.Ternary(isNodeTemplate, v1alpha1.LabelInstanceGPUCount, v1beta1.LabelInstanceGPUCount), v1.NodeSelectorOpDoesNotExist),
scheduling.NewRequirement(lo.Ternary(isNodeTemplate, v1alpha1.LabelInstanceAcceleratorCount, v1beta1.LabelInstanceAcceleratorCount), v1.NodeSelectorOpDoesNotExist),
),
},
}
Expand Down
5 changes: 3 additions & 2 deletions pkg/providers/amifamily/ami.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import (
"knative.dev/pkg/logging"

"github.com/aws/karpenter-core/pkg/apis/v1alpha5"
corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1"
"github.com/aws/karpenter/pkg/apis/v1alpha1"
"github.com/aws/karpenter/pkg/apis/v1beta1"

Expand Down Expand Up @@ -169,7 +170,7 @@ func (p *Provider) getDefaultAMIs(ctx context.Context, nodeClass *v1beta1.NodeCl
if err != nil {
return nil, fmt.Errorf("getting kubernetes version %w", err)
}
defaultAMIs := amiFamily.DefaultAMIs(kubernetesVersion)
defaultAMIs := amiFamily.DefaultAMIs(kubernetesVersion, nodeClass.IsNodeTemplate)
for _, ami := range defaultAMIs {
if id, err := p.resolveSSMParameter(ctx, ami.Query); err != nil {
logging.FromContext(ctx).With("query", ami.Query).Errorf("discovering amis from ssm, %s", err)
Expand Down Expand Up @@ -303,7 +304,7 @@ func GetFilterAndOwnerSets(terms []v1beta1.AMISelectorTerm) (res []FiltersAndOwn
func (p *Provider) getRequirementsFromImage(ec2Image *ec2.Image) scheduling.Requirements {
requirements := scheduling.NewRequirements()
for _, tag := range ec2Image.Tags {
if v1alpha5.WellKnownLabels.Has(*tag.Key) {
if v1alpha5.WellKnownLabels.Has(*tag.Key) || corev1beta1.WellKnownLabels.Has(*tag.Key) {
requirements.Add(scheduling.NewRequirement(*tag.Key, v1.NodeSelectorOpIn, *tag.Value))
}
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/providers/amifamily/bootstrap/bootstrap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ func (o Options) nodeLabelArg() string {
keys := lo.Keys(o.Labels)
sort.Strings(keys) // ensures this list is deterministic, for easy testing.
for _, key := range keys {
if v1alpha5.LabelDomainExceptions.Has(key) {
if v1alpha5.LabelDomainExceptions.Has(key) || corev1beta1.LabelDomainExceptions.Has(key) {
continue
}
labelStrings = append(labelStrings, fmt.Sprintf("%s=%v", key, o.Labels[key]))
Expand Down
38 changes: 18 additions & 20 deletions pkg/providers/amifamily/bottlerocket.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,6 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"

"github.com/aws/karpenter-core/pkg/apis/v1alpha5"

"github.com/aws/karpenter/pkg/apis/v1alpha1"
)

Expand All @@ -41,50 +39,50 @@ type Bottlerocket struct {
}

// DefaultAMIs returns the AMI name, and Requirements, with an SSM query
func (b Bottlerocket) DefaultAMIs(version string) []DefaultAMIOutput {
func (b Bottlerocket) DefaultAMIs(version string, isNodeTemplate bool) []DefaultAMIOutput {
return []DefaultAMIOutput{
{
Query: fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/x86_64/latest/image_id", version),
Requirements: scheduling.NewRequirements(
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, v1alpha5.ArchitectureAmd64),
scheduling.NewRequirement(v1alpha1.LabelInstanceGPUCount, v1.NodeSelectorOpDoesNotExist),
scheduling.NewRequirement(v1alpha1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpDoesNotExist),
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64),
scheduling.NewRequirement(lo.Ternary(isNodeTemplate, v1alpha1.LabelInstanceGPUCount, v1beta1.LabelInstanceGPUCount), v1.NodeSelectorOpDoesNotExist),
scheduling.NewRequirement(lo.Ternary(isNodeTemplate, v1alpha1.LabelInstanceAcceleratorCount, v1beta1.LabelInstanceAcceleratorCount), v1.NodeSelectorOpDoesNotExist),
),
},
{
Query: fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s-nvidia/x86_64/latest/image_id", version),
Requirements: scheduling.NewRequirements(
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, v1alpha5.ArchitectureAmd64),
scheduling.NewRequirement(v1alpha1.LabelInstanceGPUCount, v1.NodeSelectorOpExists),
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64),
scheduling.NewRequirement(lo.Ternary(isNodeTemplate, v1alpha1.LabelInstanceGPUCount, v1beta1.LabelInstanceGPUCount), v1.NodeSelectorOpExists),
),
},
{
Query: fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s-nvidia/x86_64/latest/image_id", version),
Requirements: scheduling.NewRequirements(
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, v1alpha5.ArchitectureAmd64),
scheduling.NewRequirement(v1alpha1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpExists),
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64),
scheduling.NewRequirement(lo.Ternary(isNodeTemplate, v1alpha1.LabelInstanceAcceleratorCount, v1beta1.LabelInstanceAcceleratorCount), v1.NodeSelectorOpExists),
),
},
{
Query: fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/%s/latest/image_id", version, v1alpha5.ArchitectureArm64),
Query: fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/%s/latest/image_id", version, corev1beta1.ArchitectureArm64),
Requirements: scheduling.NewRequirements(
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, v1alpha5.ArchitectureArm64),
scheduling.NewRequirement(v1alpha1.LabelInstanceGPUCount, v1.NodeSelectorOpDoesNotExist),
scheduling.NewRequirement(v1alpha1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpDoesNotExist),
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64),
scheduling.NewRequirement(lo.Ternary(isNodeTemplate, v1alpha1.LabelInstanceGPUCount, v1beta1.LabelInstanceGPUCount), v1.NodeSelectorOpDoesNotExist),
scheduling.NewRequirement(lo.Ternary(isNodeTemplate, v1alpha1.LabelInstanceAcceleratorCount, v1beta1.LabelInstanceAcceleratorCount), v1.NodeSelectorOpDoesNotExist),
),
},
{
Query: fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s-nvidia/%s/latest/image_id", version, v1alpha5.ArchitectureArm64),
Query: fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s-nvidia/%s/latest/image_id", version, corev1beta1.ArchitectureArm64),
Requirements: scheduling.NewRequirements(
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, v1alpha5.ArchitectureArm64),
scheduling.NewRequirement(v1alpha1.LabelInstanceGPUCount, v1.NodeSelectorOpExists),
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64),
scheduling.NewRequirement(lo.Ternary(isNodeTemplate, v1alpha1.LabelInstanceGPUCount, v1beta1.LabelInstanceGPUCount), v1.NodeSelectorOpExists),
),
},
{
Query: fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s-nvidia/%s/latest/image_id", version, v1alpha5.ArchitectureArm64),
Query: fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s-nvidia/%s/latest/image_id", version, corev1beta1.ArchitectureArm64),
Requirements: scheduling.NewRequirements(
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, v1alpha5.ArchitectureArm64),
scheduling.NewRequirement(v1alpha1.LabelInstanceAcceleratorCount, v1.NodeSelectorOpExists),
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64),
scheduling.NewRequirement(lo.Ternary(isNodeTemplate, v1alpha1.LabelInstanceAcceleratorCount, v1beta1.LabelInstanceAcceleratorCount), v1.NodeSelectorOpExists),
),
},
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/providers/amifamily/custom.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ func (c Custom) UserData(_ *corev1beta1.KubeletConfiguration, _ []v1.Taint, _ ma
}
}

func (c Custom) DefaultAMIs(_ string) []DefaultAMIOutput {
func (c Custom) DefaultAMIs(_ string, _ bool) []DefaultAMIOutput {
return nil
}

Expand Down
4 changes: 2 additions & 2 deletions pkg/providers/amifamily/resolver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ type Options struct {
InstanceProfile string
CABundle *string `hash:"ignore"`
// Level-triggered fields that may change out of sync.
SecurityGroups []v1alpha1.SecurityGroup
SecurityGroups []v1beta1.SecurityGroup
Tags map[string]string
Labels map[string]string `hash:"ignore"`
KubeDNSIP net.IP
Expand All @@ -74,7 +74,7 @@ type LaunchTemplate struct {

// AMIFamily can be implemented to override the default logic for generating dynamic launch template parameters
type AMIFamily interface {
DefaultAMIs(version string) []DefaultAMIOutput
DefaultAMIs(version string, isNodeTemplate bool) []DefaultAMIOutput
UserData(kubeletConfig *corev1beta1.KubeletConfiguration, taints []core.Taint, labels map[string]string, caBundle *string, instanceTypes []*cloudprovider.InstanceType, customUserData *string) bootstrap.Bootstrapper
DefaultBlockDeviceMappings() []*v1beta1.BlockDeviceMapping
DefaultMetadataOptions() *v1beta1.MetadataOptions
Expand Down
11 changes: 5 additions & 6 deletions pkg/providers/amifamily/ubuntu.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
"github.com/aws/karpenter/pkg/apis/v1beta1"
"github.com/aws/karpenter/pkg/providers/amifamily/bootstrap"

"github.com/aws/karpenter-core/pkg/apis/v1alpha5"
"github.com/aws/karpenter-core/pkg/cloudprovider"
"github.com/aws/karpenter-core/pkg/scheduling"
)
Expand All @@ -35,18 +34,18 @@ type Ubuntu struct {
}

// DefaultAMIs returns the AMI name, and Requirements, with an SSM query
func (u Ubuntu) DefaultAMIs(version string) []DefaultAMIOutput {
func (u Ubuntu) DefaultAMIs(version string, _ bool) []DefaultAMIOutput {
return []DefaultAMIOutput{
{
Query: fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/%s/hvm/ebs-gp2/ami-id", version, v1alpha5.ArchitectureAmd64),
Query: fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/%s/hvm/ebs-gp2/ami-id", version, corev1beta1.ArchitectureAmd64),
Requirements: scheduling.NewRequirements(
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, v1alpha5.ArchitectureAmd64),
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64),
),
},
{
Query: fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/%s/hvm/ebs-gp2/ami-id", version, v1alpha5.ArchitectureArm64),
Query: fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/%s/hvm/ebs-gp2/ami-id", version, corev1beta1.ArchitectureArm64),
Requirements: scheduling.NewRequirements(
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, v1alpha5.ArchitectureArm64),
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64),
),
},
}
Expand Down
5 changes: 2 additions & 3 deletions pkg/providers/amifamily/windows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ import (

v1 "k8s.io/api/core/v1"

"github.com/aws/karpenter-core/pkg/apis/v1alpha5"
"github.com/aws/karpenter-core/pkg/cloudprovider"

"github.com/aws/karpenter/pkg/apis/v1alpha1"
Expand All @@ -43,12 +42,12 @@ type Windows struct {
Build string
}

func (w Windows) DefaultAMIs(version string) []DefaultAMIOutput {
func (w Windows) DefaultAMIs(version string, _ bool) []DefaultAMIOutput {
return []DefaultAMIOutput{
{
Query: fmt.Sprintf("/aws/service/ami-windows-latest/Windows_Server-%s-English-%s-EKS_Optimized-%s/image_id", w.Version, v1alpha1.WindowsCore, version),
Requirements: scheduling.NewRequirements(
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, v1alpha5.ArchitectureAmd64),
scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64),
scheduling.NewRequirement(v1.LabelOSStable, v1.NodeSelectorOpIn, string(v1.Windows)),
scheduling.NewRequirement(v1.LabelWindowsBuild, v1.NodeSelectorOpIn, w.Build),
),
Expand Down
Loading

0 comments on commit 99caf37

Please sign in to comment.