Skip to content

Commit

Permalink
chore: Bump Karpenter Version to include the kubeReserved and systemR…
Browse files Browse the repository at this point in the history
…eserved fix (aws#5947)
  • Loading branch information
engedaam authored Apr 1, 2024
1 parent e6104ca commit 9c3e165
Show file tree
Hide file tree
Showing 13 changed files with 118 additions and 161 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ require (
k8s.io/utils v0.0.0-20240102154912-e7106e64919e
knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd
sigs.k8s.io/controller-runtime v0.17.2
sigs.k8s.io/karpenter v0.35.1-0.20240329173516-3d51e28cb05f
sigs.k8s.io/karpenter v0.35.1-0.20240401162338-a1c2c16bd5d9
sigs.k8s.io/yaml v1.4.0
)

Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -759,8 +759,8 @@ sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeG
sigs.k8s.io/controller-runtime v0.17.2/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
sigs.k8s.io/karpenter v0.35.1-0.20240329173516-3d51e28cb05f h1:cWAVRywFiixzP+diq/V3W2Hr+knaL6ZDVM8G13eneyk=
sigs.k8s.io/karpenter v0.35.1-0.20240329173516-3d51e28cb05f/go.mod h1:3XwJdKrr/gQwMzGjHXq897+m7XlIFXYZfCjftMjg86o=
sigs.k8s.io/karpenter v0.35.1-0.20240401162338-a1c2c16bd5d9 h1:cGovLkuGYjUvfo0mCXHqBQHW2Q1BpjGDq3zAkWvhjEk=
sigs.k8s.io/karpenter v0.35.1-0.20240401162338-a1c2c16bd5d9/go.mod h1:3XwJdKrr/gQwMzGjHXq897+m7XlIFXYZfCjftMjg86o=
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
Expand Down
10 changes: 2 additions & 8 deletions pkg/apis/crds/karpenter.sh_nodeclaims.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -140,11 +140,8 @@ spec:
type: integer
kubeReserved:
additionalProperties:
anyOf:
- type: integer
- type: string
type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: KubeReserved contains resources reserved for Kubernetes system components.
type: object
x-kubernetes-validations:
Expand All @@ -169,11 +166,8 @@ spec:
type: integer
systemReserved:
additionalProperties:
anyOf:
- type: integer
- type: string
type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: SystemReserved contains resources reserved for OS system daemons and kernel memory.
type: object
x-kubernetes-validations:
Expand Down
10 changes: 2 additions & 8 deletions pkg/apis/crds/karpenter.sh_nodepools.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -266,11 +266,8 @@ spec:
type: integer
kubeReserved:
additionalProperties:
anyOf:
- type: integer
- type: string
type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: KubeReserved contains resources reserved for Kubernetes system components.
type: object
x-kubernetes-validations:
Expand All @@ -295,11 +292,8 @@ spec:
type: integer
systemReserved:
additionalProperties:
anyOf:
- type: integer
- type: string
type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: SystemReserved contains resources reserved for OS system daemons and kernel memory.
type: object
x-kubernetes-validations:
Expand Down
20 changes: 2 additions & 18 deletions pkg/providers/amifamily/bootstrap/bootstrap.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,12 @@ import (
"sort"
"strings"

"github.com/mitchellh/hashstructure/v2"
"github.com/samber/lo"
core "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"knative.dev/pkg/ptr"

corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1"
"sigs.k8s.io/karpenter/pkg/utils/resources"

"github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
)
Expand Down Expand Up @@ -59,8 +57,8 @@ func (o Options) kubeletExtraArgs() (args []string) {
args = append(args, fmt.Sprintf("--pods-per-core=%d", ptr.Int32Value(o.KubeletConfig.PodsPerCore)))
}
// We have to convert some of these maps so that their values return the correct string
args = append(args, joinParameterArgs("--system-reserved", resources.StringMap(o.KubeletConfig.SystemReserved), "="))
args = append(args, joinParameterArgs("--kube-reserved", resources.StringMap(o.KubeletConfig.KubeReserved), "="))
args = append(args, joinParameterArgs("--system-reserved", o.KubeletConfig.SystemReserved, "="))
args = append(args, joinParameterArgs("--kube-reserved", o.KubeletConfig.KubeReserved, "="))
args = append(args, joinParameterArgs("--eviction-hard", o.KubeletConfig.EvictionHard, "<"))
args = append(args, joinParameterArgs("--eviction-soft", o.KubeletConfig.EvictionSoft, "<"))
args = append(args, joinParameterArgs("--eviction-soft-grace-period", lo.MapValues(o.KubeletConfig.EvictionSoftGracePeriod, func(v metav1.Duration, _ string) string { return v.Duration.String() }), "="))
Expand Down Expand Up @@ -104,17 +102,6 @@ func (o Options) nodeLabelArg() string {
return fmt.Sprintf("--node-labels=%q", strings.Join(labelStrings, ","))
}

// TODO: jmdeal@ remove once KubeletConfiguration can be properly hashed
// For more information on the resource.Quantity hash issue: https://github.com/aws/karpenter-provider-aws/issues/5447
func (o Options) HashReservedResources() string {
kubeReservedHash, systemReservedHash := uint64(0), uint64(0)
if kc := o.KubeletConfig; kc != nil {
kubeReservedHash, _ = hashstructure.Hash(resources.StringMap(kc.KubeReserved), hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true})
systemReservedHash, _ = hashstructure.Hash(resources.StringMap(kc.SystemReserved), hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true})
}
return fmt.Sprintf("%d-%d", kubeReservedHash, systemReservedHash)
}

// joinParameterArgs joins a map of keys and values by their separator. The separator will sit between the
// arguments in a comma-separated list i.e. arg1<sep>val1,arg2<sep>val2
func joinParameterArgs[K comparable, V any](name string, m map[K]V, separator string) string {
Expand All @@ -135,7 +122,4 @@ func joinParameterArgs[K comparable, V any](name string, m map[K]V, separator st
// Examples are the Bottlerocket config and the eks-bootstrap script
type Bootstrapper interface {
Script() (string, error)

// TODO: jmdeal@ remove once KubeletConfiguration can be properly hashed
HashReservedResources() string
}
6 changes: 2 additions & 4 deletions pkg/providers/amifamily/bootstrap/bottlerocket.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@ import (
"github.com/imdario/mergo"
"github.com/samber/lo"

"sigs.k8s.io/karpenter/pkg/utils/resources"

"github.com/aws/aws-sdk-go/aws"
)

Expand Down Expand Up @@ -60,10 +58,10 @@ func (b Bottlerocket) Script() (string, error) {
s.Settings.Kubernetes.ClusterDNSIP = &b.KubeletConfig.ClusterDNS[0]
}
if b.KubeletConfig.SystemReserved != nil {
s.Settings.Kubernetes.SystemReserved = resources.StringMap(b.KubeletConfig.SystemReserved)
s.Settings.Kubernetes.SystemReserved = b.KubeletConfig.SystemReserved
}
if b.KubeletConfig.KubeReserved != nil {
s.Settings.Kubernetes.KubeReserved = resources.StringMap(b.KubeletConfig.KubeReserved)
s.Settings.Kubernetes.KubeReserved = b.KubeletConfig.KubeReserved
}
if b.KubeletConfig.EvictionHard != nil {
s.Settings.Kubernetes.EvictionHard = b.KubeletConfig.EvictionHard
Expand Down
12 changes: 1 addition & 11 deletions pkg/providers/instancetype/instancetype.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ import (

"sigs.k8s.io/karpenter/pkg/cloudprovider"
"sigs.k8s.io/karpenter/pkg/utils/pretty"
"sigs.k8s.io/karpenter/pkg/utils/resources"
)

const (
Expand Down Expand Up @@ -108,15 +107,8 @@ func (p *Provider) List(ctx context.Context, kc *corev1beta1.KubeletConfiguratio
// Compute fully initialized instance types hash key
subnetZonesHash, _ := hashstructure.Hash(subnetZones, hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true})
kcHash, _ := hashstructure.Hash(kc, hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true})
// TODO: remove kubeReservedHash and systemReservedHash once v1.ResourceList objects are hashed as strings in KubeletConfiguration
// For more information on the v1.ResourceList hash issue: https://github.com/kubernetes-sigs/karpenter/issues/1080
kubeReservedHash, systemReservedHash := uint64(0), uint64(0)
if kc != nil {
kubeReservedHash, _ = hashstructure.Hash(resources.StringMap(kc.KubeReserved), hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true})
systemReservedHash, _ = hashstructure.Hash(resources.StringMap(kc.SystemReserved), hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true})
}
blockDeviceMappingsHash, _ := hashstructure.Hash(nodeClass.Spec.BlockDeviceMappings, hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true})
key := fmt.Sprintf("%d-%d-%d-%016x-%016x-%016x-%s-%s-%016x-%016x",
key := fmt.Sprintf("%d-%d-%d-%016x-%016x-%016x-%s-%s",
p.instanceTypesSeqNum,
p.instanceTypeOfferingsSeqNum,
p.unavailableOfferings.SeqNum,
Expand All @@ -125,8 +117,6 @@ func (p *Provider) List(ctx context.Context, kc *corev1beta1.KubeletConfiguratio
blockDeviceMappingsHash,
aws.StringValue((*string)(nodeClass.Spec.InstanceStorePolicy)),
aws.StringValue(nodeClass.Spec.AMIFamily),
kubeReservedHash,
systemReservedHash,
)
if item, ok := p.cache.Get(key); ok {
return item.([]*cloudprovider.InstanceType), nil
Expand Down
112 changes: 56 additions & 56 deletions pkg/providers/instancetype/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -848,10 +848,10 @@ var _ = Describe("InstanceTypes", func() {
})
It("should override system reserved cpus when specified", func() {
nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{
SystemReserved: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
v1.ResourceMemory: resource.MustParse("20Gi"),
v1.ResourceEphemeralStorage: resource.MustParse("10Gi"),
SystemReserved: map[string]string{
string(v1.ResourceCPU): "2",
string(v1.ResourceMemory): "20Gi",
string(v1.ResourceEphemeralStorage): "10Gi",
},
}
it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil)
Expand All @@ -869,15 +869,15 @@ var _ = Describe("InstanceTypes", func() {
})
It("should override kube reserved when specified", func() {
it := instancetype.NewInstanceType(ctx, info, &corev1beta1.KubeletConfiguration{
SystemReserved: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("1"),
v1.ResourceMemory: resource.MustParse("20Gi"),
v1.ResourceEphemeralStorage: resource.MustParse("1Gi"),
SystemReserved: map[string]string{
string(v1.ResourceCPU): "1",
string(v1.ResourceMemory): "20Gi",
string(v1.ResourceEphemeralStorage): "1Gi",
},
KubeReserved: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
v1.ResourceMemory: resource.MustParse("10Gi"),
v1.ResourceEphemeralStorage: resource.MustParse("2Gi"),
KubeReserved: map[string]string{
string(v1.ResourceCPU): "2",
string(v1.ResourceMemory): "10Gi",
string(v1.ResourceEphemeralStorage): "2Gi",
},
}, fake.DefaultRegion, nodeClass, nil)
Expect(it.Overhead.KubeReserved.Cpu().String()).To(Equal("2"))
Expand All @@ -894,11 +894,11 @@ var _ = Describe("InstanceTypes", func() {
Context("Eviction Hard", func() {
It("should override eviction threshold when specified as a quantity", func() {
nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{
SystemReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("20Gi"),
SystemReserved: map[string]string{
string(v1.ResourceMemory): "20Gi",
},
KubeReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("10Gi"),
KubeReserved: map[string]string{
string(v1.ResourceMemory): "10Gi",
},
EvictionHard: map[string]string{
instancetype.MemoryAvailable: "500Mi",
Expand All @@ -909,11 +909,11 @@ var _ = Describe("InstanceTypes", func() {
})
It("should override eviction threshold when specified as a percentage value", func() {
nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{
SystemReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("20Gi"),
SystemReserved: map[string]string{
string(v1.ResourceMemory): "20Gi",
},
KubeReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("10Gi"),
KubeReserved: map[string]string{
string(v1.ResourceMemory): "10Gi",
},
EvictionHard: map[string]string{
instancetype.MemoryAvailable: "10%",
Expand All @@ -924,11 +924,11 @@ var _ = Describe("InstanceTypes", func() {
})
It("should consider the eviction threshold disabled when specified as 100%", func() {
nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{
SystemReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("20Gi"),
SystemReserved: map[string]string{
string(v1.ResourceMemory): "20Gi",
},
KubeReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("10Gi"),
KubeReserved: map[string]string{
string(v1.ResourceMemory): "10Gi",
},
EvictionHard: map[string]string{
instancetype.MemoryAvailable: "100%",
Expand All @@ -939,11 +939,11 @@ var _ = Describe("InstanceTypes", func() {
})
It("should used default eviction threshold for memory when evictionHard not specified", func() {
nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{
SystemReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("20Gi"),
SystemReserved: map[string]string{
string(v1.ResourceMemory): "20Gi",
},
KubeReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("10Gi"),
KubeReserved: map[string]string{
string(v1.ResourceMemory): "10Gi",
},
EvictionSoft: map[string]string{
instancetype.MemoryAvailable: "50Mi",
Expand All @@ -956,11 +956,11 @@ var _ = Describe("InstanceTypes", func() {
Context("Eviction Soft", func() {
It("should override eviction threshold when specified as a quantity", func() {
nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{
SystemReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("20Gi"),
SystemReserved: map[string]string{
string(v1.ResourceMemory): "20Gi",
},
KubeReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("10Gi"),
KubeReserved: map[string]string{
string(v1.ResourceMemory): "10Gi",
},
EvictionSoft: map[string]string{
instancetype.MemoryAvailable: "500Mi",
Expand All @@ -971,11 +971,11 @@ var _ = Describe("InstanceTypes", func() {
})
It("should override eviction threshold when specified as a percentage value", func() {
nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{
SystemReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("20Gi"),
SystemReserved: map[string]string{
string(v1.ResourceMemory): "20Gi",
},
KubeReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("10Gi"),
KubeReserved: map[string]string{
string(v1.ResourceMemory): "10Gi",
},
EvictionHard: map[string]string{
instancetype.MemoryAvailable: "5%",
Expand All @@ -989,11 +989,11 @@ var _ = Describe("InstanceTypes", func() {
})
It("should consider the eviction threshold disabled when specified as 100%", func() {
nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{
SystemReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("20Gi"),
SystemReserved: map[string]string{
string(v1.ResourceMemory): "20Gi",
},
KubeReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("10Gi"),
KubeReserved: map[string]string{
string(v1.ResourceMemory): "10Gi",
},
EvictionSoft: map[string]string{
instancetype.MemoryAvailable: "100%",
Expand All @@ -1005,11 +1005,11 @@ var _ = Describe("InstanceTypes", func() {
It("should ignore eviction threshold when using Bottlerocket AMI", func() {
nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket
nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{
SystemReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("20Gi"),
SystemReserved: map[string]string{
string(v1.ResourceMemory): "20Gi",
},
KubeReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("10Gi"),
KubeReserved: map[string]string{
string(v1.ResourceMemory): "10Gi",
},
EvictionHard: map[string]string{
instancetype.MemoryAvailable: "1Gi",
Expand All @@ -1030,11 +1030,11 @@ var _ = Describe("InstanceTypes", func() {
})
It("should take the greater of evictionHard and evictionSoft for overhead as a value", func() {
nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{
SystemReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("20Gi"),
SystemReserved: map[string]string{
string(v1.ResourceMemory): "20Gi",
},
KubeReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("10Gi"),
KubeReserved: map[string]string{
string(v1.ResourceMemory): "10Gi",
},
EvictionSoft: map[string]string{
instancetype.MemoryAvailable: "3Gi",
Expand All @@ -1048,11 +1048,11 @@ var _ = Describe("InstanceTypes", func() {
})
It("should take the greater of evictionHard and evictionSoft for overhead as a value", func() {
nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{
SystemReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("20Gi"),
SystemReserved: map[string]string{
string(v1.ResourceMemory): "20Gi",
},
KubeReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("10Gi"),
KubeReserved: map[string]string{
string(v1.ResourceMemory): "10Gi",
},
EvictionSoft: map[string]string{
instancetype.MemoryAvailable: "2%",
Expand All @@ -1066,11 +1066,11 @@ var _ = Describe("InstanceTypes", func() {
})
It("should take the greater of evictionHard and evictionSoft for overhead with mixed percentage/value", func() {
nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{
SystemReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("20Gi"),
SystemReserved: map[string]string{
string(v1.ResourceMemory): "20Gi",
},
KubeReserved: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("10Gi"),
KubeReserved: map[string]string{
string(v1.ResourceMemory): "10Gi",
},
EvictionSoft: map[string]string{
instancetype.MemoryAvailable: "10%",
Expand Down
Loading

0 comments on commit 9c3e165

Please sign in to comment.