Skip to content

Commit

Permalink
Matrix
Browse files Browse the repository at this point in the history
  • Loading branch information
engedaam committed Sep 15, 2023
1 parent c146865 commit 87a28ec
Show file tree
Hide file tree
Showing 10 changed files with 236 additions and 4 deletions.
5 changes: 1 addition & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -153,10 +153,7 @@ delete: ## Delete the controller from your ~/.kube/config cluster
helm uninstall karpenter --namespace karpenter

docgen: ## Generate docs
go run hack/docs/metrics_gen_docs.go pkg/ $(KARPENTER_CORE_DIR)/pkg website/content/en/preview/concepts/metrics.md
go run hack/docs/instancetypes_gen_docs.go website/content/en/preview/concepts/instance-types.md
go run hack/docs/configuration_gen_docs.go website/content/en/preview/concepts/settings.md
cd charts/karpenter && helm-docs
$(WITH_GOFLAGS) ./hack/docgen.sh

codegen: ## Auto generate files based on AWS APIs response
$(WITH_GOFLAGS) ./hack/codegen.sh
Expand Down
5 changes: 5 additions & 0 deletions cmd/controller/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package main

import (
"github.com/samber/lo"
"knative.dev/pkg/logging"

"github.com/aws/karpenter/pkg/cloudprovider"
"github.com/aws/karpenter/pkg/controllers"
Expand Down Expand Up @@ -43,6 +44,10 @@ func main() {
lo.Must0(op.AddHealthzCheck("cloud-provider", awsCloudProvider.LivenessProbe))
cloudProvider := metrics.Decorate(awsCloudProvider)

if err := op.ValidateK8sVersion(ctx); err != nil {
logging.FromContext(ctx).Error(err)
}

op.
WithControllers(ctx, corecontrollers.NewControllers(
ctx,
Expand Down
5 changes: 5 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ require (
github.com/avast/retry-go v3.0.0+incompatible
github.com/aws/aws-sdk-go v1.44.328
github.com/aws/karpenter-core v0.30.1-0.20230913175729-035f16f4e5b3
github.com/aws/karpenter/tools/kompat v0.0.0-20230908214340-40a7ad1934cb
github.com/imdario/mergo v0.3.16
github.com/mitchellh/hashstructure/v2 v2.0.2
github.com/onsi/ginkgo/v2 v2.11.0
Expand All @@ -32,6 +33,7 @@ require (
require (
contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d // indirect
contrib.go.opencensus.io/exporter/prometheus v0.4.0 // indirect
github.com/Masterminds/semver/v3 v3.2.1 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
Expand Down Expand Up @@ -69,10 +71,13 @@ require (
github.com/json-iterator/go v1.1.12 // indirect
github.com/kelseyhightower/envconfig v1.4.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
Expand Down
10 changes: 10 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ contrib.go.opencensus.io/exporter/prometheus v0.4.0/go.mod h1:o7cosnyfuPVK0tB8q0
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg=
github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y=
github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
Expand All @@ -55,6 +57,8 @@ github.com/aws/aws-sdk-go v1.44.328 h1:WBwlf8ym9SDQ/GTIBO9eXyvwappKJyOetWJKl4mT7
github.com/aws/aws-sdk-go v1.44.328/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/karpenter-core v0.30.1-0.20230913175729-035f16f4e5b3 h1:ReJu80eA1kK1PIIy6o2I4eElpTcy6hYu91OzPYtCIog=
github.com/aws/karpenter-core v0.30.1-0.20230913175729-035f16f4e5b3/go.mod h1:AQl8m8OtgO2N8IlZlzAU6MTrJTJSbe6K4GwdRUNSJVc=
github.com/aws/karpenter/tools/kompat v0.0.0-20230908214340-40a7ad1934cb h1:73uNH6RnEpcAUfMElphHg48xWvVJSEdc1vgOn2+bwIA=
github.com/aws/karpenter/tools/kompat v0.0.0-20230908214340-40a7ad1934cb/go.mod h1:l/TIBsaCx/IrOr0Xvlj/cHLOf05QzuQKEZ1hx2XWmfU=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
Expand Down Expand Up @@ -245,9 +249,13 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
Expand All @@ -261,6 +269,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU=
github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM=
github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
Expand Down
14 changes: 14 additions & 0 deletions hack/docgen.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#!/usr/bin/env bash
set -euo pipefail

compatibilitymatrix() {
go run hack/docs/version_compatibility.go hack/docs/compatibility-karpenter.yaml "$(git describe --exact-match --tags || echo "no tag")"
go run hack/docs/compatibilitymetrix_gen_docs.go website/content/en/preview/upgrade-guide.md hack/docs/compatibility-karpenter.yaml
}


compatibilitymatrix
go run hack/docs/metrics_gen_docs.go pkg/ $(KARPENTER_CORE_DIR)/pkg website/content/en/preview/concepts/metrics.md
go run hack/docs/instancetypes_gen_docs.go website/content/en/preview/concepts/instance-types.md
go run hack/docs/configuration_gen_docs.go website/content/en/preview/concepts/settings.md
cd charts/karpenter && helm-docs
32 changes: 32 additions & 0 deletions hack/docs/compatibility-karpenter.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
name: "karpenter"
compatibility:
- appVersion: 0.21.x
minK8sVersion: 1.21
maxK8sVersion: 1.24
- appVersion: 0.22.x
minK8sVersion: 1.21
maxK8sVersion: 1.24
- appVersion: 0.23.x
minK8sVersion: 1.21
maxK8sVersion: 1.24
- appVersion: 0.24.x
minK8sVersion: 1.21
maxK8sVersion: 1.24
- appVersion: 0.25.x
minK8sVersion: 1.21
maxK8sVersion: 1.25
- appVersion: 0.26.x
minK8sVersion: 1.21
maxK8sVersion: 1.25
- appVersion: 0.27.x
minK8sVersion: 1.21
maxK8sVersion: 1.25
- appVersion: 0.28.x
minK8sVersion: 1.23
maxK8sVersion: 1.27
- appVersion: 0.29.x
minK8sVersion: 1.23
maxK8sVersion: 1.27
- appVersion: 0.30.x
minK8sVersion: 1.23
maxK8sVersion: 1.27
58 changes: 58 additions & 0 deletions hack/docs/compatibilitymetrix_gen_docs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"fmt"
"log"
"os"
"strings"

"github.com/aws/karpenter/tools/kompat/pkg/kompat"
)

func main() {
outputFileName := os.Args[1]
mdFile, err := os.ReadFile(outputFileName)
if err != nil {
log.Printf("Can't read %s file: %v", os.Args[1], err)
os.Exit(1)
}

genStart := "[comment]: <> (the content below is generated from hack/docs/compataiblitymetrix_gen_docs.go)"
genEnd := "[comment]: <> (end docs generated content from hack/docs/compataiblitymetrix_gen_docs.go)"
startDocSections := strings.Split(string(mdFile), genStart)
if len(startDocSections) != 2 {
log.Fatalf("expected one generated comment block start but got %d", len(startDocSections)-1)
}
endDocSections := strings.Split(string(mdFile), genEnd)
if len(endDocSections) != 2 {
log.Fatalf("expected one generated comment block end but got %d", len(endDocSections)-1)
}
topDoc := fmt.Sprintf("%s%s\n\n", startDocSections[0], genStart)
bottomDoc := fmt.Sprintf("\n%s%s", genEnd, endDocSections[1])

baseText, err := kompat.Parse(os.Args[2])
if err != nil {
log.Fatalf("unable to generate compatibility matrix")
}

log.Println("writing output to", outputFileName)
f, err := os.Create(outputFileName)
if err != nil {
log.Fatalf("unable to open %s to write generated output: %v", outputFileName, err)
}
f.WriteString(topDoc + baseText.Markdown(kompat.Options{LastN: 5}) + bottomDoc)
}
67 changes: 67 additions & 0 deletions hack/docs/version_compatibility.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"fmt"
"log"
"os"
"sort"
"strings"

"github.com/aws/karpenter/tools/kompat/pkg/kompat"
)

func main() {
if len(os.Args) != 3 {
log.Fatalf("Usage: %s karpenter version", os.Args[0])
}
if os.Args[2] == "no tag" {
log.Printf("No version")
os.Exit(0)
}

chart, err := kompat.Parse(os.Args[1])
if err != nil {
log.Fatalf("unable to generate compatibility matrix")
}

sort.Slice(chart[0].Compatibility, func(i int, j int) bool {
return chart[0].Compatibility[i].AppVersion < chart[0].Compatibility[j].AppVersion
})

version := strings.TrimPrefix(os.Args[2], "v")
appendVersion := fmt.Sprintf(
`
- appVersion: %s
minK8sVersion: %s
maxK8sVersion: %s`,
version,
chart[0].Compatibility[len(chart[0].Compatibility)-1].MinK8sVersion,
chart[0].Compatibility[len(chart[0].Compatibility)-1].MaxK8sVersion)

yamlFile, err := os.ReadFile(os.Args[1])
if err != nil {
log.Printf("Can't read %s file: %v", os.Args[1], err)
os.Exit(1)
}

log.Println("writing output to", os.Args[1])
f, err := os.Create(os.Args[1])
if err != nil {
log.Fatalf("unable to open %s to write generated output: %v", os.Args[1], err)
}
f.WriteString(string(yamlFile) + appendVersion)
}
26 changes: 26 additions & 0 deletions pkg/operator/operator.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ import (
"github.com/aws/aws-sdk-go/service/eks/eksiface"
"github.com/aws/aws-sdk-go/service/ssm"
"github.com/patrickmn/go-cache"

versionv1 "k8s.io/apimachinery/pkg/util/version"

"github.com/samber/lo"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
Expand All @@ -58,6 +61,12 @@ import (
"github.com/aws/karpenter/pkg/utils/project"
)

// Karpenter's supported version of Kubernetes
// If a user runs a karpenter image on a k8s version outside the min and max,
// One error message will be fired to notify
const minK8sVersion = "1.23"
const maxK8sVersion = "1.27"

// Operator is injected into the AWS CloudProvider's factories
type Operator struct {
*operator.Operator
Expand All @@ -71,6 +80,7 @@ type Operator struct {
AMIResolver *amifamily.Resolver
LaunchTemplateProvider *launchtemplate.Provider
PricingProvider *pricing.Provider
VersionProvider *version.Provider
InstanceTypesProvider *instancetype.Provider
InstanceProvider *instance.Provider
}
Expand Down Expand Up @@ -169,13 +179,29 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont
SecurityGroupProvider: securityGroupProvider,
AMIProvider: amiProvider,
AMIResolver: amiResolver,
VersionProvider: versionProvider,
LaunchTemplateProvider: launchTemplateProvider,
PricingProvider: pricingProvider,
InstanceTypesProvider: instanceTypeProvider,
InstanceProvider: instanceProvider,
}
}

func (op *Operator) ValidateK8sVersion(ctx context.Context) error {
v := lo.Must1(op.VersionProvider.Get(ctx))
k8sVersion := versionv1.MustParseGeneric(v)

// We will only error if the user is running karpenter on a k8s version,
// that is out of the range of the minK8sVersion and maxK8sVersion
minCompare := lo.Must1(k8sVersion.Compare(minK8sVersion))
maxCompare := lo.Must1(k8sVersion.Compare(maxK8sVersion))
if minCompare == 1 && maxCompare == -1 {
return fmt.Errorf("karpenter version is not compatible with K8s version %s", k8sVersion)
}

return nil
}

// withUserAgent adds a karpenter specific user-agent string to AWS session
func withUserAgent(sess *session.Session) *session.Session {
userAgent := fmt.Sprintf("karpenter.sh-%s", project.Version)
Expand Down
18 changes: 18 additions & 0 deletions website/content/en/preview/upgrade-guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,24 @@ Use your existing upgrade mechanisms to upgrade your core add-ons in Kubernetes

To make upgrading easier we aim to minimize introduction of breaking changes with the followings:

## Compatibility Matrix

[comment]: <> (the content below is generated from hack/docs/compataiblitymetrix_gen_docs.go)

| KUBERNETES | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 |
|------------|---------|---------|---------|---------|---------|
| karpenter | 0.21.x+ | 0.21.x+ | 0.25.x+ | 0.28.x+ | 0.28.x+ |

[comment]: <> (end docs generated content from hack/docs/compataiblitymetrix_gen_docs.go)

{{% alert title="Note" color="warning" %}}
Karpenter currently does not support the following [new `topologySpreadConstraints` keys](https://kubernetes.io/blog/2023/04/17/fine-grained-pod-topology-spread-features-beta/), promoted to beta in Kubernetes 1.27:
- `matchLabelKeys`
- `nodeAffinityPolicy`
- `nodeTaintsPolicy`

For more information on Karpenter's support for these keys, view [this tracking issue](https://github.com/aws/karpenter-core/issues/430).
{{% /alert %}}
## Compatibility issues

To make upgrading easier, we aim to minimize the introduction of breaking changes with the followings components:
Expand Down

0 comments on commit 87a28ec

Please sign in to comment.