diff --git a/Makefile b/Makefile index 0eea2816ae43..031239718e0c 100644 --- a/Makefile +++ b/Makefile @@ -153,10 +153,7 @@ delete: ## Delete the controller from your ~/.kube/config cluster helm uninstall karpenter --namespace karpenter docgen: ## Generate docs - go run hack/docs/metrics_gen_docs.go pkg/ $(KARPENTER_CORE_DIR)/pkg website/content/en/preview/concepts/metrics.md - go run hack/docs/instancetypes_gen_docs.go website/content/en/preview/concepts/instance-types.md - go run hack/docs/configuration_gen_docs.go website/content/en/preview/concepts/settings.md - cd charts/karpenter && helm-docs + $(WITH_GOFLAGS) ./hack/docgen.sh codegen: ## Auto generate files based on AWS APIs response $(WITH_GOFLAGS) ./hack/codegen.sh diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 0146572b9f2c..80ce6bad9dcc 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -16,6 +16,7 @@ package main import ( "github.com/samber/lo" + "knative.dev/pkg/logging" "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/controllers" @@ -43,6 +44,10 @@ func main() { lo.Must0(op.AddHealthzCheck("cloud-provider", awsCloudProvider.LivenessProbe)) cloudProvider := metrics.Decorate(awsCloudProvider) + if err := op.ValidateK8sVersion(ctx); err != nil { + logging.FromContext(ctx).Error(err) + } + op. WithControllers(ctx, corecontrollers.NewControllers( ctx, diff --git a/go.mod b/go.mod index 02cc86605c03..98bf4ec62e45 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.44.328 github.com/aws/karpenter-core v0.30.1-0.20230913175729-035f16f4e5b3 + github.com/aws/karpenter/tools/kompat v0.0.0-20230908214340-40a7ad1934cb github.com/imdario/mergo v0.3.16 github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/onsi/ginkgo/v2 v2.11.0 @@ -32,6 +33,7 @@ require ( require ( contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d // indirect contrib.go.opencensus.io/exporter/prometheus v0.4.0 // indirect + github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -69,10 +71,13 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.42.0 // indirect diff --git a/go.sum b/go.sum index ba805ff16c23..c2e746e284c1 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ contrib.go.opencensus.io/exporter/prometheus v0.4.0/go.mod h1:o7cosnyfuPVK0tB8q0 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg= github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= @@ -55,6 +57,8 @@ github.com/aws/aws-sdk-go v1.44.328 h1:WBwlf8ym9SDQ/GTIBO9eXyvwappKJyOetWJKl4mT7 github.com/aws/aws-sdk-go v1.44.328/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/karpenter-core v0.30.1-0.20230913175729-035f16f4e5b3 h1:ReJu80eA1kK1PIIy6o2I4eElpTcy6hYu91OzPYtCIog= github.com/aws/karpenter-core v0.30.1-0.20230913175729-035f16f4e5b3/go.mod h1:AQl8m8OtgO2N8IlZlzAU6MTrJTJSbe6K4GwdRUNSJVc= +github.com/aws/karpenter/tools/kompat v0.0.0-20230908214340-40a7ad1934cb h1:73uNH6RnEpcAUfMElphHg48xWvVJSEdc1vgOn2+bwIA= +github.com/aws/karpenter/tools/kompat v0.0.0-20230908214340-40a7ad1934cb/go.mod h1:l/TIBsaCx/IrOr0Xvlj/cHLOf05QzuQKEZ1hx2XWmfU= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -245,9 +249,13 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -261,6 +269,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= diff --git a/hack/docgen.sh b/hack/docgen.sh new file mode 100755 index 000000000000..f1fb0116df47 --- /dev/null +++ b/hack/docgen.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +compatibilitymatrix() { + go run hack/docs/version_compatibility.go hack/docs/compatibility-karpenter.yaml "$(git describe --exact-match --tags || echo "no tag")" + go run hack/docs/compatibilitymetrix_gen_docs.go website/content/en/preview/upgrade-guide.md hack/docs/compatibility-karpenter.yaml +} + + +compatibilitymatrix +go run hack/docs/metrics_gen_docs.go pkg/ $(KARPENTER_CORE_DIR)/pkg website/content/en/preview/concepts/metrics.md +go run hack/docs/instancetypes_gen_docs.go website/content/en/preview/concepts/instance-types.md +go run hack/docs/configuration_gen_docs.go website/content/en/preview/concepts/settings.md +cd charts/karpenter && helm-docs \ No newline at end of file diff --git a/hack/docs/compatibility-karpenter.yaml b/hack/docs/compatibility-karpenter.yaml new file mode 100644 index 000000000000..fedbba2be9e5 --- /dev/null +++ b/hack/docs/compatibility-karpenter.yaml @@ -0,0 +1,32 @@ +name: "karpenter" +compatibility: + - appVersion: 0.21.x + minK8sVersion: 1.21 + maxK8sVersion: 1.24 + - appVersion: 0.22.x + minK8sVersion: 1.21 + maxK8sVersion: 1.24 + - appVersion: 0.23.x + minK8sVersion: 1.21 + maxK8sVersion: 1.24 + - appVersion: 0.24.x + minK8sVersion: 1.21 + maxK8sVersion: 1.24 + - appVersion: 0.25.x + minK8sVersion: 1.21 + maxK8sVersion: 1.25 + - appVersion: 0.26.x + minK8sVersion: 1.21 + maxK8sVersion: 1.25 + - appVersion: 0.27.x + minK8sVersion: 1.21 + maxK8sVersion: 1.25 + - appVersion: 0.28.x + minK8sVersion: 1.23 + maxK8sVersion: 1.27 + - appVersion: 0.29.x + minK8sVersion: 1.23 + maxK8sVersion: 1.27 + - appVersion: 0.30.x + minK8sVersion: 1.23 + maxK8sVersion: 1.27 \ No newline at end of file diff --git a/hack/docs/compatibilitymetrix_gen_docs.go b/hack/docs/compatibilitymetrix_gen_docs.go new file mode 100644 index 000000000000..eeaeb75196f7 --- /dev/null +++ b/hack/docs/compatibilitymetrix_gen_docs.go @@ -0,0 +1,58 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "log" + "os" + "strings" + + "github.com/aws/karpenter/tools/kompat/pkg/kompat" +) + +func main() { + outputFileName := os.Args[1] + mdFile, err := os.ReadFile(outputFileName) + if err != nil { + log.Printf("Can't read %s file: %v", os.Args[1], err) + os.Exit(2) + } + + genStart := "[comment]: <> (the content below is generated from hack/docs/compataiblitymetrix_gen_docs.go)" + genEnd := "[comment]: <> (end docs generated content from hack/docs/compataiblitymetrix_gen_docs.go)" + startDocSections := strings.Split(string(mdFile), genStart) + if len(startDocSections) != 2 { + log.Fatalf("expected one generated comment block start but got %d", len(startDocSections)-1) + } + endDocSections := strings.Split(string(mdFile), genEnd) + if len(endDocSections) != 2 { + log.Fatalf("expected one generated comment block end but got %d", len(endDocSections)-1) + } + topDoc := fmt.Sprintf("%s%s\n\n", startDocSections[0], genStart) + bottomDoc := fmt.Sprintf("\n%s%s", genEnd, endDocSections[1]) + + baseText, err := kompat.Parse(os.Args[2]) + if err != nil { + log.Fatalf("unable to generate compatibility matrix") + } + + log.Println("writing output to", outputFileName) + f, err := os.Create(outputFileName) + if err != nil { + log.Fatalf("unable to open %s to write generated output: %v", outputFileName, err) + } + f.WriteString(topDoc + baseText.Markdown(kompat.Options{LastN: 5}) + bottomDoc) +} diff --git a/hack/docs/version_compatibility.go b/hack/docs/version_compatibility.go new file mode 100644 index 000000000000..49a52b445816 --- /dev/null +++ b/hack/docs/version_compatibility.go @@ -0,0 +1,67 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "log" + "os" + "sort" + "strings" + + "github.com/aws/karpenter/tools/kompat/pkg/kompat" +) + +func main() { + if len(os.Args) != 3 { + log.Fatalf("Usage: %s karpenter version", os.Args[0]) + } + if os.Args[2] == "no tag" { + log.Printf("No version") + os.Exit(0) + } + + chart, err := kompat.Parse(os.Args[1]) + if err != nil { + log.Fatalf("unable to generate compatibility matrix") + } + + sort.Slice(chart[0].Compatibility, func(i int, j int) bool { + return chart[0].Compatibility[i].AppVersion < chart[0].Compatibility[j].AppVersion + }) + + version := strings.TrimPrefix(os.Args[2], "v") + appendVersion := fmt.Sprintf( + ` + - appVersion: %s + minK8sVersion: %s + maxK8sVersion: %s`, + version, + chart[0].Compatibility[len(chart[0].Compatibility)-1].MinK8sVersion, + chart[0].Compatibility[len(chart[0].Compatibility)-1].MaxK8sVersion) + + yamlFile, err := os.ReadFile(os.Args[1]) + if err != nil { + log.Printf("Can't read %s file: %v", os.Args[1], err) + os.Exit(2) + } + + log.Println("writing output to", os.Args[1]) + f, err := os.Create(os.Args[1]) + if err != nil { + log.Fatalf("unable to open %s to write generated output: %v", os.Args[1], err) + } + f.WriteString(string(yamlFile) + appendVersion) +} diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index bc3d94232b6a..84bdb9031838 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -36,6 +36,9 @@ import ( "github.com/aws/aws-sdk-go/service/eks/eksiface" "github.com/aws/aws-sdk-go/service/ssm" "github.com/patrickmn/go-cache" + + versionv1 "k8s.io/apimachinery/pkg/util/version" + "github.com/samber/lo" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -58,6 +61,12 @@ import ( "github.com/aws/karpenter/pkg/utils/project" ) +// Karpenter's supported version of Kubernetes +// If a user runs a karpenter image on a k8s version outside the min and max, +// One error message will be fired to notify +const minK8sVersion = "1.23" +const maxK8sVersion = "1.27" + // Operator is injected into the AWS CloudProvider's factories type Operator struct { *operator.Operator @@ -71,6 +80,7 @@ type Operator struct { AMIResolver *amifamily.Resolver LaunchTemplateProvider *launchtemplate.Provider PricingProvider *pricing.Provider + VersionProvider *version.Provider InstanceTypesProvider *instancetype.Provider InstanceProvider *instance.Provider } @@ -169,6 +179,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont SecurityGroupProvider: securityGroupProvider, AMIProvider: amiProvider, AMIResolver: amiResolver, + VersionProvider: versionProvider, LaunchTemplateProvider: launchTemplateProvider, PricingProvider: pricingProvider, InstanceTypesProvider: instanceTypeProvider, @@ -176,6 +187,21 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont } } +func (op *Operator) ValidateK8sVersion(ctx context.Context) error { + v := lo.Must1(op.VersionProvider.Get(ctx)) + k8sVersion := versionv1.MustParseGeneric(v) + + // We will only error if the user is running karpenter on a k8s version, + // that is out of the range of the minK8sVersion and maxK8sVersion + minCompare := lo.Must1(k8sVersion.Compare(minK8sVersion)) + maxCompare := lo.Must1(k8sVersion.Compare(maxK8sVersion)) + if minCompare == 1 && maxCompare == -1 { + return fmt.Errorf("karpenter version is not compatible with K8s version %s", k8sVersion) + } + + return nil +} + // withUserAgent adds a karpenter specific user-agent string to AWS session func withUserAgent(sess *session.Session) *session.Session { userAgent := fmt.Sprintf("karpenter.sh-%s", project.Version) diff --git a/website/content/en/preview/upgrade-guide.md b/website/content/en/preview/upgrade-guide.md index 1843ac8b525b..be43698e536c 100644 --- a/website/content/en/preview/upgrade-guide.md +++ b/website/content/en/preview/upgrade-guide.md @@ -11,6 +11,24 @@ Use your existing upgrade mechanisms to upgrade your core add-ons in Kubernetes To make upgrading easier we aim to minimize introduction of breaking changes with the followings: +## Compatibility Matrix + +[comment]: <> (the content below is generated from hack/docs/compataiblitymetrix_gen_docs.go) + +| KUBERNETES | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 | +|------------|---------|---------|---------|---------|---------| +| karpenter | 0.21.x+ | 0.21.x+ | 0.25.x+ | 0.28.x+ | 0.28.x+ | + +[comment]: <> (end docs generated content from hack/docs/compataiblitymetrix_gen_docs.go) + +{{% alert title="Note" color="warning" %}} +Karpenter currently does not support the following [new `topologySpreadConstraints` keys](https://kubernetes.io/blog/2023/04/17/fine-grained-pod-topology-spread-features-beta/), promoted to beta in Kubernetes 1.27: +- `matchLabelKeys` +- `nodeAffinityPolicy` +- `nodeTaintsPolicy` + +For more information on Karpenter's support for these keys, view [this tracking issue](https://github.com/aws/karpenter-core/issues/430). +{{% /alert %}} ## Compatibility issues To make upgrading easier, we aim to minimize the introduction of breaking changes with the followings components: