diff --git a/Makefile b/Makefile index 0eea2816ae43..031239718e0c 100644 --- a/Makefile +++ b/Makefile @@ -153,10 +153,7 @@ delete: ## Delete the controller from your ~/.kube/config cluster helm uninstall karpenter --namespace karpenter docgen: ## Generate docs - go run hack/docs/metrics_gen_docs.go pkg/ $(KARPENTER_CORE_DIR)/pkg website/content/en/preview/concepts/metrics.md - go run hack/docs/instancetypes_gen_docs.go website/content/en/preview/concepts/instance-types.md - go run hack/docs/configuration_gen_docs.go website/content/en/preview/concepts/settings.md - cd charts/karpenter && helm-docs + $(WITH_GOFLAGS) ./hack/docgen.sh codegen: ## Auto generate files based on AWS APIs response $(WITH_GOFLAGS) ./hack/codegen.sh diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 0146572b9f2c..d3d845f0561d 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -15,7 +15,11 @@ limitations under the License. package main import ( + "fmt" + "strconv" + "github.com/samber/lo" + "knative.dev/pkg/logging" "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/controllers" @@ -29,6 +33,9 @@ import ( corewebhooks "github.com/aws/karpenter-core/pkg/webhooks" ) +const minK8sVersion = 1.23 +const maxK8sVersion = 1.27 + func main() { ctx, op := operator.NewOperator(coreoperator.NewOperator()) awsCloudProvider := cloudprovider.New( @@ -43,6 +50,19 @@ func main() { lo.Must0(op.AddHealthzCheck("cloud-provider", awsCloudProvider.LivenessProbe)) cloudProvider := metrics.Decorate(awsCloudProvider) + serverVersion, err := op.VersionProvider.Get(ctx) + if err != nil { + logging.FromContext(ctx).Error(err) + } + + floatServerVersion, err := strconv.ParseFloat(serverVersion, 32) + if err != nil { + logging.FromContext(ctx).Error(err) + } + if minK8sVersion > floatServerVersion && floatServerVersion < maxK8sVersion { + logging.FromContext(ctx).Error(fmt.Errorf("karpenter version is not compatible with K8s version %s", serverVersion)) + } + op. WithControllers(ctx, corecontrollers.NewControllers( ctx, diff --git a/go.mod b/go.mod index 02cc86605c03..98bf4ec62e45 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.44.328 github.com/aws/karpenter-core v0.30.1-0.20230913175729-035f16f4e5b3 + github.com/aws/karpenter/tools/kompat v0.0.0-20230908214340-40a7ad1934cb github.com/imdario/mergo v0.3.16 github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/onsi/ginkgo/v2 v2.11.0 @@ -32,6 +33,7 @@ require ( require ( contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d // indirect contrib.go.opencensus.io/exporter/prometheus v0.4.0 // indirect + github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -69,10 +71,13 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.42.0 // indirect diff --git a/go.sum b/go.sum index ba805ff16c23..c2e746e284c1 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ contrib.go.opencensus.io/exporter/prometheus v0.4.0/go.mod h1:o7cosnyfuPVK0tB8q0 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg= github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= @@ -55,6 +57,8 @@ github.com/aws/aws-sdk-go v1.44.328 h1:WBwlf8ym9SDQ/GTIBO9eXyvwappKJyOetWJKl4mT7 github.com/aws/aws-sdk-go v1.44.328/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/karpenter-core v0.30.1-0.20230913175729-035f16f4e5b3 h1:ReJu80eA1kK1PIIy6o2I4eElpTcy6hYu91OzPYtCIog= github.com/aws/karpenter-core v0.30.1-0.20230913175729-035f16f4e5b3/go.mod h1:AQl8m8OtgO2N8IlZlzAU6MTrJTJSbe6K4GwdRUNSJVc= +github.com/aws/karpenter/tools/kompat v0.0.0-20230908214340-40a7ad1934cb h1:73uNH6RnEpcAUfMElphHg48xWvVJSEdc1vgOn2+bwIA= +github.com/aws/karpenter/tools/kompat v0.0.0-20230908214340-40a7ad1934cb/go.mod h1:l/TIBsaCx/IrOr0Xvlj/cHLOf05QzuQKEZ1hx2XWmfU= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -245,9 +249,13 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -261,6 +269,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= diff --git a/hack/code/version_compatibility.go b/hack/code/version_compatibility.go new file mode 100644 index 000000000000..d73d1eff6a7e --- /dev/null +++ b/hack/code/version_compatibility.go @@ -0,0 +1,77 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "log" + "net/http" + "os" + "sort" + "strings" + + "github.com/PuerkitoBio/goquery" + "github.com/aws/karpenter/tools/kompat/pkg/kompat" + "github.com/samber/lo" +) + +var uri = "https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html" +var outputFileName = "hack/compatibility-karpenter.yaml" + +func main() { + if len(os.Args) != 2 { + log.Fatalf("Usage: %s karpenter version", os.Args[0]) + } + if os.Args[1] == "no tag" { + log.Printf("No version") + os.Exit(0) + } + response := lo.Must(http.Get(uri)) + defer response.Body.Close() + + doc := lo.Must(goquery.NewDocumentFromReader(response.Body)) + supportedEKSversions := doc.Find("ul").Eq(0).Find("li").Find("p").Find("code").Nodes + fmt.Print(len(supportedEKSversions)) + + chart, err := kompat.Parse(outputFileName) + if err != nil { + log.Fatalf("unable to generate compatibility matrix") + } + + sort.Slice(chart[0].Compatibility, func(i int, j int) bool { + return chart[0].Compatibility[i].AppVersion < chart[0].Compatibility[j].AppVersion + }) + + len := len(chart[0].Compatibility) + version := strings.TrimPrefix(os.Args[1], "v") + appendVersion := fmt.Sprintf( + ` + - appVersion: %s + minK8sVersion: %s + maxK8sVersion: %s`, version, chart[0].Compatibility[len-1].MinK8sVersion, chart[0].Compatibility[len-1].MaxK8sVersion) + + yamlFile, err := os.ReadFile(outputFileName) + if err != nil { + log.Printf("Can't read %s file: %v", os.Args[1], err) + os.Exit(2) + } + + log.Println("writing output to", outputFileName) + f, err := os.Create(outputFileName) + if err != nil { + log.Fatalf("unable to open %s to write generated output: %v", outputFileName, err) + } + f.WriteString(string(yamlFile) + appendVersion) +} diff --git a/hack/compatibility-karpenter.yaml b/hack/compatibility-karpenter.yaml new file mode 100644 index 000000000000..fedbba2be9e5 --- /dev/null +++ b/hack/compatibility-karpenter.yaml @@ -0,0 +1,32 @@ +name: "karpenter" +compatibility: + - appVersion: 0.21.x + minK8sVersion: 1.21 + maxK8sVersion: 1.24 + - appVersion: 0.22.x + minK8sVersion: 1.21 + maxK8sVersion: 1.24 + - appVersion: 0.23.x + minK8sVersion: 1.21 + maxK8sVersion: 1.24 + - appVersion: 0.24.x + minK8sVersion: 1.21 + maxK8sVersion: 1.24 + - appVersion: 0.25.x + minK8sVersion: 1.21 + maxK8sVersion: 1.25 + - appVersion: 0.26.x + minK8sVersion: 1.21 + maxK8sVersion: 1.25 + - appVersion: 0.27.x + minK8sVersion: 1.21 + maxK8sVersion: 1.25 + - appVersion: 0.28.x + minK8sVersion: 1.23 + maxK8sVersion: 1.27 + - appVersion: 0.29.x + minK8sVersion: 1.23 + maxK8sVersion: 1.27 + - appVersion: 0.30.x + minK8sVersion: 1.23 + maxK8sVersion: 1.27 \ No newline at end of file diff --git a/hack/docgen.sh b/hack/docgen.sh new file mode 100755 index 000000000000..a7d7ad9ae4a1 --- /dev/null +++ b/hack/docgen.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +compatibilitymatrix() { + supportedVersion=$(go run hack/code/version_compatibility.go "$(git describe --exact-match --tags || echo "no tag")") + go run hack/docs/compatibilitymetrix_gen_docs.go website/content/en/preview/upgrade-guide.md hack/compatibility-karpenter.yaml $supportedVersion +} + + +compatibilitymatrix +go run hack/docs/metrics_gen_docs.go pkg/ $(KARPENTER_CORE_DIR)/pkg website/content/en/preview/concepts/metrics.md +go run hack/docs/instancetypes_gen_docs.go website/content/en/preview/concepts/instance-types.md +go run hack/docs/configuration_gen_docs.go website/content/en/preview/concepts/settings.md +cd charts/karpenter && helm-docs \ No newline at end of file diff --git a/hack/docs/compatibilitymetrix_gen_docs.go b/hack/docs/compatibilitymetrix_gen_docs.go new file mode 100644 index 000000000000..453c67d04ab9 --- /dev/null +++ b/hack/docs/compatibilitymetrix_gen_docs.go @@ -0,0 +1,68 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "log" + "os" + "strconv" + "strings" + + "github.com/aws/karpenter/tools/kompat/pkg/kompat" +) + +func main() { + outputFileName := os.Args[1] + mdFile, err := os.ReadFile(outputFileName) + if err != nil { + log.Printf("Can't read %s file: %v", os.Args[1], err) + os.Exit(2) + } + + genStart := "[comment]: <> (the content below is generated from hack/docs/compataiblitymetrix_gen_docs.go)" + genEnd := "[comment]: <> (end docs generated content from hack/docs/compataiblitymetrix_gen_docs.go)" + startDocSections := strings.Split(string(mdFile), genStart) + if len(startDocSections) != 2 { + log.Fatalf("expected one generated comment block start but got %d", len(startDocSections)-1) + } + endDocSections := strings.Split(string(mdFile), genEnd) + if len(endDocSections) != 2 { + log.Fatalf("expected one generated comment block end but got %d", len(endDocSections)-1) + } + topDoc := fmt.Sprintf("%s%s\n\n", startDocSections[0], genStart) + bottomDoc := fmt.Sprintf("\n%s%s", genEnd, endDocSections[1]) + + baseText, err := kompat.Parse(os.Args[2]) + if err != nil { + log.Fatalf("unable to generate compatibility matrix") + } + var supportedVersions int + if len(os.Args) == 4 { + supportedVersions, err = strconv.Atoi(os.Args[3]) + if err != nil { + log.Fatalf("unable to get supported number of versons") + } + } else { + supportedVersions = 5 + } + + log.Println("writing output to", outputFileName) + f, err := os.Create(outputFileName) + if err != nil { + log.Fatalf("unable to open %s to write generated output: %v", outputFileName, err) + } + f.WriteString(topDoc + baseText.Markdown(kompat.Options{LastN: supportedVersions}) + bottomDoc) +} diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index bc3d94232b6a..401bb1d52da9 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -71,6 +71,7 @@ type Operator struct { AMIResolver *amifamily.Resolver LaunchTemplateProvider *launchtemplate.Provider PricingProvider *pricing.Provider + VersionProvider *version.Provider InstanceTypesProvider *instancetype.Provider InstanceProvider *instance.Provider } @@ -169,6 +170,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont SecurityGroupProvider: securityGroupProvider, AMIProvider: amiProvider, AMIResolver: amiResolver, + VersionProvider: versionProvider, LaunchTemplateProvider: launchTemplateProvider, PricingProvider: pricingProvider, InstanceTypesProvider: instanceTypeProvider, diff --git a/website/content/en/preview/upgrade-guide.md b/website/content/en/preview/upgrade-guide.md index 1843ac8b525b..be43698e536c 100644 --- a/website/content/en/preview/upgrade-guide.md +++ b/website/content/en/preview/upgrade-guide.md @@ -11,6 +11,24 @@ Use your existing upgrade mechanisms to upgrade your core add-ons in Kubernetes To make upgrading easier we aim to minimize introduction of breaking changes with the followings: +## Compatibility Matrix + +[comment]: <> (the content below is generated from hack/docs/compataiblitymetrix_gen_docs.go) + +| KUBERNETES | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 | +|------------|---------|---------|---------|---------|---------| +| karpenter | 0.21.x+ | 0.21.x+ | 0.25.x+ | 0.28.x+ | 0.28.x+ | + +[comment]: <> (end docs generated content from hack/docs/compataiblitymetrix_gen_docs.go) + +{{% alert title="Note" color="warning" %}} +Karpenter currently does not support the following [new `topologySpreadConstraints` keys](https://kubernetes.io/blog/2023/04/17/fine-grained-pod-topology-spread-features-beta/), promoted to beta in Kubernetes 1.27: +- `matchLabelKeys` +- `nodeAffinityPolicy` +- `nodeTaintsPolicy` + +For more information on Karpenter's support for these keys, view [this tracking issue](https://github.com/aws/karpenter-core/issues/430). +{{% /alert %}} ## Compatibility issues To make upgrading easier, we aim to minimize the introduction of breaking changes with the followings components: