Merge branch 'aws:main' into scale_test

aws · Mar 15, 2024 · aed5ffa · aed5ffa
2 parents 890bf73 + a4d93bd
commit aed5ffa
Show file tree

Hide file tree

Showing 11 changed files with 336 additions and 7 deletions.
diff --git a/charts/karpenter-crd/artifacthub-repo.yaml b/charts/karpenter-crd/artifacthub-repo.yaml
@@ -0,0 +1,7 @@
+repositoryID: fda7ffc4-4672-4218-8264-321ec3b4e3cc
+owners: []
+#   - name: awsadmin1
+#     email: [email protected]
+ignore:
+  - name: karpenter-crd
+    version: (?:^\d+$)|(?:^v?0\.0\.0)|(?:^v?\d+\-)
diff --git a/charts/karpenter/artifacthub-repo.yaml b/charts/karpenter/artifacthub-repo.yaml
@@ -0,0 +1,7 @@
+repositoryID: 356cb63f-9ee3-4956-9c20-003e416715c7
+owners: []
+#   - name: awsadmin1
+#     email: [email protected]
+ignore:
+  - name: karpenter
+    version: (?:^\d+$)|(?:^v?0\.0\.0)|(?:^v?\d+\-)
diff --git a/examples/v1beta1/minValues-family.yaml b/examples/v1beta1/minValues-family.yaml
@@ -0,0 +1,52 @@
+# This example will use spot instance type for all provisioned instances
+# and enforces minValues to instance families which means at least that number of unique instance
+# families is required by the scheduler for the NodeClaim creation.
+---
+apiVersion: karpenter.sh/v1beta1
+kind: NodePool
+metadata:
+  name: spot
+  annotations:
+    kubernetes.io/description: "NodePool for provisioning spot capacity"
+spec:
+  template:
+    spec:
+      requirements:
+        - key: karpenter.sh/capacity-type
+          operator: In
+          values: ["spot"]
+        - key: kubernetes.io/arch
+          operator: In
+          values: ["amd64"]
+        - key: kubernetes.io/os
+          operator: In
+          values: ["linux"]
+        - key: karpenter.k8s.aws/instance-category
+          operator: In
+          values: ["c", "m", "r"]
+        - key: karpenter.k8s.aws/instance-family
+          operator: Exists
+          minValues: 5
+        - key: karpenter.k8s.aws/instance-generation
+          operator: Gt
+          values: ["2"]
+      nodeClassRef:
+        apiVersion: karpenter.k8s.aws/v1beta1
+        kind: EC2NodeClass
+        name: default
+---
+apiVersion: karpenter.k8s.aws/v1beta1
+kind: EC2NodeClass
+metadata:
+  name: default
+  annotations:
+    kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes"
+spec:
+  amiFamily: AL2 # Amazon Linux 2
+  role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name
+  subnetSelectorTerms:
+    - tags:
+        karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name
+  securityGroupSelectorTerms:
+    - tags:
+        karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name
diff --git a/examples/v1beta1/minValues-multiple-keys.yaml b/examples/v1beta1/minValues-multiple-keys.yaml
@@ -0,0 +1,56 @@
+# This example will use spot instance type for all provisioned instances and enforces minValues to various keys where it is defined i.e 
+# at least 2 unique instance families from [c,m,r], 5 unique instance families [eg: "m5","m5d","m5dn","c5","c5d","c4" etc], 2 unique instance types [eg: "c5.2xlarge","c4.xlarge" etc] are required by the scheduler for the NodeClaim creation.
+# This ensures minimum flexiblity required to schedule pods into spot nodes.
+---
+apiVersion: karpenter.sh/v1beta1
+kind: NodePool
+metadata:
+  name: spot
+  annotations:
+    kubernetes.io/description: "NodePool for provisioning spot capacity"
+spec:
+  template:
+    spec:
+      requirements:
+        - key: karpenter.sh/capacity-type
+          operator: In
+          values: ["spot"]
+        - key: kubernetes.io/arch
+          operator: In
+          values: ["amd64"]
+        - key: kubernetes.io/os
+          operator: In
+          values: ["linux"]
+        - key: karpenter.k8s.aws/instance-category
+          operator: In
+          values: ["c", "m", "r"]
+          minValues: 2
+        - key: karpenter.k8s.aws/instance-family
+          operator: Exists
+          minValues: 5
+        - key: node.kubernetes.io/instance-type
+          operator: Exists
+          minValues: 2
+        - key: karpenter.k8s.aws/instance-generation
+          operator: Gt
+          values: ["2"]
+      nodeClassRef:
+        apiVersion: karpenter.k8s.aws/v1beta1
+        kind: EC2NodeClass
+        name: default
+---
+apiVersion: karpenter.k8s.aws/v1beta1
+kind: EC2NodeClass
+metadata:
+  name: default
+  annotations:
+    kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes"
+spec:
+  amiFamily: AL2 # Amazon Linux 2
+  role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name
+  subnetSelectorTerms:
+    - tags:
+        karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name
+  securityGroupSelectorTerms:
+    - tags:
+        karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name
diff --git a/go.mod b/go.mod
@@ -28,7 +28,7 @@ require (
 	k8s.io/utils v0.0.0-20240102154912-e7106e64919e
 	knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd
 	sigs.k8s.io/controller-runtime v0.17.2
-	sigs.k8s.io/karpenter v0.35.1-0.20240311230445-343eb7580913
+	sigs.k8s.io/karpenter v0.35.1-0.20240314163853-c6b8b811c3d0
 	sigs.k8s.io/yaml v1.4.0
 )
 

diff --git a/go.sum b/go.sum
@@ -757,8 +757,8 @@ sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeG
 sigs.k8s.io/controller-runtime v0.17.2/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
-sigs.k8s.io/karpenter v0.35.1-0.20240311230445-343eb7580913 h1:ldnc5SBfq/5lMXt1NSQMPr/ONvGwfoD6Cf/uCptFEGY=
-sigs.k8s.io/karpenter v0.35.1-0.20240311230445-343eb7580913/go.mod h1:DYnwDaoy2AhZwL2Ie96RGVu15+K6P5AJkSAtpPCRy8k=
+sigs.k8s.io/karpenter v0.35.1-0.20240314163853-c6b8b811c3d0 h1:IKGl5MC8nx4SO8z5nuLI1+CwOD+ej/AQowh5Iurj8uQ=
+sigs.k8s.io/karpenter v0.35.1-0.20240314163853-c6b8b811c3d0/go.mod h1:yb+1r5cysByx1xePT48lMHZU6EKlqOUNJiP1phbvFVc=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=

diff --git a/hack/release/common.sh b/hack/release/common.sh
@@ -91,13 +91,13 @@ publishHelmChart() {
   yq e -i ".version = \"${version}\"" "charts/${helm_chart}/Chart.yaml"
 
   cd charts
-  if [[ -s "${ah_config_file_name}" ]]; then
+  if [[ -s "${ah_config_file_name}" ]] && [[ "$oci_repo" == "${RELEASE_REPO_ECR}" ]]; then
     # ECR requires us to create an empty config file for an alternative
     # media type artifact push rather than /dev/null
     # https://github.com/aws/containers-roadmap/issues/1074
     temp=$(mktemp)
     echo {} > "${temp}"
-    oras push "${oci_repo}/${helm_chart}:artifacthub.io" --config "${temp}:application/vnd.cncf.artifacthub.config.v1+yaml" "${ah_config_file_name}:application/vnd.cncf.artifacthub.repository-metadata.layer.v1.yaml"
+    oras push "${oci_repo}${helm_chart}:artifacthub.io" --config "${temp}:application/vnd.cncf.artifacthub.config.v1+yaml" "${ah_config_file_name}:application/vnd.cncf.artifacthub.repository-metadata.layer.v1.yaml"
   fi
   helm dependency update "${helm_chart}"
   helm lint "${helm_chart}"

diff --git a/tools/kompat/pkg/kompat/kompat.go b/tools/kompat/pkg/kompat/kompat.go
@@ -195,7 +195,7 @@ func (k Kompat) Markdown(_ ...Options) string {
 		if c.MaxK8sVersion == "" || c.MinK8sVersion == c.MaxK8sVersion {
 			headers = append(headers, fmt.Sprintf("\\>= `%s`", c.MinK8sVersion))
 		} else {
-			headers = append(headers, fmt.Sprintf("`%s` - `%s`", c.MinK8sVersion, c.MaxK8sVersion))
+			headers = append(headers, fmt.Sprintf("\\>= `%s` \\<= `%s`", c.MinK8sVersion, c.MaxK8sVersion))
 		}
 		data = append(data, c.AppVersion)
 	}
@@ -368,7 +368,7 @@ func semverRange(semvers []string, allSemvers ...string) string {
 			return fmt.Sprintf("\\>= %s", strings.ReplaceAll(semvers[0], ".x", ""))
 		}
 	}
-	return fmt.Sprintf("%s - %s", semvers[0], semvers[len(semvers)-1])
+	return fmt.Sprintf("\\>= %s \\<= %s", strings.ReplaceAll(semvers[0], ".x", ""), strings.ReplaceAll(semvers[len(semvers)-1], ".x", ""))
 }
 
 func sortSemvers(semvers []string) {

diff --git a/website/content/en/docs/concepts/nodepools.md b/website/content/en/docs/concepts/nodepools.md
@@ -71,6 +71,12 @@ spec:
         - key: "karpenter.k8s.aws/instance-category"
           operator: In
           values: ["c", "m", "r"]
+          # minValues here enforces the scheduler to consider at least that number of unique instance-category to schedule the pods.
+          minValues: 2
+        - key: "karpenter.k8s.aws/instance-family"
+          operator: In
+          values: ["m5","m5d","c5","c5d","c4","r4"]
+          minValues: 5
         - key: "karpenter.k8s.aws/instance-cpu"
           operator: In
           values: ["4", "8", "16", "32"]
@@ -224,6 +230,69 @@ Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand i
 
 Karpenter also allows `karpenter.sh/capacity-type` to be used as a topology key for enforcing topology-spread.
 
+Along with the combination of [key,operator,values] in the requirements, Karpenter also supports `minValues` in the NodePool requirements block, allowing the scheduler to be aware of user-specified flexibility minimums while scheduling pods to a cluster. If Karpenter cannot meet this minimum flexibility for each key when scheduling a pod, it will fail the scheduling loop for that NodePool, either falling back to another NodePool which meets the pod requirements or failing scheduling the pod altogether.
+
+For example, the below spec will use spot instance type for all provisioned instances and enforces `minValues` to various keys where it is defined
+i.e at least 2 unique instance families from [c,m,r], 5 unique instance families [eg: "m5","m5d","r4","c5","c5d","c4" etc], 10 unique instance types [eg: "c5.2xlarge","c4.xlarge" etc] is required for scheduling the pods.
+
+```yaml
+spec:
+  template:
+    spec:
+      requirements:
+        - key: kubernetes.io/arch
+          operator: In
+          values: ["amd64"]
+        - key: kubernetes.io/os
+          operator: In
+          values: ["linux"]
+        - key: karpenter.k8s.aws/instance-category
+          operator: In
+          values: ["c", "m", "r"]
+          minValues: 2
+        - key: karpenter.k8s.aws/instance-family
+          operator: Exists
+          minValues: 5
+        - key: node.kubernetes.io/instance-type
+          operator: Exists
+          minValues: 10
+        - key: karpenter.k8s.aws/instance-generation
+          operator: Gt
+          values: ["2"]
+```
+
+Note that `minValues` can be used with multiple operators and multiple requirements. And if the `minValues` are defined with multiple operators for the same requirement key, scheduler considers the max of all the `minValues` for that requirement. For example, the below spec requires scheduler to consider at least 5 instance-family to schedule the pods.
+
+```yaml
+spec:
+  template:
+    spec:
+      requirements:
+        - key: kubernetes.io/arch
+          operator: In
+          values: ["amd64"]
+        - key: kubernetes.io/os
+          operator: In
+          values: ["linux"]
+        - key: karpenter.k8s.aws/instance-category
+          operator: In
+          values: ["c", "m", "r"]
+          minValues: 2
+        - key: karpenter.k8s.aws/instance-family
+          operator: Exists
+          minValues: 5
+        - key: karpenter.k8s.aws/instance-family
+          operator: In
+          values: ["m5","m5d","c5","c5d","c4","r4"]
+          minValues: 3
+        - key: node.kubernetes.io/instance-type
+          operator: Exists
+          minValues: 10
+        - key: karpenter.k8s.aws/instance-generation
+          operator: Gt
+          values: ["2"]
+```
+
 {{% alert title="Recommended" color="primary" %}}
 Karpenter allows you to be extremely flexible with your NodePools by only constraining your instance types in ways that are absolutely necessary for your cluster. By default, Karpenter will enforce that you specify the `spec.template.spec.requirements` field, but will not enforce that you specify any requirements within the field. If you choose to specify `requirements: []`, this means that you will completely flexible to _all_ instance types that your cloud provider supports.
 

diff --git a/website/content/en/preview/concepts/nodepools.md b/website/content/en/preview/concepts/nodepools.md
@@ -71,6 +71,12 @@ spec:
         - key: "karpenter.k8s.aws/instance-category"
           operator: In
           values: ["c", "m", "r"]
+          # minValues here enforces the scheduler to consider at least that number of unique instance-category to schedule the pods.
+          minValues: 2
+        - key: "karpenter.k8s.aws/instance-family"
+          operator: In
+          values: ["m5","m5d","c5","c5d","c4","r4"]
+          minValues: 5
         - key: "karpenter.k8s.aws/instance-cpu"
           operator: In
           values: ["4", "8", "16", "32"]
@@ -224,6 +230,69 @@ Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand i
 
 Karpenter also allows `karpenter.sh/capacity-type` to be used as a topology key for enforcing topology-spread.
 
+Along with the combination of [key,operator,values] in the requirements, Karpenter also supports `minValues` in the NodePool requirements block, allowing the scheduler to be aware of user-specified flexibility minimums while scheduling pods to a cluster. If Karpenter cannot meet this minimum flexibility for each key when scheduling a pod, it will fail the scheduling loop for that NodePool, either falling back to another NodePool which meets the pod requirements or failing scheduling the pod altogether.
+
+For example, the below spec will use spot instance type for all provisioned instances and enforces `minValues` to various keys where it is defined
+i.e at least 2 unique instance families from [c,m,r], 5 unique instance families [eg: "m5","m5d","r4","c5","c5d","c4" etc], 10 unique instance types [eg: "c5.2xlarge","c4.xlarge" etc] is required for scheduling the pods.
+
+```yaml
+spec:
+  template:
+    spec:
+      requirements:
+        - key: kubernetes.io/arch
+          operator: In
+          values: ["amd64"]
+        - key: kubernetes.io/os
+          operator: In
+          values: ["linux"]
+        - key: karpenter.k8s.aws/instance-category
+          operator: In
+          values: ["c", "m", "r"]
+          minValues: 2
+        - key: karpenter.k8s.aws/instance-family
+          operator: Exists
+          minValues: 5
+        - key: node.kubernetes.io/instance-type
+          operator: Exists
+          minValues: 10
+        - key: karpenter.k8s.aws/instance-generation
+          operator: Gt
+          values: ["2"]
+```
+
+Note that `minValues` can be used with multiple operators and multiple requirements. And if the `minValues` are defined with multiple operators for the same requirement key, scheduler considers the max of all the `minValues` for that requirement. For example, the below spec requires scheduler to consider at least 5 instance-family to schedule the pods.
+
+```yaml
+spec:
+  template:
+    spec:
+      requirements:
+        - key: kubernetes.io/arch
+          operator: In
+          values: ["amd64"]
+        - key: kubernetes.io/os
+          operator: In
+          values: ["linux"]
+        - key: karpenter.k8s.aws/instance-category
+          operator: In
+          values: ["c", "m", "r"]
+          minValues: 2
+        - key: karpenter.k8s.aws/instance-family
+          operator: Exists
+          minValues: 5
+        - key: karpenter.k8s.aws/instance-family
+          operator: In
+          values: ["m5","m5d","c5","c5d","c4","r4"]
+          minValues: 3
+        - key: node.kubernetes.io/instance-type
+          operator: Exists
+          minValues: 10
+        - key: karpenter.k8s.aws/instance-generation
+          operator: Gt
+          values: ["2"]
+```
+
 {{% alert title="Recommended" color="primary" %}}
 Karpenter allows you to be extremely flexible with your NodePools by only constraining your instance types in ways that are absolutely necessary for your cluster. By default, Karpenter will enforce that you specify the `spec.template.spec.requirements` field, but will not enforce that you specify any requirements within the field. If you choose to specify `requirements: []`, this means that you will completely flexible to _all_ instance types that your cloud provider supports.