Merge pull request #3964 from james-masson/enable_k8s_reserved_cpus

Enable k8s reserved cpus
bottlerocket-os · Jun 13, 2024 · 2d4790f · 2d4790f
2 parents 618ec7a + 7cb2390
commit 2d4790f
Show file tree

Hide file tree

Showing 16 changed files with 179 additions and 7 deletions.
diff --git a/Release.toml b/Release.toml
@@ -319,4 +319,5 @@ version = "1.21.0"
     "migrate_v1.21.0_pod-infra-container-image-remove-settings-generator.lz4",
     "migrate_v1.21.0_pod-infra-container-image-affected-services.lz4",
     "migrate_v1.21.0_pod-infra-container-image-services.lz4",
+    "migrate_v1.21.0_k8s-reserved-cpus-v0-1-0.lz4",
 ]
diff --git a/packages/kubernetes-1.23/kubelet-config b/packages/kubernetes-1.23/kubelet-config
@@ -87,13 +87,17 @@ kubeReserved:
   {{/if}}
   {{/if}}
   ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
+{{#unless settings.kubernetes.reserved-cpus}}
 kubeReservedCgroup: "/runtime"
+{{/unless}}
 {{#if settings.kubernetes.system-reserved}}
 systemReserved:
   {{#each settings.kubernetes.system-reserved}}
   {{@key}}: "{{this}}"
   {{/each}}
+{{#unless settings.kubernetes.reserved-cpus}}
 systemReservedCgroup: "/system"
+{{/unless}}
 {{/if}}
 cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}}
 cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
@@ -182,3 +186,6 @@ reservedMemory:
 {{/each}}
 {{/if}}
 {{/if}}
+{{#if settings.kubernetes.reserved-cpus}}
+reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}}
+{{/if}}
diff --git a/packages/kubernetes-1.24/kubelet-config b/packages/kubernetes-1.24/kubelet-config
@@ -87,13 +87,17 @@ kubeReserved:
   {{/if}}
   {{/if}}
   ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
+{{#unless settings.kubernetes.reserved-cpus}}
 kubeReservedCgroup: "/runtime"
+{{/unless}}
 {{#if settings.kubernetes.system-reserved}}
 systemReserved:
   {{#each settings.kubernetes.system-reserved}}
   {{@key}}: "{{this}}"
   {{/each}}
+{{#unless settings.kubernetes.reserved-cpus}}
 systemReservedCgroup: "/system"
+{{/unless}}
 {{/if}}
 cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}}
 cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
@@ -181,3 +185,6 @@ reservedMemory:
 {{/each}}
 {{/if}}
 {{/if}}
+{{#if settings.kubernetes.reserved-cpus}}
+reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}}
+{{/if}}
diff --git a/packages/kubernetes-1.25/kubelet-config b/packages/kubernetes-1.25/kubelet-config
@@ -87,13 +87,17 @@ kubeReserved:
   {{/if}}
   {{/if}}
   ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
+{{#unless settings.kubernetes.reserved-cpus}}
 kubeReservedCgroup: "/runtime"
+{{/unless}}
 {{#if settings.kubernetes.system-reserved}}
 systemReserved:
   {{#each settings.kubernetes.system-reserved}}
   {{@key}}: "{{this}}"
   {{/each}}
+{{#unless settings.kubernetes.reserved-cpus}}
 systemReservedCgroup: "/system"
+{{/unless}}
 {{/if}}
 cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}}
 cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
@@ -182,3 +186,6 @@ reservedMemory:
 {{/each}}
 {{/if}}
 {{/if}}
+{{#if settings.kubernetes.reserved-cpus}}
+reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}}
+{{/if}}
diff --git a/packages/kubernetes-1.26/kubelet-config b/packages/kubernetes-1.26/kubelet-config
@@ -87,13 +87,17 @@ kubeReserved:
   {{/if}}
   {{/if}}
   ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
+{{#unless settings.kubernetes.reserved-cpus}}
 kubeReservedCgroup: "/runtime"
+{{/unless}}
 {{#if settings.kubernetes.system-reserved}}
 systemReserved:
   {{#each settings.kubernetes.system-reserved}}
   {{@key}}: "{{this}}"
   {{/each}}
+{{#unless settings.kubernetes.reserved-cpus}}
 systemReservedCgroup: "/system"
+{{/unless}}
 {{/if}}
 cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}}
 cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
@@ -182,3 +186,6 @@ reservedMemory:
 {{/each}}
 {{/if}}
 {{/if}}
+{{#if settings.kubernetes.reserved-cpus}}
+reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}}
+{{/if}}
diff --git a/packages/kubernetes-1.27/kubelet-config b/packages/kubernetes-1.27/kubelet-config
@@ -91,13 +91,17 @@ kubeReserved:
   {{/if}}
   {{/if}}
   ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
+{{#unless settings.kubernetes.reserved-cpus}}
 kubeReservedCgroup: "/runtime"
+{{/unless}}
 {{#if settings.kubernetes.system-reserved}}
 systemReserved:
   {{#each settings.kubernetes.system-reserved}}
   {{@key}}: "{{this}}"
   {{/each}}
+{{#unless settings.kubernetes.reserved-cpus}}
 systemReservedCgroup: "/system"
+{{/unless}}
 {{/if}}
 cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}}
 cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
@@ -183,3 +187,6 @@ reservedMemory:
 {{/each}}
 {{/if}}
 {{/if}}
+{{#if settings.kubernetes.reserved-cpus}}
+reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}}
+{{/if}}
diff --git a/packages/kubernetes-1.28/kubelet-config b/packages/kubernetes-1.28/kubelet-config
@@ -91,13 +91,17 @@ kubeReserved:
   {{/if}}
   {{/if}}
   ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
+{{#unless settings.kubernetes.reserved-cpus}}
 kubeReservedCgroup: "/runtime"
+{{/unless}}
 {{#if settings.kubernetes.system-reserved}}
 systemReserved:
   {{#each settings.kubernetes.system-reserved}}
   {{@key}}: "{{this}}"
   {{/each}}
+{{#unless settings.kubernetes.reserved-cpus}}
 systemReservedCgroup: "/system"
+{{/unless}}
 {{/if}}
 cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}}
 cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
@@ -181,3 +185,6 @@ reservedMemory:
 {{/each}}
 {{/if}}
 {{/if}}
+{{#if settings.kubernetes.reserved-cpus}}
+reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}}
+{{/if}}
diff --git a/packages/kubernetes-1.29/kubelet-config b/packages/kubernetes-1.29/kubelet-config
@@ -91,13 +91,17 @@ kubeReserved:
   {{/if}}
   {{/if}}
   ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
+{{#unless settings.kubernetes.reserved-cpus}}
 kubeReservedCgroup: "/runtime"
+{{/unless}}
 {{#if settings.kubernetes.system-reserved}}
 systemReserved:
   {{#each settings.kubernetes.system-reserved}}
   {{@key}}: "{{this}}"
   {{/each}}
+{{#unless settings.kubernetes.reserved-cpus}}
 systemReservedCgroup: "/system"
+{{/unless}}
 {{/if}}
 cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}}
 cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
@@ -181,3 +185,6 @@ reservedMemory:
 {{/each}}
 {{/if}}
 {{/if}}
+{{#if settings.kubernetes.reserved-cpus}}
+reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}}
+{{/if}}
diff --git a/packages/kubernetes-1.30/kubelet-config b/packages/kubernetes-1.30/kubelet-config
@@ -91,13 +91,17 @@ kubeReserved:
   {{/if}}
   {{/if}}
   ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
+{{#unless settings.kubernetes.reserved-cpus}}
 kubeReservedCgroup: "/runtime"
+{{/unless}}
 {{#if settings.kubernetes.system-reserved}}
 systemReserved:
   {{#each settings.kubernetes.system-reserved}}
   {{@key}}: "{{this}}"
   {{/each}}
+{{#unless settings.kubernetes.reserved-cpus}}
 systemReservedCgroup: "/system"
+{{/unless}}
 {{/if}}
 cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}}
 cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
@@ -181,3 +185,6 @@ reservedMemory:
 {{/each}}
 {{/if}}
 {{/if}}
+{{#if settings.kubernetes.reserved-cpus}}
+reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}}
+{{/if}}
diff --git a/sources/Cargo.lock b/sources/Cargo.lock
diff --git a/sources/Cargo.toml b/sources/Cargo.toml
@@ -28,6 +28,7 @@ members = [
     "api/migration/migrations/v1.21.0/pod-infra-container-image-affected-services",
     "api/migration/migrations/v1.21.0/pod-infra-container-image-remove-settings-generator",
     "api/migration/migrations/v1.21.0/pod-infra-container-image-services",
+    "api/migration/migrations/v1.21.0/k8s-reserved-cpus-v0-1-0",
 
     "bloodhound",
 

diff --git a/sources/api/migration/migrations/v1.21.0/k8s-reserved-cpus-v0-1-0/Cargo.toml b/sources/api/migration/migrations/v1.21.0/k8s-reserved-cpus-v0-1-0/Cargo.toml
@@ -0,0 +1,10 @@
+[package]
+name = "k8s-reserved-cpus-v0-1-0"
+version = "0.1.0"
+authors = ["James Masson <[email protected]>"]
+license = "Apache-2.0 OR MIT"
+edition = "2021"
+publish = false
+
+[dependencies]
+migration-helpers = { path = "../../../migration-helpers", version = "0.1.0" }
diff --git a/sources/api/migration/migrations/v1.21.0/k8s-reserved-cpus-v0-1-0/src/main.rs b/sources/api/migration/migrations/v1.21.0/k8s-reserved-cpus-v0-1-0/src/main.rs
@@ -0,0 +1,18 @@
+use migration_helpers::common_migrations::AddSettingsMigration;
+use migration_helpers::{migrate, Result};
+use std::process;
+
+/// Add the option to set Kubernetes reserved-cpus
+fn run() -> Result<()> {
+    migrate(AddSettingsMigration(&["settings.kubernetes.reserved-cpus"]))
+}
+
+// Returning a Result from main makes it print a Debug representation of the error, but with Snafu
+// we have nice Display representations of the error, so we wrap "main" (run) and print any error.
+// https://github.com/shepmaster/snafu/issues/110
+fn main() {
+    if let Err(e) = run() {
+        eprintln!("{}", e);
+        process::exit(1);
+    }
+}
diff --git a/sources/models/modeled-types/src/lib.rs b/sources/models/modeled-types/src/lib.rs
@@ -150,6 +150,9 @@ pub mod error {
             input: String,
             source: std::num::ParseIntError,
         },
+
+        #[snafu(display("Invalid Kernel CpuSet value '{}'", input))]
+        InvalidKernelCpuSetValue { input: String },
     }
 
     /// Creates a `ValidationError` with a consistent message for strings with regex validations

diff --git a/sources/models/modeled-types/src/shared.rs b/sources/models/modeled-types/src/shared.rs
@@ -1178,3 +1178,78 @@ mod test_positive_integer {
         assert!(NonNegativeInteger::try_from(-1).is_err());
     }
 }
+
+// =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=
+
+/// KernelCpuSetValue represents a string that contains a valid Kernel CpuSet Value from
+/// here: https://man7.org/linux/man-pages/man7/cpuset.7.html#FORMATS. This matches the
+/// logic from https://github.com/kubernetes/utils/blob/d93618cff8a22d3aea7bf78d9d528fd859720c2d/cpuset/cpuset.go#L203
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+pub struct KernelCpuSetValue {
+    inner: String,
+}
+
+lazy_static! {
+    pub(crate) static ref KERNAL_CPU_SET_VALUE: Regex =
+        Regex::new(r"^([0-9]+(-[0-9]+)?,?)*([0-9]+(-[0-9]+)?)+$").unwrap();
+}
+
+impl TryFrom<&str> for KernelCpuSetValue {
+    type Error = error::Error;
+
+    fn try_from(input: &str) -> Result<Self, Self::Error> {
+        ensure!(
+            !input.is_empty(),
+            error::InvalidKernelCpuSetValueSnafu { input }
+        );
+        ensure!(
+            KERNAL_CPU_SET_VALUE.is_match(input),
+            error::InvalidKernelCpuSetValueSnafu { input }
+        );
+        Ok(KernelCpuSetValue {
+            inner: input.to_string(),
+        })
+    }
+}
+
+string_impls_for!(KernelCpuSetValue, "KernelCpuSetValue");
+
+#[cfg(test)]
+mod test_kernel_cpu_set_value {
+    use super::KernelCpuSetValue;
+    use std::convert::TryFrom;
+
+    #[test]
+    fn good_tokens() {
+        for ok in &[
+            "1",
+            "0,1,2,3",
+            "1-4",
+            "1,6-9",
+            "1-4,6",
+            "1-3,6-9,10-15",
+            "100,101",
+            "1,2,3,4,5,6,7,8,9,10",
+        ] {
+            KernelCpuSetValue::try_from(*ok).unwrap();
+        }
+    }
+
+    #[test]
+    fn bad_names() {
+        for err in &[
+            "",
+            "100.0",
+            "0...4",
+            "1..5",
+            "ten",
+            "1-",
+            "-3",
+            "9,-10",
+            "1,2,3,4,5,6,7,8,9,10,",
+            &"a".repeat(23),
+        ] {
+            KernelCpuSetValue::try_from(*err).unwrap_err();
+        }
+    }
+}
diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs
@@ -48,13 +48,13 @@ use std::net::IpAddr;
 use crate::de::{deserialize_limit, deserialize_mirrors, deserialize_node_taints};
 use modeled_types::{
     BootConfigKey, BootConfigValue, BootstrapContainerMode, CpuManagerPolicy, CredentialProvider,
-    DNSDomain, EtcHostsEntries, Identifier, IntegerPercent, KubernetesAuthenticationMode,
-    KubernetesBootstrapToken, KubernetesCloudProvider, KubernetesClusterDnsIp,
-    KubernetesClusterName, KubernetesDurationValue, KubernetesLabelKey, KubernetesLabelValue,
-    KubernetesQuantityValue, KubernetesReservedResourceKey, KubernetesTaintValue,
-    KubernetesThresholdValue, OciDefaultsCapability, OciDefaultsResourceLimitType,
-    PemCertificateString, SingleLineString, TopologyManagerPolicy, TopologyManagerScope, Url,
-    ValidBase64, ValidLinuxHostname,
+    DNSDomain, EtcHostsEntries, Identifier, IntegerPercent, KernelCpuSetValue,
+    KubernetesAuthenticationMode, KubernetesBootstrapToken, KubernetesCloudProvider,
+    KubernetesClusterDnsIp, KubernetesClusterName, KubernetesDurationValue, KubernetesLabelKey,
+    KubernetesLabelValue, KubernetesQuantityValue, KubernetesReservedResourceKey,
+    KubernetesTaintValue, KubernetesThresholdValue, OciDefaultsCapability,
+    OciDefaultsResourceLimitType, PemCertificateString, SingleLineString, TopologyManagerPolicy,
+    TopologyManagerScope, Url, ValidBase64, ValidLinuxHostname,
 };
 
 #[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
@@ -137,6 +137,7 @@ struct KubernetesSettings {
     shutdown_grace_period_for_critical_pods: KubernetesDurationValue,
     memory_manager_reserved_memory: HashMap<Identifier, KubernetesMemoryReservation>,
     memory_manager_policy: KubernetesMemoryManagerPolicy,
+    reserved_cpus: KernelCpuSetValue,
 
     // Settings where we generate a value based on the runtime environment.  The user can specify a
     // value to override the generated one, but typically would not.