Skip to content

Commit

Permalink
Merge pull request bottlerocket-os#3964 from james-masson/enable_k8s_…
Browse files Browse the repository at this point in the history
…reserved_cpus

Enable k8s reserved cpus
  • Loading branch information
yeazelm authored Jun 13, 2024
2 parents 618ec7a + 7cb2390 commit 2d4790f
Show file tree
Hide file tree
Showing 16 changed files with 179 additions and 7 deletions.
1 change: 1 addition & 0 deletions Release.toml
Original file line number Diff line number Diff line change
Expand Up @@ -319,4 +319,5 @@ version = "1.21.0"
"migrate_v1.21.0_pod-infra-container-image-remove-settings-generator.lz4",
"migrate_v1.21.0_pod-infra-container-image-affected-services.lz4",
"migrate_v1.21.0_pod-infra-container-image-services.lz4",
"migrate_v1.21.0_k8s-reserved-cpus-v0-1-0.lz4",
]
7 changes: 7 additions & 0 deletions packages/kubernetes-1.23/kubelet-config
Original file line number Diff line number Diff line change
Expand Up @@ -87,13 +87,17 @@ kubeReserved:
{{/if}}
{{/if}}
ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
{{#unless settings.kubernetes.reserved-cpus}}
kubeReservedCgroup: "/runtime"
{{/unless}}
{{#if settings.kubernetes.system-reserved}}
systemReserved:
{{#each settings.kubernetes.system-reserved}}
{{@key}}: "{{this}}"
{{/each}}
{{#unless settings.kubernetes.reserved-cpus}}
systemReservedCgroup: "/system"
{{/unless}}
{{/if}}
cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}}
cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
Expand Down Expand Up @@ -182,3 +186,6 @@ reservedMemory:
{{/each}}
{{/if}}
{{/if}}
{{#if settings.kubernetes.reserved-cpus}}
reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}}
{{/if}}
7 changes: 7 additions & 0 deletions packages/kubernetes-1.24/kubelet-config
Original file line number Diff line number Diff line change
Expand Up @@ -87,13 +87,17 @@ kubeReserved:
{{/if}}
{{/if}}
ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
{{#unless settings.kubernetes.reserved-cpus}}
kubeReservedCgroup: "/runtime"
{{/unless}}
{{#if settings.kubernetes.system-reserved}}
systemReserved:
{{#each settings.kubernetes.system-reserved}}
{{@key}}: "{{this}}"
{{/each}}
{{#unless settings.kubernetes.reserved-cpus}}
systemReservedCgroup: "/system"
{{/unless}}
{{/if}}
cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}}
cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
Expand Down Expand Up @@ -181,3 +185,6 @@ reservedMemory:
{{/each}}
{{/if}}
{{/if}}
{{#if settings.kubernetes.reserved-cpus}}
reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}}
{{/if}}
7 changes: 7 additions & 0 deletions packages/kubernetes-1.25/kubelet-config
Original file line number Diff line number Diff line change
Expand Up @@ -87,13 +87,17 @@ kubeReserved:
{{/if}}
{{/if}}
ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
{{#unless settings.kubernetes.reserved-cpus}}
kubeReservedCgroup: "/runtime"
{{/unless}}
{{#if settings.kubernetes.system-reserved}}
systemReserved:
{{#each settings.kubernetes.system-reserved}}
{{@key}}: "{{this}}"
{{/each}}
{{#unless settings.kubernetes.reserved-cpus}}
systemReservedCgroup: "/system"
{{/unless}}
{{/if}}
cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}}
cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
Expand Down Expand Up @@ -182,3 +186,6 @@ reservedMemory:
{{/each}}
{{/if}}
{{/if}}
{{#if settings.kubernetes.reserved-cpus}}
reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}}
{{/if}}
7 changes: 7 additions & 0 deletions packages/kubernetes-1.26/kubelet-config
Original file line number Diff line number Diff line change
Expand Up @@ -87,13 +87,17 @@ kubeReserved:
{{/if}}
{{/if}}
ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
{{#unless settings.kubernetes.reserved-cpus}}
kubeReservedCgroup: "/runtime"
{{/unless}}
{{#if settings.kubernetes.system-reserved}}
systemReserved:
{{#each settings.kubernetes.system-reserved}}
{{@key}}: "{{this}}"
{{/each}}
{{#unless settings.kubernetes.reserved-cpus}}
systemReservedCgroup: "/system"
{{/unless}}
{{/if}}
cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}}
cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
Expand Down Expand Up @@ -182,3 +186,6 @@ reservedMemory:
{{/each}}
{{/if}}
{{/if}}
{{#if settings.kubernetes.reserved-cpus}}
reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}}
{{/if}}
7 changes: 7 additions & 0 deletions packages/kubernetes-1.27/kubelet-config
Original file line number Diff line number Diff line change
Expand Up @@ -91,13 +91,17 @@ kubeReserved:
{{/if}}
{{/if}}
ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
{{#unless settings.kubernetes.reserved-cpus}}
kubeReservedCgroup: "/runtime"
{{/unless}}
{{#if settings.kubernetes.system-reserved}}
systemReserved:
{{#each settings.kubernetes.system-reserved}}
{{@key}}: "{{this}}"
{{/each}}
{{#unless settings.kubernetes.reserved-cpus}}
systemReservedCgroup: "/system"
{{/unless}}
{{/if}}
cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}}
cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
Expand Down Expand Up @@ -183,3 +187,6 @@ reservedMemory:
{{/each}}
{{/if}}
{{/if}}
{{#if settings.kubernetes.reserved-cpus}}
reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}}
{{/if}}
7 changes: 7 additions & 0 deletions packages/kubernetes-1.28/kubelet-config
Original file line number Diff line number Diff line change
Expand Up @@ -91,13 +91,17 @@ kubeReserved:
{{/if}}
{{/if}}
ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
{{#unless settings.kubernetes.reserved-cpus}}
kubeReservedCgroup: "/runtime"
{{/unless}}
{{#if settings.kubernetes.system-reserved}}
systemReserved:
{{#each settings.kubernetes.system-reserved}}
{{@key}}: "{{this}}"
{{/each}}
{{#unless settings.kubernetes.reserved-cpus}}
systemReservedCgroup: "/system"
{{/unless}}
{{/if}}
cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}}
cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
Expand Down Expand Up @@ -181,3 +185,6 @@ reservedMemory:
{{/each}}
{{/if}}
{{/if}}
{{#if settings.kubernetes.reserved-cpus}}
reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}}
{{/if}}
7 changes: 7 additions & 0 deletions packages/kubernetes-1.29/kubelet-config
Original file line number Diff line number Diff line change
Expand Up @@ -91,13 +91,17 @@ kubeReserved:
{{/if}}
{{/if}}
ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
{{#unless settings.kubernetes.reserved-cpus}}
kubeReservedCgroup: "/runtime"
{{/unless}}
{{#if settings.kubernetes.system-reserved}}
systemReserved:
{{#each settings.kubernetes.system-reserved}}
{{@key}}: "{{this}}"
{{/each}}
{{#unless settings.kubernetes.reserved-cpus}}
systemReservedCgroup: "/system"
{{/unless}}
{{/if}}
cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}}
cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
Expand Down Expand Up @@ -181,3 +185,6 @@ reservedMemory:
{{/each}}
{{/if}}
{{/if}}
{{#if settings.kubernetes.reserved-cpus}}
reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}}
{{/if}}
7 changes: 7 additions & 0 deletions packages/kubernetes-1.30/kubelet-config
Original file line number Diff line number Diff line change
Expand Up @@ -91,13 +91,17 @@ kubeReserved:
{{/if}}
{{/if}}
ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}"
{{#unless settings.kubernetes.reserved-cpus}}
kubeReservedCgroup: "/runtime"
{{/unless}}
{{#if settings.kubernetes.system-reserved}}
systemReserved:
{{#each settings.kubernetes.system-reserved}}
{{@key}}: "{{this}}"
{{/each}}
{{#unless settings.kubernetes.reserved-cpus}}
systemReservedCgroup: "/system"
{{/unless}}
{{/if}}
cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}}
cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
Expand Down Expand Up @@ -181,3 +185,6 @@ reservedMemory:
{{/each}}
{{/if}}
{{/if}}
{{#if settings.kubernetes.reserved-cpus}}
reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}}
{{/if}}
7 changes: 7 additions & 0 deletions sources/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions sources/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ members = [
"api/migration/migrations/v1.21.0/pod-infra-container-image-affected-services",
"api/migration/migrations/v1.21.0/pod-infra-container-image-remove-settings-generator",
"api/migration/migrations/v1.21.0/pod-infra-container-image-services",
"api/migration/migrations/v1.21.0/k8s-reserved-cpus-v0-1-0",

"bloodhound",

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
[package]
name = "k8s-reserved-cpus-v0-1-0"
version = "0.1.0"
authors = ["James Masson <[email protected]>"]
license = "Apache-2.0 OR MIT"
edition = "2021"
publish = false

[dependencies]
migration-helpers = { path = "../../../migration-helpers", version = "0.1.0" }
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
use migration_helpers::common_migrations::AddSettingsMigration;
use migration_helpers::{migrate, Result};
use std::process;

/// Add the option to set Kubernetes reserved-cpus
fn run() -> Result<()> {
migrate(AddSettingsMigration(&["settings.kubernetes.reserved-cpus"]))
}

// Returning a Result from main makes it print a Debug representation of the error, but with Snafu
// we have nice Display representations of the error, so we wrap "main" (run) and print any error.
// https://github.com/shepmaster/snafu/issues/110
fn main() {
if let Err(e) = run() {
eprintln!("{}", e);
process::exit(1);
}
}
3 changes: 3 additions & 0 deletions sources/models/modeled-types/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,9 @@ pub mod error {
input: String,
source: std::num::ParseIntError,
},

#[snafu(display("Invalid Kernel CpuSet value '{}'", input))]
InvalidKernelCpuSetValue { input: String },
}

/// Creates a `ValidationError` with a consistent message for strings with regex validations
Expand Down
75 changes: 75 additions & 0 deletions sources/models/modeled-types/src/shared.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1178,3 +1178,78 @@ mod test_positive_integer {
assert!(NonNegativeInteger::try_from(-1).is_err());
}
}

// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=

/// KernelCpuSetValue represents a string that contains a valid Kernel CpuSet Value from
/// here: https://man7.org/linux/man-pages/man7/cpuset.7.html#FORMATS. This matches the
/// logic from https://github.com/kubernetes/utils/blob/d93618cff8a22d3aea7bf78d9d528fd859720c2d/cpuset/cpuset.go#L203
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub struct KernelCpuSetValue {
inner: String,
}

lazy_static! {
pub(crate) static ref KERNAL_CPU_SET_VALUE: Regex =
Regex::new(r"^([0-9]+(-[0-9]+)?,?)*([0-9]+(-[0-9]+)?)+$").unwrap();
}

impl TryFrom<&str> for KernelCpuSetValue {
type Error = error::Error;

fn try_from(input: &str) -> Result<Self, Self::Error> {
ensure!(
!input.is_empty(),
error::InvalidKernelCpuSetValueSnafu { input }
);
ensure!(
KERNAL_CPU_SET_VALUE.is_match(input),
error::InvalidKernelCpuSetValueSnafu { input }
);
Ok(KernelCpuSetValue {
inner: input.to_string(),
})
}
}

string_impls_for!(KernelCpuSetValue, "KernelCpuSetValue");

#[cfg(test)]
mod test_kernel_cpu_set_value {
use super::KernelCpuSetValue;
use std::convert::TryFrom;

#[test]
fn good_tokens() {
for ok in &[
"1",
"0,1,2,3",
"1-4",
"1,6-9",
"1-4,6",
"1-3,6-9,10-15",
"100,101",
"1,2,3,4,5,6,7,8,9,10",
] {
KernelCpuSetValue::try_from(*ok).unwrap();
}
}

#[test]
fn bad_names() {
for err in &[
"",
"100.0",
"0...4",
"1..5",
"ten",
"1-",
"-3",
"9,-10",
"1,2,3,4,5,6,7,8,9,10,",
&"a".repeat(23),
] {
KernelCpuSetValue::try_from(*err).unwrap_err();
}
}
}
15 changes: 8 additions & 7 deletions sources/models/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,13 @@ use std::net::IpAddr;
use crate::de::{deserialize_limit, deserialize_mirrors, deserialize_node_taints};
use modeled_types::{
BootConfigKey, BootConfigValue, BootstrapContainerMode, CpuManagerPolicy, CredentialProvider,
DNSDomain, EtcHostsEntries, Identifier, IntegerPercent, KubernetesAuthenticationMode,
KubernetesBootstrapToken, KubernetesCloudProvider, KubernetesClusterDnsIp,
KubernetesClusterName, KubernetesDurationValue, KubernetesLabelKey, KubernetesLabelValue,
KubernetesQuantityValue, KubernetesReservedResourceKey, KubernetesTaintValue,
KubernetesThresholdValue, OciDefaultsCapability, OciDefaultsResourceLimitType,
PemCertificateString, SingleLineString, TopologyManagerPolicy, TopologyManagerScope, Url,
ValidBase64, ValidLinuxHostname,
DNSDomain, EtcHostsEntries, Identifier, IntegerPercent, KernelCpuSetValue,
KubernetesAuthenticationMode, KubernetesBootstrapToken, KubernetesCloudProvider,
KubernetesClusterDnsIp, KubernetesClusterName, KubernetesDurationValue, KubernetesLabelKey,
KubernetesLabelValue, KubernetesQuantityValue, KubernetesReservedResourceKey,
KubernetesTaintValue, KubernetesThresholdValue, OciDefaultsCapability,
OciDefaultsResourceLimitType, PemCertificateString, SingleLineString, TopologyManagerPolicy,
TopologyManagerScope, Url, ValidBase64, ValidLinuxHostname,
};

#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
Expand Down Expand Up @@ -137,6 +137,7 @@ struct KubernetesSettings {
shutdown_grace_period_for_critical_pods: KubernetesDurationValue,
memory_manager_reserved_memory: HashMap<Identifier, KubernetesMemoryReservation>,
memory_manager_policy: KubernetesMemoryManagerPolicy,
reserved_cpus: KernelCpuSetValue,

// Settings where we generate a value based on the runtime environment. The user can specify a
// value to override the generated one, but typically would not.
Expand Down

0 comments on commit 2d4790f

Please sign in to comment.