Support pod placement configuration for zookeeper and fluent (#259)

Signed-off-by: Xudong Sun <[email protected]>

deploy/fluent/crd.yaml

@@ -52,6 +52,29 @@ spec:
                       description: "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/"
                       type: object
                   type: object
+                tolerations:
+                  items:
+                    description: "The pod this Toleration is attached to tolerates any taint that matches the triple <key,value,effect> using the matching operator <operator>."
+                    properties:
+                      effect:
+                        description: "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.\n\n"
+                        type: string
+                      key:
+                        description: "Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys."
+                        type: string
+                      operator:
+                        description: "Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.\n\n"
+                        type: string
+                      tolerationSeconds:
+                        description: "TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system."
+                        format: int64
+                        type: integer
+                      value:
+                        description: "Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string."
+                        type: string
+                    type: object
+                  nullable: true
+                  type: array
               required:
                 - fluentBitConfigName
               type: object

deploy/fluent/fluentbit.yaml

@@ -5,6 +5,8 @@ metadata:
     namespace: default
 spec:
     fluentBitConfigName: fluent-bit-config
+    tolerations:
+        - operator: Exists
 ---
 apiVersion: anvil.dev/v1
 kind: FluentBitConfig

e2e/src/fluent_e2e.rs

@@ -31,6 +31,8 @@ pub fn fluent_bit() -> String {
         namespace: default
     spec:
         fluentBitConfigName: fluent-bit-config
+        tolerations:
+            - operator: Exists
     "
     .to_string()
 }

src/controller_examples/fluent_controller/fluentbit/exec/reconciler.rs

@@ -555,7 +555,7 @@ fn make_fluentbit_pod_spec(fluentbit: &FluentBit) -> (pod_spec: PodSpec)
         }
         volumes
     });
-    pod_spec.set_tolerations(make_tolerations(&fluentbit));
+    pod_spec.overwrite_tolerations(fluentbit.spec().tolerations());
     pod_spec
 }
 
@@ -595,18 +595,4 @@ fn make_env(fluentbit: &FluentBit) -> Vec<EnvVar> {
     env_vars
 }
 
-#[verifier(external_body)]
-fn make_tolerations(fluentbit: &FluentBit) -> Vec<Toleration> {
-    let mut tolerations = Vec::new();
-    tolerations.push(
-        Toleration::from_kube(
-            deps_hack::k8s_openapi::api::core::v1::Toleration {
-                operator: Some("Exists".to_string()),
-                ..deps_hack::k8s_openapi::api::core::v1::Toleration::default()
-            }
-        )
-    );
-    tolerations
-}
-
 }

src/controller_examples/fluent_controller/fluentbit/exec/types.rs

@@ -4,7 +4,7 @@ use crate::fluent_controller::fluentbit::spec::types::*;
 use crate::kubernetes_api_objects::error::ParseDynamicObjectError;
 use crate::kubernetes_api_objects::{
     api_resource::*, common::*, dynamic::*, marshal::*, object_meta::*, owner_reference::*,
-    resource::*, resource_requirements::*,
+    resource::*, resource_requirements::*, toleration::*,
 };
 use crate::pervasive_ext::string_view::*;
 use deps_hack::kube::Resource;
@@ -125,6 +125,18 @@ impl FluentBitSpec {
     {
         ResourceRequirements::from_kube(self.inner.resources.clone())
     }
+
+    #[verifier(external_body)]
+    pub fn tolerations(&self) -> (tolerations: Option<Vec<Toleration>>)
+        ensures
+            self@.tolerations.is_Some() == tolerations.is_Some(),
+            tolerations.is_Some() ==> tolerations.get_Some_0()@.map_values(|t: Toleration| t@) == self@.tolerations.get_Some_0(),
+    {
+        match &self.inner.tolerations {
+            Some(tols) => Some(tols.clone().into_iter().map(|t: deps_hack::k8s_openapi::api::core::v1::Toleration| Toleration::from_kube(t)).collect()),
+            None => None,
+        }
+    }
 }
 
 }

src/controller_examples/fluent_controller/fluentbit/spec/reconciler.rs

@@ -277,9 +277,9 @@ pub open spec fn make_fluentbit_pod_spec(fluentbit: FluentBitView) -> PodSpecVie
         fluentbit.metadata.name.is_Some(),
         fluentbit.metadata.namespace.is_Some(),
 {
-    PodSpecView::default()
-        .set_service_account_name(make_service_account_name(fluentbit.metadata.name.get_Some_0()))
-        .set_volumes(seq![
+    PodSpecView {
+        service_account_name: Some(make_service_account_name(fluentbit.metadata.name.get_Some_0())),
+        volumes: Some(seq![
             VolumeView::default()
                 .set_name(new_strlit("varlibcontainers")@)
                 .set_host_path(HostPathVolumeSourceView::default()
@@ -305,8 +305,8 @@ pub open spec fn make_fluentbit_pod_spec(fluentbit: FluentBitView) -> PodSpecVie
                 .set_host_path(HostPathVolumeSourceView::default()
                     .set_path(new_strlit("/var/lib/fluent-bit/")@)
                 ),
-        ])
-        .set_containers(seq![
+        ]),
+        containers: seq![
             ContainerView::default()
                 .set_name(new_strlit("fluent-bit")@)
                 .set_image(new_strlit("kubesphere/fluent-bit:v2.1.7")@)
@@ -337,7 +337,10 @@ pub open spec fn make_fluentbit_pod_spec(fluentbit: FluentBitView) -> PodSpecVie
                         .set_container_port(2020),
                 ])
                 .set_resources(fluentbit.spec.resources)
-        ])
+        ],
+        tolerations: fluentbit.spec.tolerations,
+        ..PodSpecView::default()
+    }
 }
 
 }

src/controller_examples/fluent_controller/fluentbit/spec/types.rs

@@ -3,7 +3,7 @@
 use crate::kubernetes_api_objects::error::ParseDynamicObjectError;
 use crate::kubernetes_api_objects::{
     api_resource::*, common::*, dynamic::*, marshal::*, object_meta::*, owner_reference::*,
-    resource::*, resource_requirements::*,
+    resource::*, resource_requirements::*, toleration::*,
 };
 use crate::pervasive_ext::string_view::*;
 use vstd::prelude::*;
@@ -106,6 +106,7 @@ impl ResourceView for FluentBitView {
 pub struct FluentBitSpecView {
     pub fluentbit_config_name: StringView,
     pub resources: ResourceRequirementsView,
+    pub tolerations: Option<Seq<TolerationView>>,
 }
 
 impl FluentBitSpecView {}

src/controller_examples/zookeeper_controller/exec/reconciler.rs

@@ -1281,6 +1281,7 @@ fn make_zk_pod_spec(zk: &ZookeeperCluster) -> (pod_spec: PodSpec)
 {
     let mut pod_spec = PodSpec::default();
 
+    pod_spec.overwrite_affinity(zk.spec().affinity());
     pod_spec.set_containers({
         let mut containers = Vec::new();
         containers.push({
@@ -1446,6 +1447,7 @@ fn make_zk_pod_spec(zk: &ZookeeperCluster) -> (pod_spec: PodSpec)
 
         volumes
     });
+    pod_spec.overwrite_tolerations(zk.spec().tolerations());
 
     pod_spec
 }

src/controller_examples/zookeeper_controller/exec/types.rs

@@ -1,8 +1,9 @@
 // Copyright 2022 VMware, Inc.
 // SPDX-License-Identifier: MIT
 use crate::kubernetes_api_objects::{
-    api_resource::*, common::*, dynamic::*, error::ParseDynamicObjectError, marshal::*,
-    object_meta::*, owner_reference::*, resource::*, resource_requirements::*,
+    affinity::*, api_resource::*, common::*, dynamic::*, error::ParseDynamicObjectError,
+    marshal::*, object_meta::*, owner_reference::*, resource::*, resource_requirements::*,
+    toleration::*,
 };
 use crate::pervasive_ext::string_view::*;
 use crate::zookeeper_controller::spec::types::*;
@@ -143,6 +144,30 @@ impl ZookeeperClusterSpec {
     {
         ResourceRequirements::from_kube(self.inner.resources.clone())
     }
+
+    #[verifier(external_body)]
+    pub fn affinity(&self) -> (affinity: Option<Affinity>)
+        ensures
+            self@.affinity.is_Some() == affinity.is_Some(),
+            affinity.is_Some() ==> affinity.get_Some_0()@ == self@.affinity.get_Some_0(),
+    {
+        match &self.inner.affinity {
+            Some(a) => Some(Affinity::from_kube(a.clone())),
+            None => None,
+        }
+    }
+
+    #[verifier(external_body)]
+    pub fn tolerations(&self) -> (tolerations: Option<Vec<Toleration>>)
+        ensures
+            self@.tolerations.is_Some() == tolerations.is_Some(),
+            tolerations.is_Some() ==> tolerations.get_Some_0()@.map_values(|t: Toleration| t@) == self@.tolerations.get_Some_0(),
+    {
+        match &self.inner.tolerations {
+            Some(tols) => Some(tols.clone().into_iter().map(|t: deps_hack::k8s_openapi::api::core::v1::Toleration| Toleration::from_kube(t)).collect()),
+            None => None,
+        }
+    }
 }
 
 impl ResourceWrapper<deps_hack::ZookeeperClusterSpec> for ZookeeperClusterSpec {

src/controller_examples/zookeeper_controller/spec/reconciler.rs

@@ -1071,8 +1071,9 @@ pub open spec fn make_zk_pod_spec(zk: ZookeeperClusterView) -> PodSpecView
     recommends
         zk.well_formed(),
 {
-    PodSpecView::default()
-        .set_containers(seq![
+    PodSpecView {
+        affinity: zk.spec.affinity,
+        containers: seq![
             ContainerView::default()
                 .set_name(new_strlit("zookeeper")@)
                 .set_image(zk.spec.image)
@@ -1124,12 +1125,15 @@ pub open spec fn make_zk_pod_spec(zk: ZookeeperClusterView) -> PodSpecView
                         .set_success_threshold(1)
                         .set_timeout_seconds(10)
                 )
-        ])
-        .set_volumes(seq![
+        ],
+        volumes: Some(seq![
             VolumeView::default().set_name(new_strlit("conf")@).set_config_map(
                 ConfigMapVolumeSourceView::default().set_name(zk.metadata.name.get_Some_0() + new_strlit("-configmap")@)
             )
-        ])
+        ]),
+        tolerations: zk.spec.tolerations,
+        ..PodSpecView::default()
+    }
 }
 
 }

src/controller_examples/zookeeper_controller/spec/types.rs

@@ -1,8 +1,9 @@
 // Copyright 2022 VMware, Inc.
 // SPDX-License-Identifier: MIT
 use crate::kubernetes_api_objects::{
-    api_resource::*, common::*, dynamic::*, error::ParseDynamicObjectError, marshal::*,
-    object_meta::*, owner_reference::*, resource::*, resource_requirements::*,
+    affinity::*, api_resource::*, common::*, dynamic::*, error::ParseDynamicObjectError,
+    marshal::*, object_meta::*, owner_reference::*, resource::*, resource_requirements::*,
+    toleration::*,
 };
 use crate::pervasive_ext::string_view::*;
 use vstd::prelude::*;
@@ -110,6 +111,8 @@ pub struct ZookeeperClusterSpecView {
     pub ports: ZookeeperPortsView,
     pub conf: ZookeeperConfigView,
     pub resources: ResourceRequirementsView,
+    pub affinity: Option<AffinityView>,
+    pub tolerations: Option<Seq<TolerationView>>,
 }
 
 impl ZookeeperClusterSpecView {}

src/deps_hack/src/lib.rs

@@ -45,6 +45,8 @@ pub struct ZookeeperClusterSpec {
     pub conf: ZookeeperConfig,
     #[serde(default)]
     pub resources: k8s_openapi::api::core::v1::ResourceRequirements,
+    pub affinity: Option<k8s_openapi::api::core::v1::Affinity>,
+    pub tolerations: Option<Vec<k8s_openapi::api::core::v1::Toleration>>,
 }
 
 #[derive(Debug, Clone, serde::Deserialize, serde::Serialize, schemars::JsonSchema)]
@@ -126,6 +128,7 @@ pub struct FluentBitSpec {
     pub fluentbit_config_name: String,
     #[serde(default)]
     pub resources: k8s_openapi::api::core::v1::ResourceRequirements,
+    pub tolerations: Option<Vec<k8s_openapi::api::core::v1::Toleration>>,
 }
 
 #[derive(

src/kubernetes_api_objects/affinity.rs

@@ -16,7 +16,7 @@ verus! {
 /// Affinity is a group of affinity scheduling rules.
 ///
 /// This definition is a wrapper of Affinity defined at
-/// https://github.com/Arnavion/k8s-openapi/blob/master/src/v1_26/api/core/v1/affinity.rs.
+/// https://github.com/Arnavion/k8s-openapi/blob/v0.17.0/src/v1_26/api/core/v1/affinity.rs.
 /// It is supposed to be used in exec controller code.
 ///
 /// More detailed information: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity.
@@ -51,4 +51,4 @@ pub struct AffinityView {}
 
 impl AffinityView {}
 
-}
+}