Set min replicas to 3 for zookeeper (#286)

Signed-off-by: Xudong Sun <[email protected]>
anvil-verifier · Sep 20, 2023 · 6da4124 · 6da4124
1 parent 5220708
commit 6da4124
Show file tree

Hide file tree

Showing 4 changed files with 138 additions and 9 deletions.
diff --git a/deploy/zookeeper/crd.yaml b/deploy/zookeeper/crd.yaml
@@ -617,8 +617,8 @@ spec:
                   format: int32
                   type: integer
                   x-kubernetes-validations:
-                    - rule: "self > 0"
-                      message: replicas should be larger than 0
+                    - rule: "self >= 3"
+                      message: replicas should be at least 3
                 resources:
                   description: ResourceRequirements describes the compute resource requirements.
                   nullable: true

diff --git a/e2e/src/common.rs b/e2e/src/common.rs
@@ -124,6 +124,7 @@ pub async fn get_output_and_err(mut attached: AttachedProcess) -> (String, Strin
 }
 
 pub fn run_command(program: &str, args: Vec<&str>, err_msg: &str) {
+    println!("{} {}", program, args.join(" "));
     let cmd = Command::new(program).args(args).output().expect(err_msg);
     println!("cmd output: {}", String::from_utf8_lossy(&cmd.stdout));
     println!("cmd error: {}", String::from_utf8_lossy(&cmd.stderr));

diff --git a/e2e/src/zookeeper_e2e.rs b/e2e/src/zookeeper_e2e.rs
@@ -198,6 +198,7 @@ pub async fn scaling_test(client: Client, zk_name: String) -> Result<(), Error>
     let timeout = Duration::from_secs(360);
     let start = Instant::now();
     let sts_api: Api<StatefulSet> = Api::default_namespaced(client.clone());
+
     run_command(
         "kubectl",
         vec![
@@ -206,7 +207,7 @@ pub async fn scaling_test(client: Client, zk_name: String) -> Result<(), Error>
             "zookeeper",
             "--type=json",
             "-p",
-            "[{\"op\": \"replace\", \"path\": \"/spec/replicas\", \"value\": 2}]",
+            "[{\"op\": \"replace\", \"path\": \"/spec/replicas\", \"value\": 5}]",
         ],
         "failed to scale zk",
     );
@@ -225,7 +226,7 @@ pub async fn scaling_test(client: Client, zk_name: String) -> Result<(), Error>
                 continue;
             }
             Ok(sts) => {
-                if sts.spec.unwrap().replicas != Some(2) {
+                if sts.spec.unwrap().replicas != Some(5) {
                     println!(
                         "Stateful set spec is not consistent with zookeeper cluster spec yet."
                     );
@@ -241,13 +242,13 @@ pub async fn scaling_test(client: Client, zk_name: String) -> Result<(), Error>
                     .ready_replicas
                     .as_ref()
                     .unwrap()
-                    == 2
+                    == 5
                 {
-                    println!("Scale down is done with 2 replicas ready.");
+                    println!("Scale up is done with 5 replicas ready.");
                     break;
                 } else {
                     println!(
-                        "Scale down is in progress. {} pods are ready now.",
+                        "Scale up is in progress. {} pods are ready now.",
                         sts.status
                             .as_ref()
                             .unwrap()
@@ -280,6 +281,7 @@ pub async fn scaling_test(client: Client, zk_name: String) -> Result<(), Error>
             return Err(Error::Timeout);
         }
 
+        // Check stateful set
         let sts = sts_api.get(&zk_name).await;
         match sts {
             Err(e) => {
@@ -305,7 +307,69 @@ pub async fn scaling_test(client: Client, zk_name: String) -> Result<(), Error>
                     .unwrap()
                     == 3
                 {
-                    println!("Scale up is done with 3 replicas ready.");
+                    println!("Scale down is done with 3 replicas ready.");
+                    break;
+                } else {
+                    println!(
+                        "Scale down is in progress. {} pods are ready now.",
+                        sts.status
+                            .as_ref()
+                            .unwrap()
+                            .ready_replicas
+                            .as_ref()
+                            .unwrap()
+                    );
+                    continue;
+                }
+            }
+        };
+    }
+
+    run_command(
+        "kubectl",
+        vec![
+            "patch",
+            "zk",
+            "zookeeper",
+            "--type=json",
+            "-p",
+            "[{\"op\": \"replace\", \"path\": \"/spec/replicas\", \"value\": 5}]",
+        ],
+        "failed to scale zk",
+    );
+
+    loop {
+        sleep(Duration::from_secs(5)).await;
+        if start.elapsed() > timeout {
+            return Err(Error::Timeout);
+        }
+
+        let sts = sts_api.get(&zk_name).await;
+        match sts {
+            Err(e) => {
+                println!("Get stateful set failed with error {}.", e);
+                continue;
+            }
+            Ok(sts) => {
+                if sts.spec.unwrap().replicas != Some(5) {
+                    println!(
+                        "Stateful set spec is not consistent with zookeeper cluster spec yet."
+                    );
+                    continue;
+                }
+                println!("Stateful set is found as expected.");
+                if sts.status.as_ref().unwrap().ready_replicas.is_none() {
+                    println!("No stateful set pod is ready.");
+                } else if *sts
+                    .status
+                    .as_ref()
+                    .unwrap()
+                    .ready_replicas
+                    .as_ref()
+                    .unwrap()
+                    == 5
+                {
+                    println!("Scale up is done with 5 replicas ready.");
                     break;
                 } else {
                     println!(
@@ -322,6 +386,70 @@ pub async fn scaling_test(client: Client, zk_name: String) -> Result<(), Error>
             }
         };
     }
+
+    run_command(
+        "kubectl",
+        vec![
+            "patch",
+            "zk",
+            "zookeeper",
+            "--type=json",
+            "-p",
+            "[{\"op\": \"replace\", \"path\": \"/spec/replicas\", \"value\": 3}]",
+        ],
+        "failed to scale zk",
+    );
+
+    loop {
+        sleep(Duration::from_secs(5)).await;
+        if start.elapsed() > timeout {
+            return Err(Error::Timeout);
+        }
+
+        // Check stateful set
+        let sts = sts_api.get(&zk_name).await;
+        match sts {
+            Err(e) => {
+                println!("Get stateful set failed with error {}.", e);
+                continue;
+            }
+            Ok(sts) => {
+                if sts.spec.unwrap().replicas != Some(3) {
+                    println!(
+                        "Stateful set spec is not consistent with zookeeper cluster spec yet."
+                    );
+                    continue;
+                }
+                println!("Stateful set is found as expected.");
+                if sts.status.as_ref().unwrap().ready_replicas.is_none() {
+                    println!("No stateful set pod is ready.");
+                } else if *sts
+                    .status
+                    .as_ref()
+                    .unwrap()
+                    .ready_replicas
+                    .as_ref()
+                    .unwrap()
+                    == 3
+                {
+                    println!("Scale down is done with 3 replicas ready.");
+                    break;
+                } else {
+                    println!(
+                        "Scale down is in progress. {} pods are ready now.",
+                        sts.status
+                            .as_ref()
+                            .unwrap()
+                            .ready_replicas
+                            .as_ref()
+                            .unwrap()
+                    );
+                    continue;
+                }
+            }
+        };
+    }
+
     println!("Scaling test passed.");
     Ok(())
 }

diff --git a/src/controller_examples/zookeeper_controller/spec/types.rs b/src/controller_examples/zookeeper_controller/spec/types.rs
@@ -97,7 +97,7 @@ impl ResourceView for ZookeeperClusterView {
     proof fn unmarshal_result_determined_by_unmarshal_spec() {}
 
     open spec fn state_validation(self) -> bool {
-        &&& self.spec.replicas > 0
+        &&& self.spec.replicas >= 3
     }
 
     open spec fn transition_validation(self, old_obj: ZookeeperClusterView) -> bool {