diff --git a/.github/workflows/e2e-testing.yml b/.github/workflows/controller-e2e-test.yml similarity index 97% rename from .github/workflows/e2e-testing.yml rename to .github/workflows/controller-e2e-test.yml index 50a7843ca..364cfe1ee 100644 --- a/.github/workflows/e2e-testing.yml +++ b/.github/workflows/controller-e2e-test.yml @@ -1,9 +1,9 @@ -name: e2e Testing +name: Controller E2e Test run-name: e2e tests run by ${{ github.actor }} on: workflow_dispatch: jobs: - e2e-testing: + e2e-test: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 diff --git a/README.md b/README.md index 7446c9318..437eaba04 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![Regression Testing](https://github.com/vmware-research/verifiable-controllers/actions/workflows/regression-testing.yml/badge.svg)](https://github.com/vmware-research/verifiable-controllers/actions/workflows/regression-testing.yml) [![Verus Image Build](https://github.com/vmware-research/verifiable-controllers/actions/workflows/verus-image-build.yml/badge.svg)](https://github.com/vmware-research/verifiable-controllers/actions/workflows/verus-image-build.yml) [![Controller Image Build](https://github.com/vmware-research/verifiable-controllers/actions/workflows/controller-image-build.yml/badge.svg)](https://github.com/vmware-research/verifiable-controllers/actions/workflows/controller-image-build.yml) +[![Controller e2e test](https://github.com/vmware-research/verifiable-controllers/actions/workflows/controller-e2e-test.yml/badge.svg)](https://github.com/vmware-research/verifiable-controllers/actions/workflows/controller-e2e-test.yml) # verifiable-controllers diff --git a/e2e/src/common.rs b/e2e/src/common.rs index 1f63fab43..c916a7d25 100644 --- a/e2e/src/common.rs +++ b/e2e/src/common.rs @@ -45,13 +45,17 @@ pub enum Error { RabbitmqUserPassFailed, } -pub async fn apply( +pub async fn apply_file( pth: std::path::PathBuf, client: Client, discovery: &Discovery, ) -> Result { - let ssapply = PatchParams::apply("kubectl-light").force(); let yaml = std::fs::read_to_string(&pth)?; + apply(yaml, client, discovery).await +} + +pub async fn apply(yaml: String, client: Client, discovery: &Discovery) -> Result { + let ssapply = PatchParams::apply("kubectl-light").force(); let doc = serde_yaml::from_str(&yaml)?; let obj: DynamicObject = serde_yaml::from_value(doc)?; diff --git a/e2e/src/rabbitmq_e2e.rs b/e2e/src/rabbitmq_e2e.rs index 663b9bac1..4650b4123 100644 --- a/e2e/src/rabbitmq_e2e.rs +++ b/e2e/src/rabbitmq_e2e.rs @@ -24,6 +24,22 @@ use tokio::time::sleep; use crate::common::apply; use crate::common::Error; +pub fn rabbitmq_cluster() -> String { + " + apiVersion: anvil.dev/v1 + kind: RabbitmqCluster + metadata: + name: rabbitmq + namespace: default + spec: + replicas: 3 + rabbitmqConfig: + additionalConfig: | + log.console.level = debug + " + .to_string() +} + pub async fn rabbitmq_e2e_test() -> Result<(), Error> { // check if the CRD is already registered let client = Client::try_default().await?; @@ -31,26 +47,25 @@ pub async fn rabbitmq_e2e_test() -> Result<(), Error> { let rabbitmq_crd = crds.get("rabbitmqclusters.anvil.dev").await; match rabbitmq_crd { Err(e) => { - println!("No CRD found, create one before run the e2e test!\n"); + println!("No CRD found, create one before run the e2e test."); return Err(Error::CRDGetFailed(e)); } Ok(crd) => { - println!("CRD found, continue to run the e2e test!\n"); + println!("CRD found, continue to run the e2e test."); } } // create a rabbitmq cluster let discovery = Discovery::new(client.clone()).run().await?; - let pth = PathBuf::from("./rabbitmq.yaml"); - let rabbitmq_name = apply(pth, client.clone(), &discovery).await?; + let rabbitmq_name = apply(rabbitmq_cluster(), client.clone(), &discovery).await?; let rabbitmq_sts_name = format!("{}-server", &rabbitmq_name); let rabbitmq_cm_name = format!("{}-server-conf", &rabbitmq_name); - let seconds = Duration::from_secs(600); + let timeout = Duration::from_secs(600); let start = Instant::now(); loop { sleep(Duration::from_secs(5)).await; - if start.elapsed() > seconds { + if start.elapsed() > timeout { return Err(Error::Timeout); } // Check configmap @@ -58,7 +73,7 @@ pub async fn rabbitmq_e2e_test() -> Result<(), Error> { let cm = configmaps.get(&rabbitmq_cm_name).await; match cm { Err(e) => { - println!("No configmap found, continue to wait!\n"); + println!("Get configmap failed with {}, continue to wait.", e); continue; } Ok(cm) => { @@ -67,9 +82,12 @@ pub async fn rabbitmq_e2e_test() -> Result<(), Error> { if !user_config.contains("default_user=new_user") || !user_config.contains("default_pass=new_pass") { - println!("Configmap is not consistent with rabbitmq cluster spec! e2e_test failed!\n"); + println!( + "Configmap is not consistent with rabbitmq cluster spec. E2e test failed." + ); return Err(Error::RabbitmqConfigMapFailed); } + println!("Configmap is found as expected."); } }; // Check statefulset @@ -77,17 +95,15 @@ pub async fn rabbitmq_e2e_test() -> Result<(), Error> { let sts = sts_api.get(&rabbitmq_sts_name).await; match sts { Err(e) => { - println!("No statefulset found, continue to wait!\n"); + println!("Get statefulset failed with {}, continue to wait.", e); continue; } Ok(sts) => { if sts.spec.unwrap().replicas != Some(3) { - println!("Statefulset spec is not consistent with rabbitmq cluster spec! e2e_test failed!\n"); + println!("Statefulset spec is not consistent with rabbitmq cluster spec. E2e test failed."); return Err(Error::RabbitmqStsFailed); } - if sts.status.unwrap().replicas != 3 { - continue; - } + println!("Statefulset is found as expected."); } }; @@ -96,7 +112,7 @@ pub async fn rabbitmq_e2e_test() -> Result<(), Error> { let lp = ListParams::default().labels(&format!("app={}", &rabbitmq_name)); // only want results for our pod let pod_list = pods.list(&lp).await?; if pod_list.items.len() != 3 { - println!("Pods are not ready! Continue to wait!\n"); + println!("Pods are not ready. Continue to wait."); continue; } let mut pods_ready = true; @@ -106,7 +122,10 @@ pub async fn rabbitmq_e2e_test() -> Result<(), Error> { // container should also be ready || !status.container_statuses.unwrap()[0].ready { - println!("Pods are not ready! Continue to wait!\n"); + println!( + "Pod {} is not ready. Continue to wait.", + p.metadata.name.unwrap() + ); pods_ready = false; break; } @@ -119,7 +138,7 @@ pub async fn rabbitmq_e2e_test() -> Result<(), Error> { .await?; let err = get_err(attached).await; if err != "" { - println!("User and password test failed!\n"); + println!("User and password test failed."); return Err(Error::RabbitmqUserPassFailed); } } @@ -127,7 +146,7 @@ pub async fn rabbitmq_e2e_test() -> Result<(), Error> { break; } } - println!("Rabbitmq cluster is ready! e2e test passed\n"); + println!("E2e test passed."); Ok(()) } diff --git a/e2e/src/zookeeper_e2e.rs b/e2e/src/zookeeper_e2e.rs index 902802416..3d8fa2b6e 100644 --- a/e2e/src/zookeeper_e2e.rs +++ b/e2e/src/zookeeper_e2e.rs @@ -20,6 +20,40 @@ use tokio::time::sleep; use crate::common::apply; use crate::common::Error; +pub fn zookeeper_cluster() -> String { + " + apiVersion: anvil.dev/v1 + kind: ZookeeperCluster + metadata: + name: zookeeper + namespace: default + spec: + replicas: 3 + image: pravega/zookeeper:0.2.14 + resources: + requests: + memory: \"256Mi\" + cpu: \"500m\" + conf: + initLimit: 10 + syncLimit: 2 + tickTime: 2000 + globalOutstandingLimit: 1000 + preAllocSize: 65536 + snapCount: 10000 + commitLogCount: 500 + snapSizeLimitInKb: 4194304 + maxCnxns: 0 + maxClientCnxns: 60 + minSessionTimeout: 4000 + maxSessionTimeout: 40000 + autoPurgeSnapRetainCount: 3 + autoPurgePurgeInterval: 1 + quorumListenOnAllIps: false + " + .to_string() +} + pub async fn zookeeper_e2e_test() -> Result<(), Error> { // check if the CRD is already registered let client = Client::try_default().await?; @@ -27,24 +61,23 @@ pub async fn zookeeper_e2e_test() -> Result<(), Error> { let zk_crd = crds.get("zookeeperclusters.anvil.dev").await; match zk_crd { Err(e) => { - println!("No CRD found, create one before run the e2e test!\n"); + println!("No CRD found, create one before run the e2e test."); return Err(Error::CRDGetFailed(e)); } Ok(crd) => { - println!("CRD found, continue to run the e2e test!\n"); + println!("CRD found, continue to run the e2e test."); } } // create a zookeeper cluster let discovery = Discovery::new(client.clone()).run().await?; - let pth = PathBuf::from("./zookeeper.yaml"); - let zk_name = apply(pth, client.clone(), &discovery).await?; + let zk_name = apply(zookeeper_cluster(), client.clone(), &discovery).await?; - let seconds = Duration::from_secs(360); + let timeout = Duration::from_secs(360); let start = Instant::now(); loop { sleep(Duration::from_secs(5)).await; - if start.elapsed() > seconds { + if start.elapsed() > timeout { return Err(Error::Timeout); } // Check statefulset @@ -52,17 +85,15 @@ pub async fn zookeeper_e2e_test() -> Result<(), Error> { let sts = sts_api.get(&zk_name).await; match sts { Err(e) => { - println!("No statefulset found, continue to wait!\n"); + println!("Get statefulset failed with error {}.", e); continue; } Ok(sts) => { if sts.spec.unwrap().replicas != Some(3) { - println!("Statefulset spec is not consistent with zookeeper cluster spec! e2e_test failed!\n"); + println!("Statefulset spec is not consistent with zookeeper cluster spec. E2e test failed."); return Err(Error::ZookeeperStsFailed); } - if sts.status.unwrap().replicas != 3 { - continue; - } + println!("Statefulset is found as expected."); } }; // Check pods @@ -70,7 +101,7 @@ pub async fn zookeeper_e2e_test() -> Result<(), Error> { let lp = ListParams::default().labels(&format!("app={}", &zk_name)); // only want results for our pod let pod_list = pods.list(&lp).await?; if pod_list.items.len() != 3 { - println!("Pods are not ready! Continue to wait!\n"); + println!("Pods are not ready. Continue to wait."); continue; } let mut pods_ready = true; @@ -80,7 +111,10 @@ pub async fn zookeeper_e2e_test() -> Result<(), Error> { // container should also be ready || !status.container_statuses.unwrap()[0].ready { - println!("Pods are not ready! Continue to wait!\n"); + println!( + "Pod {} not ready. Continue to wait.", + p.metadata.name.unwrap() + ); pods_ready = false; break; } @@ -89,6 +123,6 @@ pub async fn zookeeper_e2e_test() -> Result<(), Error> { break; } } - println!("Zookeeper cluster is ready! e2e test passed\n"); + println!("E2e test passed."); Ok(()) }