Skip to content

Commit

Permalink
Make e2e tests self-contained
Browse files Browse the repository at this point in the history 
Signed-off-by: Xudong Sun <[email protected]>
  • Loading branch information
@marshtompsxd
marshtompsxd committed Sep 6, 2023
1 parent 8a664e5 commit 27324d4
Show file tree
Hide file tree
Showing 5 changed files with 93 additions and 35 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/e2e-testing.yml → .github/workflows/controller-e2e-test.yml
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
name: e2e Testing
name: Controller E2e Test
run-name: e2e tests run by ${{ github.actor }}
on:
workflow_dispatch:
jobs:
e2e-testing:
e2e-test:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
Expand Down
1 change: 1 addition & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
[![Regression Testing](https://github.com/vmware-research/verifiable-controllers/actions/workflows/regression-testing.yml/badge.svg)](https://github.com/vmware-research/verifiable-controllers/actions/workflows/regression-testing.yml)
[![Verus Image Build](https://github.com/vmware-research/verifiable-controllers/actions/workflows/verus-image-build.yml/badge.svg)](https://github.com/vmware-research/verifiable-controllers/actions/workflows/verus-image-build.yml)
[![Controller Image Build](https://github.com/vmware-research/verifiable-controllers/actions/workflows/controller-image-build.yml/badge.svg)](https://github.com/vmware-research/verifiable-controllers/actions/workflows/controller-image-build.yml)
[![Controller e2e test](https://github.com/vmware-research/verifiable-controllers/actions/workflows/controller-e2e-test.yml/badge.svg)](https://github.com/vmware-research/verifiable-controllers/actions/workflows/controller-e2e-test.yml)

# verifiable-controllers

Expand Down
8 changes: 6 additions & 2 deletions e2e/src/common.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,17 @@ pub enum Error {
RabbitmqUserPassFailed,
}

pub async fn apply(
pub async fn apply_file(
pth: std::path::PathBuf,
client: Client,
discovery: &Discovery,
) -> Result<String, Error> {
let ssapply = PatchParams::apply("kubectl-light").force();
let yaml = std::fs::read_to_string(&pth)?;
apply(yaml, client, discovery).await
}

pub async fn apply(yaml: String, client: Client, discovery: &Discovery) -> Result<String, Error> {
let ssapply = PatchParams::apply("kubectl-light").force();
let doc = serde_yaml::from_str(&yaml)?;

let obj: DynamicObject = serde_yaml::from_value(doc)?;
Expand Down
53 changes: 36 additions & 17 deletions e2e/src/rabbitmq_e2e.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,41 +24,56 @@ use tokio::time::sleep;
use crate::common::apply;
use crate::common::Error;

pub fn rabbitmq_cluster() -> String {
"
apiVersion: anvil.dev/v1
kind: RabbitmqCluster
metadata:
name: rabbitmq
namespace: default
spec:
replicas: 3
rabbitmqConfig:
additionalConfig: |
log.console.level = debug
"
.to_string()
}

pub async fn rabbitmq_e2e_test() -> Result<(), Error> {
// check if the CRD is already registered
let client = Client::try_default().await?;
let crds: Api<CustomResourceDefinition> = Api::all(client.clone());
let rabbitmq_crd = crds.get("rabbitmqclusters.anvil.dev").await;
match rabbitmq_crd {
Err(e) => {
println!("No CRD found, create one before run the e2e test!\n");
println!("No CRD found, create one before run the e2e test.");
return Err(Error::CRDGetFailed(e));
}
Ok(crd) => {
println!("CRD found, continue to run the e2e test!\n");
println!("CRD found, continue to run the e2e test.");
}
}

// create a rabbitmq cluster
let discovery = Discovery::new(client.clone()).run().await?;
let pth = PathBuf::from("./rabbitmq.yaml");
let rabbitmq_name = apply(pth, client.clone(), &discovery).await?;
let rabbitmq_name = apply(rabbitmq_cluster(), client.clone(), &discovery).await?;
let rabbitmq_sts_name = format!("{}-server", &rabbitmq_name);
let rabbitmq_cm_name = format!("{}-server-conf", &rabbitmq_name);

let seconds = Duration::from_secs(600);
let timeout = Duration::from_secs(600);
let start = Instant::now();
loop {
sleep(Duration::from_secs(5)).await;
if start.elapsed() > seconds {
if start.elapsed() > timeout {
return Err(Error::Timeout);
}
// Check configmap
let configmaps: Api<ConfigMap> = Api::default_namespaced(client.clone());
let cm = configmaps.get(&rabbitmq_cm_name).await;
match cm {
Err(e) => {
println!("No configmap found, continue to wait!\n");
println!("Get configmap failed with {}, continue to wait.", e);
continue;
}
Ok(cm) => {
Expand All @@ -67,27 +82,28 @@ pub async fn rabbitmq_e2e_test() -> Result<(), Error> {
if !user_config.contains("default_user=new_user")
|| !user_config.contains("default_pass=new_pass")
{
println!("Configmap is not consistent with rabbitmq cluster spec! e2e_test failed!\n");
println!(
"Configmap is not consistent with rabbitmq cluster spec. E2e test failed."
);
return Err(Error::RabbitmqConfigMapFailed);
}
println!("Configmap is found as expected.");
}
};
// Check statefulset
let sts_api: Api<StatefulSet> = Api::default_namespaced(client.clone());
let sts = sts_api.get(&rabbitmq_sts_name).await;
match sts {
Err(e) => {
println!("No statefulset found, continue to wait!\n");
println!("Get statefulset failed with {}, continue to wait.", e);
continue;
}
Ok(sts) => {
if sts.spec.unwrap().replicas != Some(3) {
println!("Statefulset spec is not consistent with rabbitmq cluster spec! e2e_test failed!\n");
println!("Statefulset spec is not consistent with rabbitmq cluster spec. E2e test failed.");
return Err(Error::RabbitmqStsFailed);
}
if sts.status.unwrap().replicas != 3 {
continue;
}
println!("Statefulset is found as expected.");
}
};

Expand All @@ -96,7 +112,7 @@ pub async fn rabbitmq_e2e_test() -> Result<(), Error> {
let lp = ListParams::default().labels(&format!("app={}", &rabbitmq_name)); // only want results for our pod
let pod_list = pods.list(&lp).await?;
if pod_list.items.len() != 3 {
println!("Pods are not ready! Continue to wait!\n");
println!("Pods are not ready. Continue to wait.");
continue;
}
let mut pods_ready = true;
Expand All @@ -106,7 +122,10 @@ pub async fn rabbitmq_e2e_test() -> Result<(), Error> {
// container should also be ready
|| !status.container_statuses.unwrap()[0].ready
{
println!("Pods are not ready! Continue to wait!\n");
println!(
"Pod {} is not ready. Continue to wait.",
p.metadata.name.unwrap()
);
pods_ready = false;
break;
}
Expand All @@ -119,15 +138,15 @@ pub async fn rabbitmq_e2e_test() -> Result<(), Error> {
.await?;
let err = get_err(attached).await;
if err != "" {
println!("User and password test failed!\n");
println!("User and password test failed.");
return Err(Error::RabbitmqUserPassFailed);
}
}
if pods_ready {
break;
}
}
println!("Rabbitmq cluster is ready! e2e test passed\n");
println!("E2e test passed.");
Ok(())
}

Expand Down
62 changes: 48 additions & 14 deletions e2e/src/zookeeper_e2e.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,57 +20,88 @@ use tokio::time::sleep;
use crate::common::apply;
use crate::common::Error;

pub fn zookeeper_cluster() -> String {
"
apiVersion: anvil.dev/v1
kind: ZookeeperCluster
metadata:
name: zookeeper
namespace: default
spec:
replicas: 3
image: pravega/zookeeper:0.2.14
resources:
requests:
memory: \"256Mi\"
cpu: \"500m\"
conf:
initLimit: 10
syncLimit: 2
tickTime: 2000
globalOutstandingLimit: 1000
preAllocSize: 65536
snapCount: 10000
commitLogCount: 500
snapSizeLimitInKb: 4194304
maxCnxns: 0
maxClientCnxns: 60
minSessionTimeout: 4000
maxSessionTimeout: 40000
autoPurgeSnapRetainCount: 3
autoPurgePurgeInterval: 1
quorumListenOnAllIps: false
"
.to_string()
}

pub async fn zookeeper_e2e_test() -> Result<(), Error> {
// check if the CRD is already registered
let client = Client::try_default().await?;
let crds: Api<CustomResourceDefinition> = Api::all(client.clone());
let zk_crd = crds.get("zookeeperclusters.anvil.dev").await;
match zk_crd {
Err(e) => {
println!("No CRD found, create one before run the e2e test!\n");
println!("No CRD found, create one before run the e2e test.");
return Err(Error::CRDGetFailed(e));
}
Ok(crd) => {
println!("CRD found, continue to run the e2e test!\n");
println!("CRD found, continue to run the e2e test.");
}
}

// create a zookeeper cluster
let discovery = Discovery::new(client.clone()).run().await?;
let pth = PathBuf::from("./zookeeper.yaml");
let zk_name = apply(pth, client.clone(), &discovery).await?;
let zk_name = apply(zookeeper_cluster(), client.clone(), &discovery).await?;

let seconds = Duration::from_secs(360);
let timeout = Duration::from_secs(360);
let start = Instant::now();
loop {
sleep(Duration::from_secs(5)).await;
if start.elapsed() > seconds {
if start.elapsed() > timeout {
return Err(Error::Timeout);
}
// Check statefulset
let sts_api: Api<StatefulSet> = Api::default_namespaced(client.clone());
let sts = sts_api.get(&zk_name).await;
match sts {
Err(e) => {
println!("No statefulset found, continue to wait!\n");
println!("Get statefulset failed with error {}.", e);
continue;
}
Ok(sts) => {
if sts.spec.unwrap().replicas != Some(3) {
println!("Statefulset spec is not consistent with zookeeper cluster spec! e2e_test failed!\n");
println!("Statefulset spec is not consistent with zookeeper cluster spec. E2e test failed.");
return Err(Error::ZookeeperStsFailed);
}
if sts.status.unwrap().replicas != 3 {
continue;
}
println!("Statefulset is found as expected.");
}
};
// Check pods
let pods: Api<Pod> = Api::default_namespaced(client.clone());
let lp = ListParams::default().labels(&format!("app={}", &zk_name)); // only want results for our pod
let pod_list = pods.list(&lp).await?;
if pod_list.items.len() != 3 {
println!("Pods are not ready! Continue to wait!\n");
println!("Pods are not ready. Continue to wait.");
continue;
}
let mut pods_ready = true;
Expand All @@ -80,7 +111,10 @@ pub async fn zookeeper_e2e_test() -> Result<(), Error> {
// container should also be ready
|| !status.container_statuses.unwrap()[0].ready
{
println!("Pods are not ready! Continue to wait!\n");
println!(
"Pod {} not ready. Continue to wait.",
p.metadata.name.unwrap()
);
pods_ready = false;
break;
}
Expand All @@ -89,6 +123,6 @@ pub async fn zookeeper_e2e_test() -> Result<(), Error> {
break;
}
}
println!("Zookeeper cluster is ready! e2e test passed\n");
println!("E2e test passed.");
Ok(())
}

0 comments on commit 27324d4

Please sign in to comment.