Refactor and enhance e2e tests

.github/workflows/ci.yml

@@ -61,6 +61,23 @@ jobs:
           vargo build --release
       - name: Run unit tests
         run: cargo test
+  fluent-e2e-test:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: "^1.20"
+      - name: Install kind
+        run: go install sigs.k8s.io/[email protected]
+      - name: Install Rust toolchain
+        run: |
+          curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused -fsSL "https://sh.rustup.rs" | sh -s -- --default-toolchain none -y
+      - name: Deploy fluent controller
+        run: ./local-test.sh fluent
+      - name: Run fluent e2e tests
+        run: cd e2e && cargo run -- fluent
   rabbitmq-e2e-test:
     runs-on: ubuntu-20.04
     steps:

deploy/fluent/deploy_local.yaml

@@ -1,22 +1,22 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: fluent-operator
+  name: fluent-controller
   namespace: fluent
   labels:
-    app.kubernetes.io/name: fluent-operator
+    app.kubernetes.io/name: fluent-controller
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app.kubernetes.io/name: fluent-operator
+      app.kubernetes.io/name: fluent-controller
   template:
     metadata:
       labels:
-        app.kubernetes.io/name: fluent-operator
+        app.kubernetes.io/name: fluent-controller
     spec:
       containers:
         - image: local/fluent-controller:v0.1.0
           imagePullPolicy: IfNotPresent
           name: controller
-      serviceAccountName: fluent-operator
+      serviceAccountName: fluent-controller

deploy/fluent/deploy_remote.yaml

@@ -1,21 +1,21 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: fluent-operator
+  name: fluent-controller
   namespace: fluent
   labels:
-    app.kubernetes.io/name: fluent-operator
+    app.kubernetes.io/name: fluent-controller
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app.kubernetes.io/name: fluent-operator
+      app.kubernetes.io/name: fluent-controller
   template:
     metadata:
       labels:
-        app.kubernetes.io/name: fluent-operator
+        app.kubernetes.io/name: fluent-controller
     spec:
       containers:
         - image: ghcr.io/vmware-research/verifiable-controllers/fluent-controller:latest
           name: controller
-      serviceAccountName: fluent-operator
+      serviceAccountName: fluent-controller

deploy/fluent/fluentbit.yaml

@@ -1,7 +1,7 @@
 apiVersion: anvil.dev/v1
 kind: FluentBit
 metadata:
-    name: fluent-bit-config
+    name: fluent-bit
     namespace: default
 spec:
     fluentBitConfigName: fluent-bit-config

deploy/fluent/rbac.yaml

@@ -9,131 +9,66 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   labels:
-    app.kubernetes.io/component: operator
-    app.kubernetes.io/name: fluent-operator
-  name: fluent-operator
+    app.kubernetes.io/name: fluent-controller
+  name: fluent-controller
   namespace: fluent
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   labels:
-    app.kubernetes.io/component: controller
-    app.kubernetes.io/name: fluent-operator
-  name: fluent-operator
+    app.kubernetes.io/name: fluent-controller
+  name: fluent-controller-role
 rules:
   - apiGroups:
-      - apps
+      - anvil.dev
     resources:
-      - daemonsets
-      - statefulsets
+      - "*"
     verbs:
-      - create
-      - delete
-      - get
-      - list
-      - patch
-      - update
-      - watch
+      - "*"
   - apiGroups:
       - ""
     resources:
       - pods
-    verbs:
-      - get
-  - apiGroups:
-      - ""
-    resources:
-      - secrets
+      - services
+      - endpoints
+      - persistentvolumeclaims
+      - events
       - configmaps
+      - secrets
       - serviceaccounts
-      - configmaps
-      - services
-      - namespaces
-    verbs:
-      - create
-      - delete
-      - get
-      - list
-      - patch
-      - update
-      - watch
-  - apiGroups:
-      - fluentbit.fluent.io
-    resources:
-      - collectors
-      - fluentbits
-      - clusterfluentbitconfigs
-      - clusterfilters
-      - clusterinputs
-      - clusteroutputs
-      - clusterparsers
-      - fluentbitconfigs
-      - filters
-      - outputs
-      - parsers
     verbs:
-      - create
-      - delete
-      - get
-      - list
-      - patch
-      - update
-      - watch
+      - "*"
   - apiGroups:
-      - fluentd.fluent.io
+      - apps
     resources:
-      - fluentds
-      - fluentds/status
-      - clusterfluentdconfigs
-      - clusterfluentdconfigs/status
-      - fluentdconfigs
-      - fluentdconfigs/status
-      - clusterfilters
-      - filters
-      - clusteroutputs
-      - outputs
+      - deployments
+      - daemonsets
+      - replicasets
+      - statefulsets
     verbs:
-      - create
-      - delete
-      - get
-      - list
-      - patch
-      - update
-      - watch
+      - "*"
   - apiGroups:
       - rbac.authorization.k8s.io
     resources:
+      - rolebindings
+      - roles
       - clusterrolebindings
-    verbs:
-      - create
-      - list
-      - get
-      - watch
-      - patch
-  - apiGroups:
-      - rbac.authorization.k8s.io
-    resources:
       - clusterroles
     verbs:
-      - create
-      - list
-      - get
-      - watch
-      - patch
+      - "*"
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   labels:
-    app.kubernetes.io/component: controller
-    app.kubernetes.io/name: fluent-operator
-  name: fluent-operator
+    app.kubernetes.io/name: fluent-controller
+  name: fluent-controller-rolebinding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: fluent-operator
+  name: fluent-controller-role
 subjects:
   - kind: ServiceAccount
-    name: fluent-operator
+    name: fluent-controller
     namespace: fluent

deploy/rabbitmq/rbac.yaml

@@ -8,6 +8,8 @@ metadata:
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  labels:
+    app.kubernetes.io/name: rabbitmq-controller
   name: rabbitmq-controller
   namespace: rabbitmq
 ---

e2e/src/common.rs

@@ -1,8 +1,12 @@
 #![allow(unused_imports)]
 #![allow(unused_variables)]
 
+use futures::{StreamExt, TryStreamExt};
 use kube::{
-    api::{Api, DeleteParams, DynamicObject, ListParams, Patch, PatchParams, ResourceExt},
+    api::{
+        Api, AttachedProcess, DeleteParams, DynamicObject, ListParams, Patch, PatchParams,
+        ResourceExt,
+    },
     core::crd::CustomResourceExt,
     core::GroupVersionKind,
     discovery::{ApiCapabilities, ApiResource, Discovery, Scope},
@@ -35,6 +39,9 @@ pub enum Error {
     #[error("Statefulset is not consistent with zookeeper cluster spec!")]
     ZookeeperStsFailed,
 
+    #[error("Failed to set/get data to/from the zookeeper cluster!")]
+    ZookeeperWorkloadFailed,
+
     #[error("Statefulset is not consistent with rabbitmq cluster spec!")]
     RabbitmqStsFailed,
 
@@ -96,3 +103,20 @@ fn dynamic_api(
         Api::default_namespaced_with(client, &ar)
     }
 }
+
+pub async fn get_output_and_err(mut attached: AttachedProcess) -> (String, String) {
+    let stdout = tokio_util::io::ReaderStream::new(attached.stdout().unwrap());
+    let out = stdout
+        .filter_map(|r| async { r.ok().and_then(|v| String::from_utf8(v.to_vec()).ok()) })
+        .collect::<Vec<_>>()
+        .await
+        .join("");
+    let stderr = tokio_util::io::ReaderStream::new(attached.stderr().unwrap());
+    let err = stderr
+        .filter_map(|r| async { r.ok().and_then(|v| String::from_utf8(v.to_vec()).ok()) })
+        .collect::<Vec<_>>()
+        .await
+        .join("");
+    attached.join().await.unwrap();
+    (out, err)
+}