chore: refactor e2e tests to have fewer dependencies

.gitignore

@@ -25,3 +25,7 @@ tmp/
 ginkgo.report
 __debug*
 test-results.xml
+coverprofile.out
+junit-report.xml
+canary-checker.properties
+*.code-workspace

Makefile

@@ -26,9 +26,10 @@ endif
 
 all: manager
 
-# Run tests
-test: generate fmt vet manifests
-	go test ./... -coverprofile cover.out
+
+.PHONY: test
+test: manifests generate fmt ginkgo
+	ginkgo -vv -r  --cover  --keep-going --junit-report junit-report.xml -- --trace --db-trace
 
 # Build manager binary
 manager: generate fmt vet
@@ -201,6 +202,11 @@ else
 	UPX=$(shell which upx)
 endif
 
+
+.PHONY: ginkgo
+ginkgo:
+	go install github.com/onsi/ginkgo/v2/ginkgo
+
 .bin/controller-gen: .bin
 		GOBIN=$(PWD)/.bin go install sigs.k8s.io/controller-tools/cmd/[email protected]
 		CONTROLLER_GEN=$(GOBIN)/controller-gen
@@ -227,25 +233,10 @@ endif
 	curl -sSLo .bin/karina https://github.com/flanksource/karina/releases/download/v0.50.0/karina_$(OS)-$(ARCH) && \
 	chmod +x .bin/karina
 
-.PHONY: telepresence
-telepresence:
-ifeq (, $(shell which telepresence))
-ifeq ($(OS), darwin)
-	brew install --cask macfuse
-	brew install datawire/blackbird/telepresence-legacy
-else
-	sudo apt-get install -y conntrack
-	wget https://s3.amazonaws.com/datawire-static-files/telepresence/telepresence-0.109.tar.gz
-	tar xzf telepresence-0.109.tar.gz
-	sudo mv telepresence-0.109/bin/telepresence /usr/local/bin/
-	sudo mv telepresence-0.109/libexec/sshuttle-telepresence /usr/local/bin/
-endif
-endif
-
 .bin:
 	mkdir -p .bin
 
-bin: .bin .bin/wait4x .bin/karina .bin/go-junit-report  telepresence
+bin: .bin .bin/wait4x .bin/karina
 
 # Generate all the resources and formats your code, i.e: CRDs, controller-gen, static
 .PHONY: resources

fixtures/git/_image

@@ -0,0 +1 @@
+flanksource/canary-checker-full

fixtures/git/_setup.sh

@@ -38,4 +38,4 @@ curl -vvv  -u gitea_admin:admin   -H "Content-Type: application/json"  http://lo
 kill $PID
 
 
-kubectl create secret generic gitea --from-literal=username=gitea_admin --from-literal=password=admin --from-literal=url=http:///gitea-http.gitea.svc:3000/gitea_admin/test_repo.git
+kubectl create secret generic gitea --from-literal=username=gitea_admin --from-literal=password=admin --from-literal=url=http:///gitea-http.gitea.svc:3000/gitea_admin/test_repo.git --namespace canaries

fixtures/k8s/kubernetes-minimal_pass.yaml

@@ -0,0 +1,23 @@
+apiVersion: canaries.flanksource.com/v1
+kind: Canary
+metadata:
+  name: kube-system-checks
+spec:
+  interval: 30
+  kubernetes:
+    - namespace: kube-system
+      name: kube-system
+      kind: Pod
+      # ready: true
+      # resource:
+      #   labelSelector: k8s-app=kube-dns
+      namespaceSelector:
+        name: default
+      display:
+        expr: |
+          dyn(results).
+          map(i, i.Object).
+          filter(i, !k8s.isHealthy(i)).
+          map(i, "%s/%s -> %s".format([i.metadata.namespace, i.metadata.name, k8s.getHealth(i).message])).join('\n')
+      test:
+        expr: dyn(results).all(x, k8s.isHealthy(x))

fixtures/k8s/kubernetes_bundle.yaml

@@ -12,6 +12,7 @@ spec:
         expr: |
           dyn(results).map(r, {
             'name': r.Object.metadata.name,
+            'namespace': r.Object.metadata.?namespace.orValue(null),
             'labels': r.Object.metadata.labels,
             'pass': k8s.isHealthy(r.Object),
             'message': k8s.getHealth(r.Object).message,
@@ -20,10 +21,13 @@ spec:
     - kind: Pod
       ready: true
       name: pod-bundle
+      resource:
+        labelSelector: app != k8s-not-ready,  Expected-Fail != true, canary-checker.flanksource.com/generated != true, !canary-checker.flanksource.com/check
       transform:
         expr: |
           dyn(results).map(r, {
             'name': r.Object.metadata.name,
+            'namespace': r.Object.metadata.namespace,
             'labels': r.Object.metadata.labels,
             'pass': k8s.isHealthy(r.Object),
             'message': k8s.getHealth(r.Object).message,

fixtures/k8s/namespace_pass.yaml

@@ -23,7 +23,7 @@ spec:
       ingressName: test-namespace-pod
       ingressHost: "test-namespace-pod.127.0.0.1.nip.io"
       readyTimeout: 5000
-      httpTimeout: 15000
+      httpTimeout: 30000
       deleteTimeout: 12000
       ingressTimeout: 20000
       deadline: 60000

fixtures/restic/_image

@@ -0,0 +1 @@
+flanksource/canary-checker-full

test/e2e.sh

@@ -9,15 +9,12 @@ export CLUSTER_NAME=kind-test
 export PATH=$(pwd)/.bin:/usr/local/bin:$PATH
 export ROOT=$(pwd)
 export TEST_FOLDER=${TEST_FOLDER:-$1}
-export DOMAIN=${DOMAIN:-127.0.0.1.nip.io}
-export TELEPRESENCE_USE_DEPLOYMENT=0
 export TEST_BINARY=./test.test
 SKIP_SETUP=${SKIP_SETUP:-false}
 SKIP_KARINA=${SKIP_KARINA:-false}
-SKIP_TELEPRESENCE=${SKIP_TELEPRESENCE:-false}
 
 if [[ "$1" == "" ]]; then
-  echo "Usage ./test/e2e.sh TEST_FOLDER  [RunRegex] [--skip-setup] [--skip-karina] [--skip-telepresence] [--skip-all] "
+  echo "Usage ./test/e2e.sh TEST_FOLDER  [RunRegex] [--skip-setup] [--skip-karina] [--skip-all] "
   exit 1
 fi
 
@@ -27,16 +24,12 @@ fi
 if [[ "$*" == *"--skip-karina"* ]]; then
   SKIP_KARINA=true
 fi
-if [[ "$*" == *"--skip-telepresence"* ]]; then
-  SKIP_TELEPRESENCE=true
-fi
 if [[ "$*" == *"--skip-all"* ]]; then
-  SKIP_TELEPRESENCE=true
   SKIP_KARINA=true
   SKIP_SETUP=true
 fi
 
-echo "Testing $TEST_FOLDER with skip_setup=$SKIP_SETUP skip_karina=$SKIP_KARINA" skip_telepresence=$SKIP_TELEPRESENCE
+echo "Testing $TEST_FOLDER with skip_setup=$SKIP_SETUP skip_karina=$SKIP_KARINA"
 
 if [[ "$SKIP_KARINA" != "true" ]] ; then
   echo "::group::Provisioning"
@@ -60,18 +53,7 @@ if [[ "$SKIP_KARINA" != "true" ]] ; then
   echo "::endgroup::"
 fi
 
-_DOMAIN=$(kubectl get cm -n quack quack-config -o json | jq -r ".data.domain" || echo)
-if [[ "$_DOMAIN" != "" ]]; then
-  echo Using domain: $_DOMAIN
-  export DOMAIN=$_DOMAIN
-fi
-
-cat > /tmp/namespace.yml <<EOF
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: canaries
-EOF
+kubectl create ns canaries || true
 
 if [ "$SKIP_SETUP" != "true" ]; then
   echo "::group::Setting up"
@@ -84,10 +66,6 @@ if [ "$SKIP_SETUP" != "true" ]; then
     bash $TEST_FOLDER/_setup.sh || echo Setup failed, attempting tests anyway
   fi
 
-  if [ -e $TEST_FOLDER/../namespace.yml ]; then
-    $KARINA apply /tmp/namespace.yml -v
-  fi
-
   if [ -e $TEST_FOLDER/_setup.yaml ]; then
     $KARINA apply $(pwd)/$TEST_FOLDER/_setup.yaml -v
   fi
@@ -107,6 +85,13 @@ if [ "$SKIP_SETUP" != "true" ]; then
   echo "::endgroup::"
 fi
 
+image=ubuntu
+
+if [[ -e $TEST_FOLDER/_image ]]; then
+  image=$(cat $TEST_FOLDER/_image )
+  echo Using image: $image
+fi
+
 cd $ROOT/test
 
 if [[ "$TEST_REGEX" != "" ]]; then
@@ -115,19 +100,32 @@ fi
 
 if [[ ! -e $TEST_BINARY ]]; then
   echo "::group::Compiling tests"
-  make build
+  GOOS=linux GOARCH=amd64 make build
   echo "::endgroup::"
 fi
 echo "::group::Testing"
 
-if [[ "$SKIP_TELEPRESENCE" != "true" ]]; then
-  telepresence="telepresence --mount false -m vpn-tcp --namespace canaries --run"
-fi
-
-cmd="$telepresence ginkgo -p  --junit-report test-results.xml $TEST_BINARY --test-folder $TEST_FOLDER $EXTRA"
-$cmd  2>&1 | tee test.out
 
-if  grep "Test Suite Failed" test.out ; then
+rm test.out test-results.xml test-results.json || true
+runner=ginkgo-$(date +%s)
+
+kubectl create clusterrolebinding ginkgo-default  --clusterrole=cluster-admin --serviceaccount=default:default || true
+kubectl run $runner --image $image -n default  --command -- bash -c 'sleep 3600'
+function cleanup {
+  kubectl delete po $runner --wait=false
+  kubectl delete clusterrolebinding ginkgo-default
+}
+trap cleanup EXIT
+kubectl wait --for=condition=Ready pod/$runner -n default --timeout=5m
+echo "Copying $TEST_FOLDER to $runner"
+kubectl cp ../$TEST_FOLDER $runner:/tmp/fixtures
+kubectl cp  $TEST_BINARY $runner:/tmp/test
+set +e
+kubectl exec -it $runner -- bash -c  "/tmp/test --test-folder /tmp/fixtures $EXTRA  --ginkgo.junit-report /tmp/test-results.xml -v 5"
+out=$?
+kubectl cp  $runner:/tmp/test-results.xml test-results.xml
+
+if  [[ $out != 0 ]]  ; then
   echo "::endgroup::"
   exit 1
 fi