chore: add chainsaw e2e tests support

.chainsaw.yaml

@@ -0,0 +1,13 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/configuration-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Configuration
+metadata:
+  name: configuration
+spec:
+  parallel: 2
+  timeouts:
+    assert: 5m0s
+    cleanup: 5m0s
+    delete: 5m0s
+    error: 5m0s
+    exec: 5m0s

.github/workflows/test.yaml

@@ -170,6 +170,93 @@ jobs:
       - name: Stop PGO
         run: docker stop 'postgres-operator' || true
 
+  chainsaw-k3d:
+    runs-on: ubuntu-20.04
+    needs: [go-test]
+    strategy:
+      fail-fast: false
+      matrix:
+        kubernetes: [v1.28, v1.27, v1.26, v1.25]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with: { go-version: 1.x }
+
+      - name: Start k3s
+        uses: ./.github/actions/k3d
+        with:
+          k3s-channel: "${{ matrix.kubernetes }}"
+          prefetch-images: |
+            registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi8-4.30-19
+            registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.47-2
+            registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.21-0
+            registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:latest
+            registry.developers.crunchydata.com/crunchydata/crunchy-upgrade:latest
+            registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-14.10-0
+            registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-14.10-3.1-0
+            registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-15.5-0
+            registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-15.5-3.3-0
+            registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-16.1-0
+            registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-16.1-3.3-0
+            registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-16.1-3.4-0
+            registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi8-7.8-0
+      - run: go mod download
+      - name: Build executable
+        run: PGO_VERSION='${{ github.sha }}' make build-postgres-operator
+
+      - name: Get pgMonitor files.
+        run: make get-pgmonitor
+        env:
+          PGMONITOR_DIR: "${{ github.workspace }}/hack/tools/pgmonitor"
+          QUERIES_CONFIG_DIR: "${{ github.workspace }}/hack/tools/queries" 
+
+      # Start a Docker container with the working directory mounted.
+      - name: Start PGO
+        run: |
+          kubectl apply --server-side -k ./config/namespace
+          kubectl apply --server-side -k ./config/dev
+          hack/create-kubeconfig.sh postgres-operator pgo
+          docker run --detach --network host --read-only \
+            --volume "$(pwd):/mnt" --workdir '/mnt' --env 'PATH=/mnt/bin' \
+            --env 'QUERIES_CONFIG_DIR=/mnt/hack/tools/queries' \
+            --env 'KUBECONFIG=hack/.kube/postgres-operator/pgo' \
+            --env 'RELATED_IMAGE_PGADMIN=registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi8-4.30-19' \
+            --env 'RELATED_IMAGE_PGBACKREST=registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.47-2' \
+            --env 'RELATED_IMAGE_PGBOUNCER=registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.21-0' \
+            --env 'RELATED_IMAGE_PGEXPORTER=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:latest' \
+            --env 'RELATED_IMAGE_PGUPGRADE=registry.developers.crunchydata.com/crunchydata/crunchy-upgrade:latest' \
+            --env 'RELATED_IMAGE_POSTGRES_14=registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-14.10-0' \
+            --env 'RELATED_IMAGE_POSTGRES_14_GIS_3.1=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-14.10-3.1-0' \
+            --env 'RELATED_IMAGE_POSTGRES_15=registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-15.5-0' \
+            --env 'RELATED_IMAGE_POSTGRES_15_GIS_3.3=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-15.5-3.3-0' \
+            --env 'RELATED_IMAGE_POSTGRES_16=registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-16.1-0' \
+            --env 'RELATED_IMAGE_POSTGRES_16_GIS_3.3=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-16.1-3.3-0' \
+            --env 'RELATED_IMAGE_POSTGRES_16_GIS_3.4=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-16.1-3.4-0' \
+            --env 'RELATED_IMAGE_STANDALONE_PGADMIN=registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi8-7.8-0' \
+            --env 'PGO_FEATURE_GATES=TablespaceVolumes=true' \
+            --name 'postgres-operator' ubuntu \
+            postgres-operator
+
+      - name: Install chainsaw
+        uses: kyverno/action-install-chainsaw@07b6c986572f2abaf6647c85d37cbecfddc4a6ab # v0.1.3
+
+      # - run: make generate-kuttl
+      #   env:
+      #     KUTTL_PG_UPGRADE_FROM_VERSION: '15'
+      #     KUTTL_PG_UPGRADE_TO_VERSION: '16'
+      #     KUTTL_PG_VERSION: '15'
+      #     KUTTL_POSTGIS_VERSION: '3.4'
+      #     KUTTL_PSQL_IMAGE: 'registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-16.1-0'
+
+      - run: |
+          make check-chainsaw && exit
+          failed=$?
+          echo '::group::PGO logs'; docker logs 'postgres-operator'; echo '::endgroup::'
+          exit $failed
+
+      - name: Stop PGO
+        run: docker stop 'postgres-operator' || true
+
   coverage-report:
     if: ${{ success() || contains(needs.*.result, 'success') }}
     runs-on: ubuntu-latest

Makefile

@@ -19,6 +19,7 @@ GO_CMD = $(GO_ENV) $(GO)
 GO_TEST ?= $(GO) test
 KUTTL ?= kubectl-kuttl
 KUTTL_TEST ?= $(KUTTL) test
+CHAINSAW ?= chainsaw
 
 # Disable optimizations if creating a debug build
 ifeq ("$(DEBUG_BUILD)", "true")
@@ -214,6 +215,12 @@ check-envtest-existing: createnamespaces
 		$(GO_TEST) -count=1 -cover -p=1 -tags=envtest ./...
 	kubectl delete -k ./config/dev
 
+# Expects operator to be running
+.PHONY: check-chainsaw
+check-chainsaw: ## Run chainsaw end-to-end tests
+check-chainsaw:
+	$(CHAINSAW) test --test-dir ./testing/chainsaw
+
 # Expects operator to be running
 .PHONY: check-kuttl
 check-kuttl: ## Run kuttl end-to-end tests

testing/chainsaw/README.md

@@ -0,0 +1,92 @@
+# KUTTL
+
+## Installing
+
+Docs for install: https://kuttl.dev/docs/cli.html#setup-the-kuttl-kubectl-plugin
+
+Options:
+  - Download and install the binary
+  - Install the `kubectl krew` [plugin manager](https://github.com/kubernetes-sigs/krew)
+    and `kubectl krew install kuttl`
+
+## Cheat sheet
+
+### Suppressing Noisy Logs
+
+KUTTL gives you the option to suppress events from the test logging output. To enable this feature
+update the `kuttl` parameter when calling the `make` target
+
+```
+KUTTL_TEST='kuttl test --suppress-log=events' make check-kuttl
+```
+
+To suppress the events permanently, you can add the following to the KUTTL config (kuttl-test.yaml)
+```
+suppress:
+- events
+```
+
+### Run test suite
+
+Make sure that the operator is running in your Kubernetes environment and that your `kubeconfig` is
+set up. Then run the make targets:
+
+```
+make generate-kuttl check-kuttl
+```
+
+### Running a single test
+
+A single test is considered to be one directory under `kuttl/e2e-generated`, for example
+`kuttl/e2e-generated/restore` is the `restore` test.
+
+There are two ways to run a single test in isolation: 
+- using an env var with the make target: `KUTTL_TEST='kuttl test --test <test-name>' make check-kuttl`
+- using `kubectl kuttl --test` flag: `kubectl kuttl test testing/kuttl/e2e-generated --test <test-name>`
+
+### Writing additional tests
+
+To make it easier to read tests, we want to put our `assert.yaml`/`errors.yaml` files after the
+files that create/update the objects for a step. To achieve this, infix an extra `-` between the
+step number and the object/step name.
+
+For example, if the `00` test step wants to create a cluster and then assert that the cluster is ready,
+the files would be named
+
+```yaml
+00--cluster.yaml # note the extra `-` to ensure that it sorts above the following file
+00-assert.yaml
+```
+
+### Generating tests
+
+KUTTL is good at setting up K8s objects for testing, but does not have a native way to dynamically
+change those K8s objects before applying them. That means that, if we wanted to write a cluster
+connection test for PG 13 and PG 14, we would end up writing two nearly identical tests.
+
+Rather than write those multiple tests, we are using `envsubst` to replace some common variables
+and writing those files to the `testing/kuttl/e2e-generated*` directories.
+
+These templated test files can be generated by setting some variables in the command line and
+calling the `make generate-kuttl` target:
+
+```console
+KUTTL_PG_VERSION=13 KUTTL_POSTGIS_VERSION=3.0 make generate-kuttl
+```
+
+This will loop through the files under the `e2e` and `e2e-other` directories and create matching
+files under the `e2e-generated` and `e2e-generated-other` directories that can be checked for
+correctness before running the tests.
+
+Please note, `make check-kuttl` does not run the `e2e-other` tests.  To run the `postgis-cluster`
+test, you can use:
+
+```
+kubectl kuttl test testing/kuttl/e2e-generated-other/ --timeout=180 --test postgis-cluster`
+```
+
+To run the `gssapi` test, please see testing/kuttl/e2e-other/gssapi/README.md.
+
+To prevent errors, we want to set defaults for all the environment variables used in the source
+YAML files; so if you add a new test with a new variable, please update the Makefile with a
+reasonable/preferred default.