Skip to content

Commit

Permalink
Add e2e tests for failpoint scenarios
Browse files Browse the repository at this point in the history
  • Loading branch information
soma00333 committed Jan 22, 2025
1 parent d3d6d29 commit 195a6cf
Show file tree
Hide file tree
Showing 5 changed files with 488 additions and 4 deletions.
36 changes: 36 additions & 0 deletions .github/workflows/test-e2e-failpoint.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
name: E2E Failpoint Tests

on:
push:
pull_request:

jobs:
test-e2e:
name: Run on Ubuntu
runs-on: ubuntu-latest
steps:
- name: Clone the code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683

- name: Setup Go
uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a
with:
go-version-file: 'go.mod'

- name: Install the latest version of kind
run: |
curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64
chmod +x ./kind
sudo mv ./kind /usr/local/bin/kind
- name: Verify kind installation
run: kind version

- name: Create kind cluster
run: kind create cluster

- name: Running Test e2e
run: |
go mod tidy
make gofail-enable
make test-e2e-failpoint
25 changes: 25 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,18 @@ test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated
}
go test ./test/e2e/ -v -ginkgo.v

.PHONY: test-e2e-failpoint
test-e2e-fault: manifests generate fmt vet ## Run the e2e tests using gofail. Expected an isolated environment using Kind.
@command -v kind >/dev/null 2>&1 || { \
echo "Kind is not installed. Please install Kind manually."; \
exit 1; \
}
@kind get clusters | grep -q 'kind' || { \
echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \
exit 1; \
}
go test ./test/e2e_failpoint/ -v -ginkgo.v

.PHONY: lint
lint: golangci-lint ## Run golangci-lint linter
$(GOLANGCI_LINT) run
Expand Down Expand Up @@ -166,6 +178,15 @@ api-docs: crd-ref-docs ## Generate api references docs.
--templates-dir=./docs/api-references/template/ \
--config=./docs/api-references/config.yaml

##@ gofail
.PHONY: gofail-enable
gofail-enable: gofail
gofail enable .

.PHONY: gofail-disable
gofail-disable: gofail
gofail disable .

##@ Dependencies

## Location to install dependencies to
Expand Down Expand Up @@ -213,6 +234,10 @@ golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
$(GOLANGCI_LINT): $(LOCALBIN)
$(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION))

.PHONY: gofail
gofail:
go install go.etcd.io/gofail

# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
# $1 - target path with name of binary
# $2 - package url which can be installed
Expand Down
4 changes: 0 additions & 4 deletions internal/controller/etcdcluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -227,8 +227,6 @@ func (r *EtcdClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
}
// We will interrupt this state and crash the operator before updating the StatefulSet replicas.
// gofail: var CrashAfterAddMember struct{}
// logger.Info("gofail CrashAfterAddMember triggered")
// os.Exit(1)

logger.Info("Learner member added successfully", "peerURLs", peerURL)
} else {
Expand All @@ -246,8 +244,6 @@ func (r *EtcdClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)

// We will interrupt this state and crash the operator before updating the StatefulSet replicas.
// gofail: var CrashAfterRemoveMember struct{}
// logger.Info("gofail CrashAfterRemoveMember triggered")
// os.Exit(1)

logger.Info("Member removed successfully", "memberID", memberID)
}
Expand Down
120 changes: 120 additions & 0 deletions test/e2e_failpoint/failpoint_suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
/*
Copyright 2024.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e_failpoint

import (
"fmt"
"os"
"os/exec"
"testing"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

"go.etcd.io/etcd-operator/test/utils"
)

var (
// Optional Environment Variables:
// - PROMETHEUS_INSTALL_SKIP=true: Skips Prometheus Operator installation during test setup.
// - CERT_MANAGER_INSTALL_SKIP=true: Skips CertManager installation during test setup.
// These variables are useful if Prometheus or CertManager is already installed, avoiding
// re-installation and conflicts.
skipPrometheusInstall = os.Getenv("PROMETHEUS_INSTALL_SKIP") == "true"
skipCertManagerInstall = os.Getenv("CERT_MANAGER_INSTALL_SKIP") == "true"
// isPrometheusOperatorAlreadyInstalled will be set true when prometheus CRDs be found on the cluster
isPrometheusOperatorAlreadyInstalled = false
// isCertManagerAlreadyInstalled will be set true when CertManager CRDs be found on the cluster
isCertManagerAlreadyInstalled = false

// projectImage is the name of the image which will be build and loaded
// with the code source changes to be tested.
projectImage = "example.com/etcd-operator:v0.0.1"
)

// TestE2E runs the end-to-end (e2e) test suite for the project. These tests execute in an isolated,
// temporary environment to validate project changes with the the purposed to be used in CI jobs.
// The default setup requires Kind, builds/loads the Manager Docker image locally, and installs
// CertManager and Prometheus.
func TestE2E(t *testing.T) {
RegisterFailHandler(Fail)
_, _ = fmt.Fprintf(GinkgoWriter, "Starting etcd-operator integration test suite\n")
RunSpecs(t, "e2e suite")
}

var _ = BeforeSuite(func() {
By("Ensure that Prometheus is enabled")
_ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#")

By("generating files")
cmd := exec.Command("make", "generate")
_, err := utils.Run(cmd)
ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to run make generate")

By("generating manifests")
cmd = exec.Command("make", "manifests")
_, err = utils.Run(cmd)
ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to run make manifests")

By("building the manager(Operator) image")
cmd = exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectImage))
_, err = utils.Run(cmd)
ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager(Operator) image")

// TODO(user): If you want to change the e2e test vendor from Kind, ensure the image is
// built and available before running the tests. Also, remove the following block.
By("loading the manager(Operator) image on Kind")
err = utils.LoadImageToKindClusterWithName(projectImage)
ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager(Operator) image into Kind")

// The tests-e2e are intended to run on a temporary cluster that is created and destroyed for testing.
// To prevent errors when tests run in environments with Prometheus or CertManager already installed,
// we check for their presence before execution.
// Setup Prometheus and CertManager before the suite if not skipped and if not already installed
if !skipPrometheusInstall {
By("checking if prometheus is installed already")
isPrometheusOperatorAlreadyInstalled = utils.IsPrometheusCRDsInstalled()
if !isPrometheusOperatorAlreadyInstalled {
_, _ = fmt.Fprintf(GinkgoWriter, "Installing Prometheus Operator...\n")
Expect(utils.InstallPrometheusOperator()).To(Succeed(), "Failed to install Prometheus Operator")
} else {
_, _ = fmt.Fprintf(GinkgoWriter, "WARNING: Prometheus Operator is already installed. Skipping installation...\n")
}
}
if !skipCertManagerInstall {
By("checking if cert manager is installed already")
isCertManagerAlreadyInstalled = utils.IsCertManagerCRDsInstalled()
if !isCertManagerAlreadyInstalled {
_, _ = fmt.Fprintf(GinkgoWriter, "Installing CertManager...\n")
Expect(utils.InstallCertManager()).To(Succeed(), "Failed to install CertManager")
} else {
_, _ = fmt.Fprintf(GinkgoWriter, "WARNING: CertManager is already installed. Skipping installation...\n")
}
}
})

var _ = AfterSuite(func() {
// Teardown Prometheus and CertManager after the suite if not skipped and if they were not already installed
if !skipPrometheusInstall && !isPrometheusOperatorAlreadyInstalled {
_, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling Prometheus Operator...\n")
utils.UninstallPrometheusOperator()
}
if !skipCertManagerInstall && !isCertManagerAlreadyInstalled {
_, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling CertManager...\n")
utils.UninstallCertManager()
}
})
Loading

0 comments on commit 195a6cf

Please sign in to comment.