[WIP] test: create a test backup

.circleci/config.yml

@@ -54,8 +54,10 @@ jobs:
             mc mb minio/sensu-backup-test
       - run:
           name: Run e2e tests
+          no_output_timeout: 1200
           command: |
             export SENSU_API_URL="http://127.0.0.1:31180"
+            export S3_ADDR="127.0.0.1:31234"
             KUBECONFIG=~/.kube/config \
             OPERATOR_IMAGE=sensu/sensu-operator:test \
             TEST_NAMESPACE=default \

README.md

@@ -196,6 +196,7 @@ $ minikube start --kubernetes-version v1.10.0
 $ eval $(minikube docker-env)
 $ make
 $ ./example/rbac/create-role
+$ ./test/deploy-minio
 $ KUBECONFIG=~/.kube/config \
   OPERATOR_IMAGE=sensu/sensu-operator:v0.0.1 \
   TEST_NAMESPACE=default \

pkg/apis/sensu/v1beta1/backup_types.go

@@ -118,6 +118,9 @@ type S3BackupSource struct {
 	// Endpoint if blank points to aws. If specified, can point to s3 compatible object
 	// stores.
 	Endpoint string `json:"endpoint,omitempty"`
+
+	ForcePathStyle bool
+	DisableSSL     bool
 }
 
 // ABSBackupSource provides the spec how to store backups on ABS.

pkg/apis/sensu/v1beta1/restore_types.go

@@ -85,6 +85,9 @@ type S3RestoreSource struct {
 	// Endpoint if blank points to aws. If specified, can point to s3 compatible object
 	// stores.
 	Endpoint string `json:"endpoint"`
+
+	ForcePathStyle bool
+	DisableSSL     bool
 }
 
 type ABSRestoreSource struct {

pkg/controller/backup-operator/s3_backup.go

@@ -31,7 +31,7 @@ import (
 // handleS3 saves etcd cluster's backup to specificed S3 path.
 func handleS3(ctx context.Context, kubecli kubernetes.Interface, s *api.S3BackupSource, endpoints []string, clientTLSSecret, namespace string) (*api.BackupStatus, error) {
 	// TODO: controls NewClientFromSecret with ctx. This depends on upstream kubernetes to support API calls with ctx.
-	cli, err := s3factory.NewClientFromSecret(kubecli, namespace, s.Endpoint, s.AWSSecret)
+	cli, err := s3factory.NewClientFromSecret(kubecli, namespace, s.Endpoint, s.AWSSecret, s.ForcePathStyle, s.DisableSSL)
 	if err != nil {
 		return nil, err
 	}

pkg/controller/restore-operator/http.go

@@ -91,7 +91,7 @@ func (r *Restore) serveBackup(w http.ResponseWriter, req *http.Request) error {
 			return errors.New("invalid s3 restore source field (spec.s3), must specify all required subfields")
 		}
 
-		s3Cli, err := s3factory.NewClientFromSecret(r.kubecli, r.namespace, s3RestoreSource.Endpoint, s3RestoreSource.AWSSecret)
+		s3Cli, err := s3factory.NewClientFromSecret(r.kubecli, r.namespace, s3RestoreSource.Endpoint, s3RestoreSource.AWSSecret, s3RestoreSource.ForcePathStyle, s3RestoreSource.DisableSSL)
 		if err != nil {
 			return fmt.Errorf("failed to create S3 client: %v", err)
 		}

pkg/util/awsutil/s3factory/client.go

@@ -39,7 +39,7 @@ type S3Client struct {
 }
 
 // NewClientFromSecret returns a S3 client based on given k8s secret containing aws credentials.
-func NewClientFromSecret(kubecli kubernetes.Interface, namespace, endpoint, awsSecret string) (w *S3Client, err error) {
+func NewClientFromSecret(kubecli kubernetes.Interface, namespace, endpoint, awsSecret string, forcePathStyle, disableSSL bool) (w *S3Client, err error) {
 	defer func() {
 		if err != nil {
 			err = fmt.Errorf("new S3 client failed: %v", err)
@@ -50,7 +50,7 @@ func NewClientFromSecret(kubecli kubernetes.Interface, namespace, endpoint, awsS
 	if err != nil {
 		return nil, fmt.Errorf("failed to create aws config dir: (%v)", err)
 	}
-	so, err := setupAWSConfig(kubecli, namespace, awsSecret, endpoint, w.configDir)
+	so, err := setupAWSConfig(kubecli, namespace, awsSecret, endpoint, w.configDir, forcePathStyle, disableSSL)
 	if err != nil {
 		return nil, fmt.Errorf("failed to setup aws config: (%v)", err)
 	}
@@ -68,10 +68,12 @@ func (w *S3Client) Close() {
 }
 
 // setupAWSConfig setup local AWS config/credential files from Kubernetes aws secret.
-func setupAWSConfig(kubecli kubernetes.Interface, ns, secret, endpoint, configDir string) (*session.Options, error) {
+func setupAWSConfig(kubecli kubernetes.Interface, ns, secret, endpoint, configDir string, forcePathStyle, disableSSL bool) (*session.Options, error) {
 	options := &session.Options{}
 	options.SharedConfigState = session.SharedConfigEnable
 
+	options.Config.DisableSSL = &disableSSL
+	options.Config.S3ForcePathStyle = &forcePathStyle
 	// empty string defaults to aws
 	options.Config.Endpoint = &endpoint
 

test/e2e/basic_test.go

@@ -21,34 +21,40 @@ import (
 	"time"
 
 	"github.com/sensu/sensu-operator/pkg/util/k8sutil"
+	"github.com/sensu/sensu-operator/pkg/util/retryutil"
 	"github.com/sensu/sensu-operator/test/e2e/e2eutil"
 	"github.com/sensu/sensu-operator/test/e2e/framework"
+
+	"github.com/minio/minio-go"
+	"k8s.io/apimachinery/pkg/api/errors"
 )
 
 func TestCreateCluster(t *testing.T) {
 	if os.Getenv(envParallelTest) == envParallelTestTrue {
 		t.Parallel()
 	}
 	f := framework.Global
-	testSensu, err := e2eutil.CreateCluster(t, f.CRClient, f.Namespace, e2eutil.NewCluster("test-sensu-", 3))
+
+	clusterSize := 3
+	sensuCluster, err := e2eutil.CreateCluster(t, f.CRClient, f.Namespace, e2eutil.NewCluster("test-sensu-", clusterSize))
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	defer func() {
-		if err := e2eutil.DeleteCluster(t, f.CRClient, f.KubeClient, testSensu); err != nil {
+		if err := e2eutil.DeleteCluster(t, f.CRClient, f.KubeClient, sensuCluster); err != nil {
 			t.Fatal(err)
 		}
 	}()
 
-	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, testSensu); err != nil {
-		t.Fatalf("failed to create 3 members sensu cluster: %v", err)
+	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, clusterSize, 20, sensuCluster); err != nil {
+		t.Fatalf("failed to create %d members sensu cluster: %v", clusterSize, err)
 	}
 
-	testSensuName := testSensu.ObjectMeta.Name
+	sensuClusterName := sensuCluster.ObjectMeta.Name
 
-	sensuNodePortServiceName := fmt.Sprintf("%s-api-external", testSensuName)
-	sensuNodePortService := e2eutil.NewAPINodePortService(testSensuName, sensuNodePortServiceName)
+	sensuNodePortServiceName := fmt.Sprintf("%s-api-external", sensuClusterName)
+	sensuNodePortService := e2eutil.NewAPINodePortService(sensuClusterName, sensuNodePortServiceName)
 
 	if _, err := f.KubeClient.CoreV1().Services("default").Create(sensuNodePortService); err != nil {
 		t.Fatalf("failed to create API service of type node port: %v", err)
@@ -59,14 +65,14 @@ func TestCreateCluster(t *testing.T) {
 		}
 	}()
 
-	dummyDeployment := e2eutil.NewDummyDeployment(testSensuName)
+	dummyDeployment := e2eutil.NewDummyDeployment(sensuClusterName)
 	dummyDeployment, err = k8sutil.CreateAndWaitDeployment(f.KubeClient, f.Namespace, dummyDeployment, 60*time.Second)
 	if err != nil {
 		t.Fatalf("failed to create dummy deployment: %v", err)
 	}
 	defer func() {
 		if err := e2eutil.DeleteDummyDeployment(f.KubeClient, "default", dummyDeployment.ObjectMeta.Name); err != nil {
-			t.Fatal(err)
+			t.Logf("failed to delete dummy deployment: %v", err)
 		}
 	}()
 
@@ -93,7 +99,7 @@ func TestCreateCluster(t *testing.T) {
 		t.Fatalf("failed to get cluster member list: %v", err)
 	}
 	clusterMembers := clusterMemberList.Members
-	if len(clusterMembers) != 3 {
+	if len(clusterMembers) != clusterSize {
 		t.Fatalf("expected to find three cluster members but found %d", len(clusterMembers))
 	}
 
@@ -106,4 +112,152 @@ func TestCreateCluster(t *testing.T) {
 			t.Fatalf("not all cluster members are healthy: %+v", clusterHealth)
 		}
 	}
+
+	s3Addr := os.Getenv("S3_ADDR")
+	if s3Addr == "" {
+		s3Addr = "192.168.99.100:31234"
+	}
+
+	minioClient, err := minio.New(s3Addr, "admin", "password", false)
+	if err != nil {
+		t.Fatalf("failed to initialize minio client: %v", err)
+	}
+
+	exists, err := minioClient.BucketExists("sensu-backup-test")
+	if err != nil {
+		t.Fatalf("failed to check if bucket exists: %v", err)
+	}
+	if !exists {
+		if err := minioClient.MakeBucket("sensu-backup-test", "eu-central-1"); err != nil {
+			t.Fatalf("failed to create bucket: %v", err)
+		}
+	}
+
+	sensuBackup := e2eutil.NewSensuBackup(sensuClusterName, "sensu-test-backup")
+	_, err = f.CRClient.SensuV1beta1().SensuBackups("default").Create(sensuBackup)
+	if err != nil {
+		t.Fatalf("failed to create sensu backup: %v", err)
+	}
+	defer func() {
+		if err := f.CRClient.SensuV1beta1().SensuBackups("default").Delete(sensuBackup.ObjectMeta.Name, nil); err != nil {
+			t.Fatalf("failed to delete test backup: %v", err)
+		}
+	}()
+
+	err = retryutil.Retry(1*time.Second, 10, func() (bool, error) {
+		_, err = minioClient.StatObject("sensu-backup-test", "sensu-test-backup", minio.StatObjectOptions{})
+		if err != nil {
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		t.Fatalf("failed to stat backup in 10 seconds: %v", err)
+	}
+
+	// Delete the old cluster and agents to start a new, empty
+	// cluster afterwards where we apply the backup.
+
+	if err := e2eutil.DeleteDummyDeployment(f.KubeClient, "default", dummyDeployment.ObjectMeta.Name); err != nil {
+		t.Fatalf("failed to delete dummy deployment: %v", err)
+	}
+	if err := e2eutil.DeleteCluster(t, f.CRClient, f.KubeClient, sensuCluster); err != nil {
+		t.Fatalf("failed to delete sensu cluster in preperation for restore: %v", err)
+	}
+
+	sensuCluster, err = e2eutil.CreateCluster(t, f.CRClient, f.Namespace, e2eutil.NewCluster("test-sensu-", clusterSize))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sensuClusterPods, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, clusterSize, 20, sensuCluster)
+	if err != nil {
+		t.Fatalf("failed to create %d members sensu cluster: %v", clusterSize, err)
+	}
+
+	sensuClusterName = sensuCluster.ObjectMeta.Name
+
+	// Renew the client for the new cluster
+	sensuClient, err = e2eutil.NewSensuClient(apiURL)
+	if err != nil {
+		t.Fatalf("failed to initialize sensu client: %v", err)
+	}
+
+	entities, err = sensuClient.ListEntities("default")
+	if err != nil {
+		t.Fatalf("failed to list entities: %v", err)
+	}
+	// There should be no entities: it's a new cluster and no
+	// agents are connected
+	if len(entities) != 0 {
+		t.Fatalf("expected to find zero entities but found %d", len(entities))
+	}
+
+	// Now apply the backup ...
+
+	t.Log("Restoring cluster from backup")
+
+	sensuRestore := e2eutil.NewSensuRestore(sensuClusterName, "sensu-test-backup")
+	_, err = f.CRClient.SensuV1beta1().SensuRestores("default").Create(sensuRestore)
+	if err != nil {
+		t.Fatalf("failed to create sensu restore: %v", err)
+	}
+	defer func() {
+		if err := f.CRClient.SensuV1beta1().SensuRestores("default").Delete(sensuRestore.ObjectMeta.Name, nil); err != nil {
+			t.Fatalf("failed to delete test backup: %v", err)
+		}
+	}()
+
+	// Restoring a backup means a new pods are started, i.e. we
+	// have to wait until the old members are gone and the new are up
+	remainingPods, err := e2eutil.WaitUntilMembersWithNamesDeleted(t, f.CRClient, 12, sensuCluster, sensuClusterPods...)
+	if err != nil {
+		statusError, ok := err.(*errors.StatusError)
+		// If the cluster is not found (404), its members are
+		// deleted already and we can continue
+		if !ok || statusError.ErrStatus.Code != 404 {
+			t.Fatalf("failed to see members (%v) be deleted in time: %v", remainingPods, err)
+		}
+	}
+
+	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, clusterSize, 20, sensuCluster); err != nil {
+		t.Fatalf("failed to create %d members sensu cluster: %v", clusterSize, err)
+	}
+
+	// Renew the client for the new cluster
+	sensuClient, err = e2eutil.NewSensuClient(apiURL)
+	if err != nil {
+		t.Fatalf("failed to initialize sensu client: %v", err)
+	}
+
+	clusterMemberList, err = sensuClient.MemberList()
+	if err != nil {
+		t.Fatalf("failed to get cluster member list: %v", err)
+	}
+	clusterMembers = clusterMemberList.Members
+	if len(clusterMembers) != clusterSize {
+		t.Fatalf("expected to find three cluster members but found %d", len(clusterMembers))
+	}
+
+	clusterHealth, err = sensuClient.Health()
+	if err != nil {
+		t.Fatalf("failed to get cluster health: %v", err)
+	}
+	for _, memberHealth := range clusterHealth {
+		if !memberHealth.Healthy {
+			t.Fatalf("not all cluster members are healthy: %+v", clusterHealth)
+		}
+	}
+
+	// ... and check the list of entities again. Since two entities
+	// were registered at the time we took the backup, we should
+	// find two
+
+	entities, err = sensuClient.ListEntities("default")
+	if err != nil {
+		t.Fatalf("failed to list entities: %v", err)
+	}
+	if len(entities) != 2 {
+		t.Fatalf("expected to find two entities after restore but found %d", len(entities))
+	}
 }