From 7563d063b9404fb8975549d4c6cb1f35677bf4ba Mon Sep 17 00:00:00 2001 From: linghan-hub <56351212+linghan-hub@users.noreply.github.com> Date: Mon, 6 Nov 2023 15:42:38 +0800 Subject: [PATCH] chore: abstract e2e test configuration related (#5752) --- .../componentresourceconstraint_custom.yaml} | 0 .../custom_class.yaml} | 0 .../07_cv.yaml => config/postgresql_cv.yaml} | 0 test/e2e/testdata/smoketest/config.go | 69 ++++++++++++++++--- .../smoketest/mariadb/00_mariadbcluster.yaml | 16 ++--- .../testdata/smoketest/milvus/01_vscale.yaml | 6 +- .../{08_upgrade.yaml => 07_upgrade.yaml} | 0 .../{09_restart.yaml => 08_restart.yaml} | 0 ...0_reconfigure.yaml => 09_reconfigure.yaml} | 0 ...ebackup.yaml => 10_backup_basebackup.yaml} | 0 ...yaml => 11_backup_basebackup_restore.yaml} | 2 +- ...ot.yaml => 12_backup_volume_snapshot.yaml} | 0 ...=> 13_backup_volume_snapshot_restore.yaml} | 0 test/e2e/testdata/smoketest/smoketestrun.go | 29 ++++---- .../wesql/{03_vscale.yaml => 01_vscale.yaml} | 0 ...{04_hscale_out.yaml => 02_hscale_out.yaml} | 0 .../{05_hscale_in.yaml => 03_hscale_in.yaml} | 0 .../{06_vexpand.yaml => 04_vexpand.yaml} | 0 .../wesql/{07_stop.yaml => 05_stop.yaml} | 0 .../wesql/{08_start.yaml => 06_start.yaml} | 0 .../{09_restart.yaml => 07_restart.yaml} | 0 ...0_reconfigure.yaml => 08_reconfigure.yaml} | 0 ...abackup.yaml => 09_backup_xtrabackup.yaml} | 0 ...yaml => 10_backup_xtrabackup_restore.yaml} | 2 +- ...ot.yaml => 11_backup_volume_snapshot.yaml} | 0 ...=> 12_backup_volume_snapshot_restore.yaml} | 4 +- 26 files changed, 87 insertions(+), 41 deletions(-) rename test/e2e/testdata/{smoketest/wesql/01_componentresourceconstraint_custom.yaml => config/componentresourceconstraint_custom.yaml} (100%) rename test/e2e/testdata/{smoketest/wesql/02_custom_class.yaml => config/custom_class.yaml} (100%) rename test/e2e/testdata/{smoketest/postgresql/07_cv.yaml => config/postgresql_cv.yaml} (100%) rename test/e2e/testdata/smoketest/postgresql/{08_upgrade.yaml => 07_upgrade.yaml} (100%) rename test/e2e/testdata/smoketest/postgresql/{09_restart.yaml => 08_restart.yaml} (100%) rename test/e2e/testdata/smoketest/postgresql/{10_reconfigure.yaml => 09_reconfigure.yaml} (100%) rename test/e2e/testdata/smoketest/postgresql/{11_backup_basebackup.yaml => 10_backup_basebackup.yaml} (100%) rename test/e2e/testdata/smoketest/postgresql/{12_backup_basebackup_restore.yaml => 11_backup_basebackup_restore.yaml} (96%) rename test/e2e/testdata/smoketest/postgresql/{13_backup_volume_snapshot.yaml => 12_backup_volume_snapshot.yaml} (100%) rename test/e2e/testdata/smoketest/postgresql/{14_backup_volume_snapshot_restore.yaml => 13_backup_volume_snapshot_restore.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{03_vscale.yaml => 01_vscale.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{04_hscale_out.yaml => 02_hscale_out.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{05_hscale_in.yaml => 03_hscale_in.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{06_vexpand.yaml => 04_vexpand.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{07_stop.yaml => 05_stop.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{08_start.yaml => 06_start.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{09_restart.yaml => 07_restart.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{10_reconfigure.yaml => 08_reconfigure.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{11_backup_xtrabackup.yaml => 09_backup_xtrabackup.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{12_backup_xtrabackup_restore.yaml => 10_backup_xtrabackup_restore.yaml} (96%) rename test/e2e/testdata/smoketest/wesql/{13_backup_volume_snapshot.yaml => 11_backup_volume_snapshot.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{14_backup_volume_snapshot_restore.yaml => 12_backup_volume_snapshot_restore.yaml} (91%) diff --git a/test/e2e/testdata/smoketest/wesql/01_componentresourceconstraint_custom.yaml b/test/e2e/testdata/config/componentresourceconstraint_custom.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/01_componentresourceconstraint_custom.yaml rename to test/e2e/testdata/config/componentresourceconstraint_custom.yaml diff --git a/test/e2e/testdata/smoketest/wesql/02_custom_class.yaml b/test/e2e/testdata/config/custom_class.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/02_custom_class.yaml rename to test/e2e/testdata/config/custom_class.yaml diff --git a/test/e2e/testdata/smoketest/postgresql/07_cv.yaml b/test/e2e/testdata/config/postgresql_cv.yaml similarity index 100% rename from test/e2e/testdata/smoketest/postgresql/07_cv.yaml rename to test/e2e/testdata/config/postgresql_cv.yaml diff --git a/test/e2e/testdata/smoketest/config.go b/test/e2e/testdata/smoketest/config.go index 63e85f0d1cc..96b4860f08e 100644 --- a/test/e2e/testdata/smoketest/config.go +++ b/test/e2e/testdata/smoketest/config.go @@ -44,9 +44,9 @@ func Config() { Context("Configure running e2e information", func() { It("create a secret to save the access key and ", func() { - secret := "k get secret " + ConfigType + "-credential-for-backuprepo -n kb-system | grep " + + secret := "kubectl get secret " + ConfigType + "-credential-for-backuprepo -n kb-system | grep " + ConfigType + "-credential-for-backuprepo | awk '{print $1}'" - if len(e2eutil.ExecCommand(secret)) > 0 { + if checkResourceExists(secret) { log.Println("secret " + ConfigType + "-credential-for-backuprepo already exists") } else { var accessKey, secretKey string @@ -67,7 +67,7 @@ func Config() { }) It(" configure backup-repo", func() { repo := "kubectl get BackupRepo | grep my-repo | awk '{print $1}'" - if len(e2eutil.ExecCommand(repo)) > 0 { + if checkResourceExists(repo) { log.Println("BackupRepo already exists") } else { var yaml string @@ -79,7 +79,33 @@ func Config() { b := e2eutil.OpsYaml(yaml, "create") Expect(b).Should(BeTrue()) } - + }) + It(" configure componentresourceconstraint custom", func() { + componentResourceConstraint := "kubectl get ComponentResourceConstraint | grep kb-resource-constraint-e2e | awk '{print $1}'" + if checkResourceExists(componentResourceConstraint) { + log.Println("ComponentResourceConstraint already exists") + } else { + b := e2eutil.OpsYaml(dir+"/testdata/config/componentresourceconstraint_custom.yaml", "create") + Expect(b).Should(BeTrue()) + } + }) + It(" configure custom class", func() { + componentClassDefinition := "kubectl get ComponentClassDefinition | grep custom-class | awk '{print $1}'" + if checkResourceExists(componentClassDefinition) { + log.Println("ComponentClassDefinition already exists") + } else { + b := e2eutil.OpsYaml(dir+"/testdata/config/custom_class.yaml", "create") + Expect(b).Should(BeTrue()) + } + }) + It(" configure pg cluster version", func() { + clusterVersion := "kubectl get ClusterVersion | grep postgresql-14.7.2-latest | awk '{print $1}'" + if checkResourceExists(clusterVersion) { + log.Println("postgresql-14.7.2-latest clusterVersion already exists") + } else { + b := e2eutil.OpsYaml(dir+"/testdata/config/postgresql_cv.yaml", "create") + Expect(b).Should(BeTrue()) + } }) }) } @@ -91,20 +117,41 @@ func DeleteConfig() { AfterEach(func() { }) - Context("delete e2e config resources", func() { + Context("delete e2e test resources", func() { It("Check backup exists ", func() { backupArr := e2eutil.ExecCommandReadline("kubectl get backup | awk '{print $1}'") if len(backupArr) > 0 { - deleteBackups := e2eutil.ExecuteCommand("kubectl delete backup --all") - Expect(deleteBackups).Should(BeTrue()) + deleteResource("kubectl delete backup --all") } }) It("delete secret and backuprepo", func() { - deleteSecret := e2eutil.ExecuteCommand("kubectl delete secret " + ConfigType + "-credential-for-backuprepo -n kb-system") - Expect(deleteSecret).Should(BeTrue()) - deleteBr := e2eutil.ExecuteCommand("kubectl delete backuprepo my-repo") - Expect(deleteBr).Should(BeTrue()) + deleteResource("kubectl delete secret " + ConfigType + "-credential-for-backuprepo -n kb-system") + deleteResource("kubectl delete backuprepo my-repo") + }) + + It("delete resources", func() { + deleteResource("kubectl delete ComponentResourceConstraint kb-resource-constraint-e2e") + deleteResource("kubectl delete ComponentClassDefinition custom-class") }) + It("delete cv", func() { + deleteResource("kubectl delete ClusterVersion postgresql-14.7.2-latest") + }) + + It("delete clusters", func() { + deleteResource("kubectl delete cluster --all") + }) }) } + +func checkResourceExists(command string) bool { + if len(e2eutil.ExecCommand(command)) > 0 { + return true + } + return false +} + +func deleteResource(cmd string) { + deleteCv := e2eutil.ExecuteCommand(cmd) + Expect(deleteCv).Should(BeTrue()) +} diff --git a/test/e2e/testdata/smoketest/mariadb/00_mariadbcluster.yaml b/test/e2e/testdata/smoketest/mariadb/00_mariadbcluster.yaml index a03992ce21d..615d8ff7e20 100644 --- a/test/e2e/testdata/smoketest/mariadb/00_mariadbcluster.yaml +++ b/test/e2e/testdata/smoketest/mariadb/00_mariadbcluster.yaml @@ -1,22 +1,22 @@ --- -# Source: mariadb/templates/rbac.yaml +# Source: mariadb-cluster/templates/rbac.yaml apiVersion: v1 kind: ServiceAccount metadata: name: kb-mariadb-cluster namespace: default labels: - helm.sh/chart: mariadb-0.7.0-alpha.0 + helm.sh/chart: mariadb-cluster-0.7.0-alpha.0 app.kubernetes.io/version: "10.6.15" app.kubernetes.io/instance: mariadb-cluster --- -# Source: mariadb/templates/rbac.yaml +# Source: mariadb-cluster/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: kb-mariadb-cluster labels: - helm.sh/chart: mariadb-0.7.0-alpha.0 + helm.sh/chart: mariadb-cluster-0.7.0-alpha.0 app.kubernetes.io/version: "10.6.15" app.kubernetes.io/instance: mariadb-cluster roleRef: @@ -28,13 +28,13 @@ subjects: name: kb-mariadb-cluster namespace: default --- -# Source: mariadb/templates/rbac.yaml +# Source: mariadb-cluster/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: kb-mariadb-cluster labels: - helm.sh/chart: mariadb-0.7.0-alpha.0 + helm.sh/chart: mariadb-cluster-0.7.0-alpha.0 app.kubernetes.io/version: "10.6.15" app.kubernetes.io/instance: mariadb-cluster roleRef: @@ -46,13 +46,13 @@ subjects: name: kb-mariadb-cluster namespace: default --- -# Source: mariadb/templates/cluster.yaml +# Source: mariadb-cluster/templates/cluster.yaml apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: name: mariadb-cluster labels: - helm.sh/chart: mariadb-0.7.0-alpha.0 + helm.sh/chart: mariadb-cluster-0.7.0-alpha.0 app.kubernetes.io/version: "10.6.15" app.kubernetes.io/instance: mariadb-cluster spec: diff --git a/test/e2e/testdata/smoketest/milvus/01_vscale.yaml b/test/e2e/testdata/smoketest/milvus/01_vscale.yaml index 5c13b64a5d8..2496ef93731 100644 --- a/test/e2e/testdata/smoketest/milvus/01_vscale.yaml +++ b/test/e2e/testdata/smoketest/milvus/01_vscale.yaml @@ -7,6 +7,6 @@ spec: type: VerticalScaling verticalScaling: - componentName: milvus - requests: - cpu: "500m" - memory: 500Mi \ No newline at end of file + requests: + cpu: "500m" + memory: 500Mi \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/postgresql/08_upgrade.yaml b/test/e2e/testdata/smoketest/postgresql/07_upgrade.yaml similarity index 100% rename from test/e2e/testdata/smoketest/postgresql/08_upgrade.yaml rename to test/e2e/testdata/smoketest/postgresql/07_upgrade.yaml diff --git a/test/e2e/testdata/smoketest/postgresql/09_restart.yaml b/test/e2e/testdata/smoketest/postgresql/08_restart.yaml similarity index 100% rename from test/e2e/testdata/smoketest/postgresql/09_restart.yaml rename to test/e2e/testdata/smoketest/postgresql/08_restart.yaml diff --git a/test/e2e/testdata/smoketest/postgresql/10_reconfigure.yaml b/test/e2e/testdata/smoketest/postgresql/09_reconfigure.yaml similarity index 100% rename from test/e2e/testdata/smoketest/postgresql/10_reconfigure.yaml rename to test/e2e/testdata/smoketest/postgresql/09_reconfigure.yaml diff --git a/test/e2e/testdata/smoketest/postgresql/11_backup_basebackup.yaml b/test/e2e/testdata/smoketest/postgresql/10_backup_basebackup.yaml similarity index 100% rename from test/e2e/testdata/smoketest/postgresql/11_backup_basebackup.yaml rename to test/e2e/testdata/smoketest/postgresql/10_backup_basebackup.yaml diff --git a/test/e2e/testdata/smoketest/postgresql/12_backup_basebackup_restore.yaml b/test/e2e/testdata/smoketest/postgresql/11_backup_basebackup_restore.yaml similarity index 96% rename from test/e2e/testdata/smoketest/postgresql/12_backup_basebackup_restore.yaml rename to test/e2e/testdata/smoketest/postgresql/11_backup_basebackup_restore.yaml index 89bf37bf5dc..3617a399eb2 100644 --- a/test/e2e/testdata/smoketest/postgresql/12_backup_basebackup_restore.yaml +++ b/test/e2e/testdata/smoketest/postgresql/11_backup_basebackup_restore.yaml @@ -3,7 +3,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: pg-cluster-snapshot + name: pg-cluster-restore annotations: kubeblocks.io/restore-from-backup: '{"postgresql":{"name":"backup-postgresql-cluster","namespace":"default","volumeRestorePolicy":"Parallel"}}' spec: diff --git a/test/e2e/testdata/smoketest/postgresql/13_backup_volume_snapshot.yaml b/test/e2e/testdata/smoketest/postgresql/12_backup_volume_snapshot.yaml similarity index 100% rename from test/e2e/testdata/smoketest/postgresql/13_backup_volume_snapshot.yaml rename to test/e2e/testdata/smoketest/postgresql/12_backup_volume_snapshot.yaml diff --git a/test/e2e/testdata/smoketest/postgresql/14_backup_volume_snapshot_restore.yaml b/test/e2e/testdata/smoketest/postgresql/13_backup_volume_snapshot_restore.yaml similarity index 100% rename from test/e2e/testdata/smoketest/postgresql/14_backup_volume_snapshot_restore.yaml rename to test/e2e/testdata/smoketest/postgresql/13_backup_volume_snapshot_restore.yaml diff --git a/test/e2e/testdata/smoketest/smoketestrun.go b/test/e2e/testdata/smoketest/smoketestrun.go index acb21898d0b..29c2d958f5b 100644 --- a/test/e2e/testdata/smoketest/smoketestrun.go +++ b/test/e2e/testdata/smoketest/smoketestrun.go @@ -46,8 +46,6 @@ type Options struct { Dynamic dynamic.Interface } -var arr = []string{"00", "componentresourceconstraint", "restore", "class", "cv", "backup"} - func SmokeTest() { BeforeEach(func() { }) @@ -159,6 +157,20 @@ func runTestCases(files []string) { } } } + if strings.Contains(file, "restore") { + backups := "kubectl get backup | awk '{print $1}' | tail -n +2" + log.Println(backups) + backupList := e2eutil.ExecCommandReadline(backups) + log.Println(backupList) + Eventually(func(g Gomega) { + for _, backup := range backupList { + cmd := "kubectl get backup " + backup + " -o=jsonpath='{.status.phase}'" + log.Println(cmd) + log.Println(e2eutil.ExecCommand(cmd)) + g.Expect(e2eutil.ExecCommand(cmd)).Should(Equal("Completed")) + } + }, timeout, interval).Should(Succeed()) + } b := e2eutil.OpsYaml(file, "create") if strings.Contains(file, "00") || strings.Contains(file, "restore") { clusterName, nameSpace = e2eutil.GetName(file) @@ -197,11 +209,6 @@ func runTestCases(files []string) { TestResults = append(TestResults, e2eResult) } } - if len(files) > 0 { - for _, file := range files { - deleteResource(file) - } - } } func troubleShooting(clusterName string) string { @@ -211,11 +218,3 @@ func troubleShooting(clusterName string) string { clusterEvents := e2eutil.ExecCommand(commond) return allResourceStatus + clusterEvents } - -func deleteResource(file string) { - for _, s := range arr { - if strings.Contains(file, s) { - e2eutil.OpsYaml(file, "delete") - } - } -} diff --git a/test/e2e/testdata/smoketest/wesql/03_vscale.yaml b/test/e2e/testdata/smoketest/wesql/01_vscale.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/03_vscale.yaml rename to test/e2e/testdata/smoketest/wesql/01_vscale.yaml diff --git a/test/e2e/testdata/smoketest/wesql/04_hscale_out.yaml b/test/e2e/testdata/smoketest/wesql/02_hscale_out.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/04_hscale_out.yaml rename to test/e2e/testdata/smoketest/wesql/02_hscale_out.yaml diff --git a/test/e2e/testdata/smoketest/wesql/05_hscale_in.yaml b/test/e2e/testdata/smoketest/wesql/03_hscale_in.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/05_hscale_in.yaml rename to test/e2e/testdata/smoketest/wesql/03_hscale_in.yaml diff --git a/test/e2e/testdata/smoketest/wesql/06_vexpand.yaml b/test/e2e/testdata/smoketest/wesql/04_vexpand.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/06_vexpand.yaml rename to test/e2e/testdata/smoketest/wesql/04_vexpand.yaml diff --git a/test/e2e/testdata/smoketest/wesql/07_stop.yaml b/test/e2e/testdata/smoketest/wesql/05_stop.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/07_stop.yaml rename to test/e2e/testdata/smoketest/wesql/05_stop.yaml diff --git a/test/e2e/testdata/smoketest/wesql/08_start.yaml b/test/e2e/testdata/smoketest/wesql/06_start.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/08_start.yaml rename to test/e2e/testdata/smoketest/wesql/06_start.yaml diff --git a/test/e2e/testdata/smoketest/wesql/09_restart.yaml b/test/e2e/testdata/smoketest/wesql/07_restart.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/09_restart.yaml rename to test/e2e/testdata/smoketest/wesql/07_restart.yaml diff --git a/test/e2e/testdata/smoketest/wesql/10_reconfigure.yaml b/test/e2e/testdata/smoketest/wesql/08_reconfigure.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/10_reconfigure.yaml rename to test/e2e/testdata/smoketest/wesql/08_reconfigure.yaml diff --git a/test/e2e/testdata/smoketest/wesql/11_backup_xtrabackup.yaml b/test/e2e/testdata/smoketest/wesql/09_backup_xtrabackup.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/11_backup_xtrabackup.yaml rename to test/e2e/testdata/smoketest/wesql/09_backup_xtrabackup.yaml diff --git a/test/e2e/testdata/smoketest/wesql/12_backup_xtrabackup_restore.yaml b/test/e2e/testdata/smoketest/wesql/10_backup_xtrabackup_restore.yaml similarity index 96% rename from test/e2e/testdata/smoketest/wesql/12_backup_xtrabackup_restore.yaml rename to test/e2e/testdata/smoketest/wesql/10_backup_xtrabackup_restore.yaml index ef2aaac777d..a87529978c5 100644 --- a/test/e2e/testdata/smoketest/wesql/12_backup_xtrabackup_restore.yaml +++ b/test/e2e/testdata/smoketest/wesql/10_backup_xtrabackup_restore.yaml @@ -23,4 +23,4 @@ spec: - ReadWriteOnce resources: requests: - storage: 20Gi + storage: 21Gi diff --git a/test/e2e/testdata/smoketest/wesql/13_backup_volume_snapshot.yaml b/test/e2e/testdata/smoketest/wesql/11_backup_volume_snapshot.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/13_backup_volume_snapshot.yaml rename to test/e2e/testdata/smoketest/wesql/11_backup_volume_snapshot.yaml diff --git a/test/e2e/testdata/smoketest/wesql/14_backup_volume_snapshot_restore.yaml b/test/e2e/testdata/smoketest/wesql/12_backup_volume_snapshot_restore.yaml similarity index 91% rename from test/e2e/testdata/smoketest/wesql/14_backup_volume_snapshot_restore.yaml rename to test/e2e/testdata/smoketest/wesql/12_backup_volume_snapshot_restore.yaml index 8b9e1e56ed2..e1bff802924 100644 --- a/test/e2e/testdata/smoketest/wesql/14_backup_volume_snapshot_restore.yaml +++ b/test/e2e/testdata/smoketest/wesql/12_backup_volume_snapshot_restore.yaml @@ -3,7 +3,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: mysql-cluster-restore + name: mysql-cluster-restore-snapshot annotations: kubeblocks.io/restore-from-backup: '{"mysql":{"name":"backup-mysql-cluster-snapshot","namespace":"default","volumeRestorePolicy":"Parallel"}}' spec: @@ -23,4 +23,4 @@ spec: - ReadWriteOnce resources: requests: - storage: 20Gi + storage: 25Gi