Make mongodb feature_tests compatible for any db tests (wal-g#983)

* Moved tests_func/mognoload -> tests_func/mongodb/mongoload

* Moved tests_func/config -> tests_func/mongodb/config

* Remove useless mognodb.conf in base test image

* rename docker file and docker-compose

* Rename features, add feature discovery, simplified code

* Add support of tf.env for tests
Refactor: moved code that search FEATURE from env to right place

.github/workflows/dockertests.yml

@@ -101,8 +101,8 @@ jobs:
                    'make mongo_test',
                    'make MONGO_VERSION="4.4.3" MONGO_MAJOR="4.4" mongo_features',
                    'make MONGO_VERSION="4.2.12" MONGO_MAJOR="4.2" mongo_features',
-                   'make MONGO_FEATURE="backup" MONGO_VERSION="4.0.22" MONGO_MAJOR="4.0" mongo_features',
-                   'make MONGO_FEATURE="backup" MONGO_VERSION="3.6.21" MONGO_MAJOR="3.6" mongo_features',
+                   'make FEATURE="backup" MONGO_VERSION="4.0.22" MONGO_MAJOR="4.0" mongo_features',
+                   'make FEATURE="backup" MONGO_VERSION="3.6.21" MONGO_MAJOR="3.6" mongo_features',
                    'make redis_test',
                    'make TEST="pg_wale_tablespace_compatibility_test" pg_integration_test',
                    'make TEST="pg_tablespace_support_test" pg_integration_test',

.goimportsignore

@@ -0,0 +1 @@
+tests_func/staging

Makefile

@@ -119,11 +119,11 @@ mongo_install: mongo_build
 mongo_features:
 	set -e
 	make go_deps
-	cd tests_func/ && MONGO_MAJOR=$(MONGO_MAJOR) MONGO_VERSION=$(MONGO_VERSION) go test -v -count=1 -timeout 20m  -tf.test=true -tf.debug=false -tf.clean=true -tf.stop=true
+	cd tests_func/ && MONGO_MAJOR=$(MONGO_MAJOR) MONGO_VERSION=$(MONGO_VERSION) go test -v -count=1 -timeout 20m  -tf.test=true -tf.debug=false -tf.clean=true -tf.stop=true -tf.database=mongodb
 
 clean_mongo_features:
 	set -e
-	cd tests_func/ && MONGO_MAJOR=$(MONGO_MAJOR) MONGO_VERSION=$(MONGO_VERSION) go test -v -count=1  -timeout 5m -tf.test=false -tf.debug=false -tf.clean=true -tf.stop=true
+	cd tests_func/ && MONGO_MAJOR=$(MONGO_MAJOR) MONGO_VERSION=$(MONGO_VERSION) go test -v -count=1  -timeout 5m -tf.test=false -tf.debug=false -tf.clean=true -tf.stop=true -tf.database=mongodb
 
 fdb_build: $(CMD_FILES) $(PKG_FILES)
 	(cd $(MAIN_FDB_PATH) && go build -mod vendor -tags "$(BUILD_TAGS)" -o wal-g -ldflags "-s -w")

tests_func/config/config.go → tests_func/config.go

@@ -1,16 +1,14 @@
-package config
+package functests
 
 var Env = map[string]string{
-	"STAGING_DIR":        "staging",
-	"ENV_FILE":           "staging/env.file",
 	"IMAGES_DIR":         "images",
 	"DOCKER_BRIDGE_NAME": "walg-func-test",
 	"DOCKER_IP4_SUBNET":  "10.%s.0/24",
 	"DOCKER_IP6_SUBNET":  "fd00:dead:beef:%s::/96",
 
-	"COMPOSE_FILE":       "docker-compose.yml",
-	"TEST_ID":            "13",
-	"TEST_CLEANUP_DELAY": "60",
+	"COMPOSE_FILE_SUFFIX": "-docker-compose.yml",
+	"TEST_ID":             "13",
+	"TEST_CLEANUP_DELAY":  "60",
 
 	"WALG_S3_PREFIX":   "s3://dbaas/mongodb-backup/test_uuid/test_mongodb",
 	"WALG_CLIENT_PATH": "/usr/bin/wal-g",

tests_func/env.go

@@ -7,37 +7,46 @@ import (
 	"path"
 	"strconv"
 
-	"github.com/wal-g/wal-g/tests_func/config"
 	"github.com/wal-g/wal-g/tests_func/utils"
 )
 
+const (
+	EnvDirPerm os.FileMode = 0755
+	EnvFilePerv os.FileMode = 0644
+)
+
 func EnvExists(path string) bool {
 	_, err := os.Stat(path)
 	return err == nil
 }
 
-func SetupEnv(envFilePath, stagingDir string) error {
-	if err := os.Mkdir(stagingDir, 0755); err != nil {
-		return fmt.Errorf("can not create staging dir: %v", err)
+func SetupEnv(fromEnv map[string]string, envFilePath, stagingDir string) (map[string]string, error) {
+	if _, err := os.Stat(stagingDir); err == nil {
+		err = os.Chmod(stagingDir, EnvDirPerm)
+		if err != nil {
+			return nil, fmt.Errorf("can not chmod staging dir: %v", err)
+		}
+	} else if err := os.Mkdir(stagingDir, EnvDirPerm); err != nil {
+		return nil, fmt.Errorf("can not create staging dir: %v", err)
 	}
-	env := utils.MergeEnvs(config.Env, DynConf(config.Env))
-	file, err := os.OpenFile(envFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	env := utils.MergeEnvs(fromEnv, DynConf(fromEnv))
+	file, err := os.OpenFile(envFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, EnvFilePerv)
 	if err != nil {
-		return fmt.Errorf("can not open env file for writing: %v", err)
+		return nil, fmt.Errorf("can not open database file for writing: %v", err)
 	}
 	defer func() { _ = file.Close() }()
 
 	if err := utils.WriteEnv(env, file); err != nil {
-		return fmt.Errorf("can not write to env file: %v", err)
+		return nil, fmt.Errorf("can not write to database file: %v", err)
 	}
 
-	return nil
+	return env, nil
 }
 
 func ReadEnv(path string) (map[string]string, error) {
-	file, err := os.OpenFile(path, os.O_RDONLY, 0644)
+	file, err := os.OpenFile(path, os.O_RDONLY, EnvFilePerv)
 	if err != nil {
-		return nil, fmt.Errorf("can not open env file: %v", err)
+		return nil, fmt.Errorf("can not open database file: %v", err)
 	}
 	defer func() { _ = file.Close() }()
 	envLines, err := utils.ReadLines(file)

tests_func/godogs_test.go

@@ -3,23 +3,28 @@ package functests
 import (
 	"flag"
 	"os"
+	"path"
 	"testing"
 
 	"github.com/DATA-DOG/godog"
 	"github.com/DATA-DOG/godog/colors"
 	"github.com/wal-g/tracelog"
-	"github.com/wal-g/wal-g/tests_func/config"
 )
 
 type TestOpts struct {
-	test  bool
-	clean bool
-	stop  bool
-	debug bool
+	test          bool
+	clean         bool
+	stop          bool
+	debug         bool
+	featurePrefix string
+	database      string
 }
 
 const (
 	testOptsPrefix = "tf."
+
+	stagingDir = "staging"
+	envFile    = "env.file"
 )
 
 var (
@@ -31,13 +36,20 @@ var (
 	}
 
 	testOpts = TestOpts{}
+
+	databases = map[string]bool{
+		"mongodb": true,
+		"redis":   true,
+	}
 )
 
 func init() {
 	flag.BoolVar(&testOpts.test, testOptsPrefix+"test", true, "run tests")
 	flag.BoolVar(&testOpts.stop, testOptsPrefix+"stop", true, "shutdown test environment")
 	flag.BoolVar(&testOpts.clean, testOptsPrefix+"clean", true, "delete test environment")
 	flag.BoolVar(&testOpts.debug, testOptsPrefix+"debug", false, "enable debug logging")
+	flag.StringVar(&testOpts.featurePrefix, testOptsPrefix+"featurePrefix", "", "features prefix")
+	flag.StringVar(&testOpts.database, testOptsPrefix+"database", "", "database name [mongodb|redis]")
 	godog.BindFlags("godog.", flag.CommandLine, &godogOpts)
 }
 
@@ -50,11 +62,45 @@ func TestMain(m *testing.M) {
 		tracelog.ErrorLogger.FatalOnError(err)
 	}
 
-	envFilePath := config.Env["ENV_FILE"]
+	database := testOpts.database
+
+	if _, ok := databases[database]; !ok {
+		tracelog.ErrorLogger.Fatalf("Database '%s' is not valid, please provide test database -tf.database=dbname\n", database)
+	}
+
+	stagingPath := stagingDir
+	envFilePath := path.Join(stagingDir, envFile)
+
+	Env["ENV_FILE"] = envFilePath // set ENV_FILE for docker-compose
+	Env["DOCKER_FILE"] = "Dockerfile." + database
+	Env["COMPOSE_FILE"] = database + Env["COMPOSE_FILE_SUFFIX"]
+
+	newEnv := !EnvExists(envFilePath)
+
+	var env map[string]string
+	var err error
+
+	if newEnv {
+		env, err = SetupEnv(Env, envFilePath, stagingPath)
+		tracelog.ErrorLogger.FatalOnError(err)
+
+		err := SetupStaging(env["IMAGES_DIR"], stagingPath)
+		tracelog.ErrorLogger.FatalOnError(err)
+	}
+
+	foundFeatures, err := scanFeatureDirs(database, testOpts.featurePrefix)
+	tracelog.ErrorLogger.FatalOnError(err)
+
+	if len(foundFeatures) == 0 {
+		tracelog.ErrorLogger.Fatalln("No features found")
+	}
 
-	tctx, err := NewTestContext()
+	tctx, err := NewTestContext(envFilePath, database, env, foundFeatures)
 	tracelog.ErrorLogger.FatalOnError(err)
 
+	// if newEnv == false, database empty, so we should load from database file
+	tctx.LoadEnv()
+
 	if testOpts.test {
 		godogOpts.Paths = tctx.Features
 
@@ -70,16 +116,6 @@ func TestMain(m *testing.M) {
 		}
 	}
 
-	if !EnvExists(envFilePath) {
-		tracelog.ErrorLogger.Fatalln("Environment file is not found: ", envFilePath)
-	}
-
-	env, err := ReadEnv(envFilePath)
-	tracelog.ErrorLogger.FatalOnError(err)
-
-	tctx.Env = env
-	tctx.Infra = InfraFromTestContext(tctx)
-
 	if testOpts.stop {
 		err = tctx.StopEnv()
 		tracelog.ErrorLogger.FatalOnError(err)

tests_func/docker-compose.yml → tests_func/mongodb-docker-compose.yml

@@ -20,6 +20,7 @@ services:
             - ${ENV_FILE}
     mongodb01:
         build:
+            dockerfile: ${DOCKER_FILE}
             context: .
             args:
                 - MONGO_MAJOR=${MONGO_MAJOR}
@@ -42,6 +43,7 @@ services:
             - ${ENV_FILE}
     mongodb02:
         build:
+            dockerfile: ${DOCKER_FILE}
             context: .
             args:
                 - MONGO_MAJOR=${MONGO_MAJOR}

tests_func/mongoload/cmd/ammogenerator/ammo_generator.go → tests_func/mongodb/mongoload/cmd/ammogenerator/ammo_generator.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/wal-g/tracelog"
-	"github.com/wal-g/wal-g/tests_func/mongoload/internal"
+	"github.com/wal-g/wal-g/tests_func/mongodb/mongoload/internal"
 )
 
 var Cmd = &cobra.Command{

tests_func/mongoload/cmd/loader/loader.go → tests_func/mongodb/mongoload/cmd/loader/loader.go

@@ -8,8 +8,8 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/wal-g/tracelog"
-	"github.com/wal-g/wal-g/tests_func/mongoload"
-	"github.com/wal-g/wal-g/tests_func/mongoload/internal"
+	"github.com/wal-g/wal-g/tests_func/mongodb/mongoload"
+	"github.com/wal-g/wal-g/tests_func/mongodb/mongoload/internal"
 )
 
 var ammoFile string

tests_func/mongoload/internal/stats.go → tests_func/mongodb/mongoload/internal/stats.go

@@ -4,7 +4,7 @@ import (
 	"encoding/json"
 	"io"
 
-	"github.com/wal-g/wal-g/tests_func/mongoload/models"
+	"github.com/wal-g/wal-g/tests_func/mongodb/mongoload/models"
 )
 
 func updateStatWithMongoOpLog(stat *models.OpsStat, log OpInfo) {

tests_func/mongoload/load.go → tests_func/mongodb/mongoload/load.go

@@ -6,9 +6,10 @@ import (
 	"io"
 	"sync"
 
-	"github.com/wal-g/wal-g/tests_func/mongoload/internal"
-	"github.com/wal-g/wal-g/tests_func/mongoload/models"
 	"go.mongodb.org/mongo-driver/mongo"
+
+	"github.com/wal-g/wal-g/tests_func/mongodb/mongoload/internal"
+	"github.com/wal-g/wal-g/tests_func/mongodb/mongoload/models"
 )
 
 func HandleLoad(ctx context.Context, reader io.Reader, mc *mongo.Client, workers int) (models.LoadStat, error) {

tests_func/mongodb/mongoload/main/ammogenerator/ammo_generator.go

@@ -0,0 +1,9 @@
+package main
+
+import (
+	"github.com/wal-g/wal-g/tests_func/mongodb/mongoload/cmd/ammogenerator"
+)
+
+func main() {
+	ammogenerator.Execute()
+}

tests_func/mongodb/mongoload/main/loader/loader.go

@@ -0,0 +1,9 @@
+package main
+
+import (
+	"github.com/wal-g/wal-g/tests_func/mongodb/mongoload/cmd/loader"
+)
+
+func main() {
+	loader.Execute()
+}

tests_func/steps.go

@@ -14,8 +14,8 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/wal-g/tracelog"
 	"github.com/wal-g/wal-g/tests_func/helpers"
-	"github.com/wal-g/wal-g/tests_func/mongoload"
-	"github.com/wal-g/wal-g/tests_func/mongoload/models"
+	"github.com/wal-g/wal-g/tests_func/mongodb/mongoload"
+	"github.com/wal-g/wal-g/tests_func/mongodb/mongoload/models"
 )
 
 type TestingfWrap func(format string, args ...interface{})
@@ -310,15 +310,19 @@ func (tctx *TestContext) enableAuth(host string) error {
 	return mc.EnableAuth()
 }
 
-func (tctx *TestContext) loadMongodbOpsFromConfig(host string, loadId string) error {
+func (tctx *TestContext) getMongoConfigPath(loadId, filename string) string {
+	// Mongo configs stored in "mongodb/config"
+	return path.Join("mongodb", "config", loadId, filename)
+}
 
-	ammoFile, err := os.Open(path.Join("config", loadId, "config.json"))
+func (tctx *TestContext) loadMongodbOpsFromConfig(host string, loadId string) error {
+	ammoFile, err := os.Open(tctx.getMongoConfigPath(loadId, "config.json"))
 	if err != nil {
 		return err
 	}
 	defer func() { _ = ammoFile.Close() }()
 
-	expectedFile, err := os.Open(path.Join("config", loadId, "expected.json"))
+	expectedFile, err := os.Open(tctx.getMongoConfigPath(loadId, "expected.json"))
 	if err != nil {
 		return err
 	}