[WIP] Greenplum backup push (wal-g#992)

* [WIP] add greenplum backup-push command

* [WIP] fix GetGreenplumSegmentsInfo query

* [WIP] fix queries

* [WIP] cleanup

* [WIP] fix

* [WIP] add port to command

* [WIP] fix backupNamePrefix flag

* fix postgres backup name regexp

* fix extracting backup names

* fix connection to segments

* fix extracting backups for backup-list

* add logging

* fix delta backup

* add test

* fix test

* included test into CI

* fix lint

* fix

* fix comments

* Revert backup prefix logic

* Refactor args handling

* change wal-g docker binary name

* Remove unused stuff && fix linter complaints

* Add config path specification

* Add wal-g output to debug

* Implement single config per host

Add WALG_STORAGE_PREFIX config option
Remove -c shorthand to avoid confusion with --config
Remove backup-push argument in docker test

* Add config option to tests && fix linter error

* Update go.mod && go.sum

Co-authored-by: nastprol <[email protected]>
Co-authored-by: usernamedt <[email protected]>

.github/workflows/dockertests.yml

@@ -118,6 +118,7 @@ jobs:
                    'make MYSQL_TEST=mysql_base_tests mysql_integration_test',
                    'make MYSQL_TEST=mysql_delete_tests mysql_integration_test',
                    'make MYSQL_TEST=mysql_copy_tests mysql_integration_test',
+                   'make gp_test',
       ]
       # do not cancel all tests if one failed
       fail-fast: false

.gitignore

@@ -9,6 +9,7 @@ main/pg/wal-g
 main/mysql/wal-g
 main/redis/wal-g
 main/mongo/wal-g
+main/gp/wal-g
 
 # tests
 test/testdata/00000*

Makefile

@@ -4,6 +4,7 @@ MAIN_SQLSERVER_PATH := main/sqlserver
 MAIN_REDIS_PATH := main/redis
 MAIN_MONGO_PATH := main/mongo
 MAIN_FDB_PATH := main/fdb
+MAIN_GP_PATH := main/gp
 DOCKER_COMMON := golang ubuntu s3
 CMD_FILES = $(wildcard cmd/**/*.go)
 PKG_FILES = $(wildcard internal/**/*.go internal/**/**/*.go internal/*.go)
@@ -29,7 +30,7 @@ endif
 
 .PHONY: unittest fmt lint clean
 
-test: deps unittest pg_build mysql_build redis_build mongo_build unlink_brotli pg_integration_test mysql_integration_test redis_integration_test fdb_integration_test
+test: deps unittest pg_build mysql_build redis_build mongo_build gp_build unlink_brotli pg_integration_test mysql_integration_test redis_integration_test fdb_integration_test gp_integration_test
 
 pg_test: deps pg_build unlink_brotli pg_integration_test
 
@@ -162,6 +163,21 @@ clean_redis_features:
 	set -e
 	cd tests_func/ && REDIS_VERSION=$(REDIS_VERSION) go test -v -count=1  -timeout 5m -tf.test=false -tf.debug=false -tf.clean=true -tf.stop=true -tf.database=redis
 
+gp_build: $(CMD_FILES) $(PKG_FILES)
+	(cd $(MAIN_GP_PATH) && go build -mod vendor -tags "$(BUILD_TAGS)" -o wal-g -ldflags "-s -w -X github.com/wal-g/wal-g/cmd/gp.buildDate=`date -u +%Y.%m.%d_%H:%M:%S` -X github.com/wal-g/wal-g/cmd/gp.gitRevision=`git rev-parse --short HEAD` -X github.com/wal-g/wal-g/cmd/gp.walgVersion=`git tag -l --points-at HEAD`")
+
+gp_clean:
+	(cd $(MAIN_GP_PATH) && go clean)
+	./cleanup.sh
+
+gp_install: gp_build
+	mv $(MAIN_GP_PATH)/wal-g $(GOBIN)/wal-g
+
+gp_test: deps gp_build unlink_brotli gp_integration_test
+
+gp_integration_test: load_docker_common
+	docker-compose build gp gp_tests
+	docker-compose up --exit-code-from gp_tests gp_tests
 
 unittest:
 	go list ./... | grep -Ev 'vendor|submodules|tmp' | xargs go vet
@@ -217,7 +233,7 @@ link_external_deps: link_brotli link_libsodium
 unlink_external_deps: unlink_brotli unlink_libsodium
 
 install:
-	@echo "Nothing to be done. Use pg_install/mysql_install/mongo_install/fdb_install/... instead."
+	@echo "Nothing to be done. Use pg_install/mysql_install/mongo_install/fdb_install/gp_install... instead."
 
 link_brotli:
 	./link_brotli.sh

cmd/gp/backup_push.go

@@ -0,0 +1,85 @@
+package gp
+
+import (
+	"strconv"
+
+	"github.com/wal-g/wal-g/internal/databases/greenplum"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/wal-g/tracelog"
+	"github.com/wal-g/wal-g/internal"
+)
+
+const (
+	backupPushShortDescription = "Makes backup and uploads it to storage"
+
+	permanentFlag             = "permanent"
+	fullBackupFlag            = "full"
+	verifyPagesFlag           = "verify"
+	storeAllCorruptBlocksFlag = "store-all-corrupt"
+	useRatingComposerFlag     = "rating-composer"
+	addUserDataFlag           = "add-user-data"
+
+	permanentShorthand             = "p"
+	fullBackupShorthand            = "f"
+	verifyPagesShorthand           = "v"
+	storeAllCorruptBlocksShorthand = "s"
+	useRatingComposerShorthand     = "r"
+)
+
+var (
+	// backupPushCmd represents the backupPush command
+	backupPushCmd = &cobra.Command{
+		Use:   "backup-push",
+		Short: backupPushShortDescription, // TODO : improve description
+		Args:  cobra.NoArgs,
+		Run: func(cmd *cobra.Command, args []string) {
+			if userData == "" {
+				userData = viper.GetString(internal.SentinelUserDataSetting)
+			}
+
+			arguments := greenplum.NewBackupArguments(permanent, userData, prepareSegmentFwdArgs())
+			backupHandler, err := greenplum.NewBackupHandler(arguments)
+			tracelog.ErrorLogger.FatalOnError(err)
+			backupHandler.HandleBackupPush()
+		},
+	}
+	permanent = false
+	userData  = ""
+
+	// as for now, WAL-G will simply forward these arguments to the segments
+	// todo: handle delta-from-name and delta-from-userdata
+	fullBackup            = false
+	verifyPageChecksums   = false
+	storeAllCorruptBlocks = false
+	useRatingComposer     = false
+)
+
+// prepare arguments that are going to be forwarded to segments
+func prepareSegmentFwdArgs() []greenplum.SegmentFwdArg {
+	verifyPageChecksums = verifyPageChecksums || viper.GetBool(internal.VerifyPageChecksumsSetting)
+	storeAllCorruptBlocks = storeAllCorruptBlocks || viper.GetBool(internal.StoreAllCorruptBlocksSetting)
+	useRatingComposer = useRatingComposer || viper.GetBool(internal.UseRatingComposerSetting)
+
+	return []greenplum.SegmentFwdArg{
+		{Name: fullBackupFlag, Value: strconv.FormatBool(fullBackup)},
+	}
+}
+
+func init() {
+	cmd.AddCommand(backupPushCmd)
+
+	backupPushCmd.Flags().BoolVarP(&permanent, permanentFlag, permanentShorthand,
+		false, "Pushes permanent backup")
+	backupPushCmd.Flags().BoolVarP(&fullBackup, fullBackupFlag, fullBackupShorthand,
+		false, "Make full backup-push")
+	backupPushCmd.Flags().BoolVarP(&verifyPageChecksums, verifyPagesFlag, verifyPagesShorthand,
+		false, "Verify page checksums")
+	backupPushCmd.Flags().BoolVarP(&storeAllCorruptBlocks, storeAllCorruptBlocksFlag, storeAllCorruptBlocksShorthand,
+		false, "Store all corrupt blocks found during page checksum verification")
+	backupPushCmd.Flags().BoolVarP(&useRatingComposer, useRatingComposerFlag, useRatingComposerShorthand,
+		false, "Use rating tar composer (beta)")
+	backupPushCmd.Flags().StringVar(&userData, addUserDataFlag,
+		"", "Write the provided user data to the backup sentinel and metadata files.")
+}

cmd/gp/gp.go

@@ -0,0 +1,47 @@
+package gp
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/wal-g/tracelog"
+	"github.com/wal-g/wal-g/internal"
+)
+
+var dbShortDescription = "GreenplumDB backup tool"
+
+// These variables are here only to show current version. They are set in makefile during build process
+var walgVersion = "devel"
+var gitRevision = "devel"
+var buildDate = "devel"
+
+var cmd = &cobra.Command{
+	Use:     "wal-g",
+	Short:   dbShortDescription, // TODO : improve description
+	Version: strings.Join([]string{walgVersion, gitRevision, buildDate, "GreenplumDB"}, "\t"),
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		err := internal.AssertRequiredSettingsSet()
+		tracelog.ErrorLogger.FatalOnError(err)
+	},
+}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main().
+func Execute() {
+	if err := cmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func init() {
+	internal.ConfigureSettings(internal.GP)
+	cobra.OnInitialize(internal.InitConfig, internal.Configure)
+
+	cmd.PersistentFlags().StringVar(&internal.CfgFile, "config", "", "config file (default is $HOME/.wal-g.yaml)")
+	_ = cmd.MarkFlagRequired("config") // config is required for Greenplum WAL-G
+	cmd.InitDefaultVersionFlag()
+	internal.AddConfigFlags(cmd)
+}

docker-compose.yml

@@ -71,6 +71,7 @@ services:
       &&  mkdir -p /export/mysqlconfonly
       &&  mkdir -p /export/mysqldeletebinlogs
       &&  mkdir -p /export/archivereadyrename
+      &&  mkdir -p /export/gpfullbucket
       &&  /usr/bin/minio server /export'
 
   s3-another:
@@ -601,3 +602,24 @@ services:
       - s3
     links:
       - s3
+
+  gp:
+    build:
+      dockerfile: docker/gp/Dockerfile
+      context: .
+    image: wal-g/gp
+    container_name: wal-g_gp
+
+  gp_tests:
+    build:
+      dockerfile: docker/gp_tests/Dockerfile
+      context: .
+    image: wal-g/gp_tests
+    container_name: wal-g_gp_tests
+    env_file:
+      - docker/common/common_walg.env
+    depends_on:
+      - gp
+      - s3
+    links:
+      - s3

docker/gp/Dockerfile

@@ -0,0 +1,35 @@
+FROM wal-g/ubuntu:latest
+
+RUN apt-get update && \
+    apt-get install --yes --no-install-recommends --no-install-suggests locales iputils-ping ssh python-dev iproute2 less sudo \
+    gnupg \
+    gpg-agent \
+    pinentry-qt \
+    time \
+    bc \
+    jq
+
+WORKDIR /usr/local
+RUN git clone https://github.com/greenplum-db/gpdb.git gpdb_src
+
+RUN ./gpdb_src/concourse/scripts/setup_gpadmin_user.bash
+
+WORKDIR /usr/local/gpdb_src
+RUN locale-gen en_US.utf8
+RUN ./README.Ubuntu.bash
+
+WORKDIR /usr/local/gpdb_src
+RUN ./configure --with-perl --with-python --with-libxml --with-gssapi --prefix=/usr/local/gpdb_src > /dev/nul && \
+    make -j8 > /dev/nul && \
+    make -j8 install > /dev/null
+
+ADD docker/gp/run_greenplum.sh /home/gpadmin/run_greenplum.sh
+
+RUN chown gpadmin:gpadmin /home/gpadmin/run_greenplum.sh \
+ && chmod a+x /home/gpadmin/run_greenplum.sh \
+ && echo "export MASTER_DATA_DIRECTORY=/usr/local/gpdb_src/gpAux/gpdemo/datadirs/qddir/demoDataDir-1" > /home/gpadmin/.bash_profile \
+ && echo "source /usr/local/gpdb_src/greenplum_path.sh" > /home/gpadmin/.bash_profile \
+ && chown gpadmin:gpadmin /home/gpadmin/.bash_profile
+
+RUN echo "gpadmin ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers \
+ && echo "root ALL=NOPASSWD: ALL" >> /etc/sudoers

docker/gp/run_greenplum.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+sudo /etc/init.d/ssh start
+
+. /usr/local/gpdb_src/greenplum_path.sh
+
+cd /usr/local/gpdb_src
+
+/usr/local/gpdb_src/bin/gpssh-exkeys -h `hostname`
+
+make create-demo-cluster
+
+. /usr/local/gpdb_src/gpAux/gpdemo/gpdemo-env.sh

docker/gp_tests/Dockerfile

@@ -0,0 +1,47 @@
+FROM wal-g/golang:latest as build
+
+WORKDIR /go/src/github.com/wal-g/wal-g
+
+RUN apt-get update && \
+    apt-get install --yes --no-install-recommends --no-install-suggests
+
+COPY go.mod go.mod
+COPY vendor/ vendor/
+COPY internal/ internal/
+COPY cmd/ cmd/
+COPY main/ main/
+COPY utility/ utility/
+COPY Makefile Makefile
+
+
+RUN sed -i 's|#cgo LDFLAGS: -lbrotli.*|&-static -lbrotlicommon-static -lm|' \
+        vendor/github.com/google/brotli/go/cbrotli/cgo.go && \
+    cd main/gp && \
+    go build -mod vendor -tags brotli -race -o wal-g -ldflags "-s -w -X main.buildDate=`date -u +%Y.%m.%d_%H:%M:%S`"
+
+RUN make gp_build
+
+RUN cd main/pg && \
+    go build -mod vendor -tags brotli -race -o wal-g -ldflags "-s -w -X main.buildDate=`date -u +%Y.%m.%d_%H:%M:%S`"
+
+RUN make pg_build
+
+FROM wal-g/gp:latest
+
+USER root
+
+COPY docker/pg/PGP_KEY /tmp/PGP_KEY
+COPY docker/pg/gpg.conf /home/gpadmin/.gnupg/gpg.conf
+COPY docker/pg/gpg-agent.conf /home/gpadmin/.gnupg/gpg-agent.conf
+
+COPY --from=build /go/src/github.com/wal-g/wal-g/main/gp/wal-g /usr/bin/wal-g-gp
+COPY --from=build /go/src/github.com/wal-g/wal-g/main/pg/wal-g /usr/bin
+
+COPY docker/gp_tests/scripts/ /tmp
+COPY docker/pg_tests/scripts/scripts/ /tmp/pg_scripts
+
+RUN chmod 777 /tmp/configs/
+
+RUN /bin/bash -c 'sudo chmod +x -R /tmp/tests; chmod +x /tmp/run_integration_tests.sh'
+
+CMD su gpadmin bash -c '/tmp/run_integration_tests.sh'

docker/gp_tests/scripts/configs/common_config.json

@@ -0,0 +1,10 @@
+"AWS_SECRET_ACCESS_KEY": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+"PGSSLMODE": "allow",
+"WALG_NETWORK_RATE_LIMIT": "10485760",
+"WALG_DISK_RATE_LIMIT": "41943040",
+"AWS_ACCESS_KEY_ID": "AKIAIOSFODNN7EXAMPLE",
+"WALG_UPLOAD_CONCURRENCY": "10",
+"AWS_ENDPOINT": "http://s3:9000",
+"PGDATABASE": "postgres",
+"AWS_S3_FORCE_PATH_STYLE": "true",
+"WALG_COMPRESSION_METHOD": "brotli"

docker/gp_tests/scripts/configs/full_backup_test_config.json

@@ -0,0 +1,4 @@
+"WALE_S3_PREFIX": "s3://gpfullbucket",
+"WALG_DELTA_MAX_STEPS": "6",
+"WALG_PGP_KEY_PATH": "/tmp/PGP_KEY",
+"WALG_LOG_LEVEL": "DEVEL"

docker/gp_tests/scripts/run_integration_tests.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e -x
+
+pushd /tmp
+for i in tests/*.sh; do
+  echo
+  echo "===== RUNNING $i ====="
+  set -x
+  ./"$i";
+
+  set +x
+  echo "===== SUCCESS $i ====="
+  echo
+done
+popd

docker/gp_tests/scripts/tests/gp_full_backup_test.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e -x
+
+CONFIG_FILE="/tmp/configs/full_backup_test_config.json"
+COMMON_CONFIG="/tmp/configs/common_config.json"
+TMP_CONFIG="/tmp/configs/tmp_config.json"
+cat ${CONFIG_FILE} > ${TMP_CONFIG}
+echo "," >> ${TMP_CONFIG}
+cat ${COMMON_CONFIG} >> ${TMP_CONFIG}
+/tmp/pg_scripts/wrap_config_file.sh ${TMP_CONFIG}
+
+/home/gpadmin/run_greenplum.sh
+
+source /usr/local/gpdb_src/gpAux/gpdemo/gpdemo-env.sh && /usr/local/gpdb_src/bin/createdb
+sleep 10
+
+wal-g-gp backup-push --config=${TMP_CONFIG}
+
+echo "Greenplum backup-push test was successful"