Copyfix (wal-g#789)

* fix mysql copy

* fix tests && and more tests

* configuration from conf fix

* fix path incorrect concatinations + fmt & tests

Co-authored-by: reshke <Kirill Reshke>

cleanup.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 rm -rf tmp/
 rm -rf vendor/github.com/google/brotli/*

cmd/mysql/copy.go

@@ -9,9 +9,9 @@ const (
 	copyName             = "copy"
 	copyShortDescription = "copy specific or all backups"
 
-	copyAllFlag             = "all"
+	copyAllFlag         = "all"
 	copyAllSDescription = "copy all backups"
-	allShorthand   = "a"
+	allShorthand        = "a"
 
 	backupNameFlag         = "backup"
 	backupShorthand        = "b"
@@ -30,7 +30,7 @@ var (
 	backupName     string
 	fromConfigFile string
 	toConfigFile   string
-	all bool
+	all            bool
 
 	copyCmd = &cobra.Command{
 		Use:   copyName,

cmd/mysql/mysql.go

@@ -23,7 +23,9 @@ var Cmd = &cobra.Command{
 	Version: strings.Join([]string{WalgVersion, GitRevision, BuildDate, "MySQL"}, "\t"),
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
 		err := internal.AssertRequiredSettingsSet()
-		tracelog.ErrorLogger.FatalOnError(err)
+		if err != nil {
+			tracelog.WarningLogger.PrintError(err)
+		}
 	},
 }
 

docker-compose.yml

@@ -62,6 +62,20 @@ services:
       &&  mkdir -p /export/redisbucket
       &&  mkdir -p /export/mysqlcopybackupfrom
       &&  mkdir -p /export/mysqlcopybackupto
+      &&  mkdir -p /export/mysqlconfonly
+      &&  /usr/bin/minio server /export'
+
+  s3-another:
+    image: minio/minio
+    container_name: wal-g_s32
+    ports:
+      - "9001:9001"
+    environment:
+      - "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLEBUTANOHER"
+      - "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEYBUTANOTHERTOO"
+    entrypoint: sh
+    command: >
+      -c 'mkdir -p /export/mysqlcopybackupto
       &&  /usr/bin/minio server /export'
 
   ssh:
@@ -420,8 +434,10 @@ services:
       - docker/common/common_walg.env
     depends_on:
       - s3
+      - s3-another
     links:
       - s3
+      - s3-another
 
   mariadb:
     build:

docker/mysql/Dockerfile

@@ -17,6 +17,7 @@ COPY docker/mysql/client.cnf /root/.my.cnf
 COPY docker/mysql/client.cnf /etc/mysql/debian.cnf
 COPY docker/mysql/init.sql /etc/mysql/init.sql
 COPY docker/mysql/export_common.sh /usr/local/export_common.sh
+COPY docker/mysql/export_test_funcs.sh /usr/local/export_test_funcs.sh
 
 # append
 COPY docker/mysql/my.cnf /tmp/my.cnf

docker/mysql/export_common.sh

@@ -8,28 +8,12 @@ export WALG_STREAM_CREATE_COMMAND="xtrabackup --backup \
     --host=localhost \
     --parallel=2 \
     --datadir=${MYSQLDATA}"
+
 export WALG_STREAM_RESTORE_COMMAND="xbstream -x -C ${MYSQLDATA}"
 export WALG_MYSQL_BACKUP_PREPARE_COMMAND="xtrabackup --prepare --target-dir=${MYSQLDATA}"
+# shellcheck disable=SC2016
 export WALG_MYSQL_BINLOG_REPLAY_COMMAND='mysqlbinlog --stop-datetime="$WALG_MYSQL_BINLOG_END_TS" "$WALG_MYSQL_CURRENT_BINLOG" | mysql'
 export WALG_MYSQL_BINLOG_DST=/tmp
 
-# test tools
-mysql_kill_and_clean_data() {
-    kill -9 `pidof mysqld` || true
-    rm -rf "${MYSQLDATA}"/*
-    rm -rf /root/.walg_mysql_binlogs_cache
-}
-
-mysql_set_gtid_purged() {
-    gtids=$(tr -d '\n' < /var/lib/mysql/xtrabackup_binlog_info | awk '{print $3}')
-    echo "Gtids from backup $gtids"
-    mysql -e "RESET MASTER; SET @@GLOBAL.GTID_PURGED='$gtids';"
-}
-
-sysbench() {
-    /usr/bin/sysbench --verbosity=0 /usr/share/sysbench/oltp_insert.lua $@
-}
 
-date3339() {
-    date --rfc-3339=ns | sed 's/ /T/'
-}
+. /usr/local/export_test_funcs.sh

docker/mysql/export_test_funcs.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# test tools
+mysql_kill_and_clean_data() {
+    kill -9 "$(pidof mysqld)" || true
+    rm -rf "${MYSQLDATA}"/*
+    rm -rf "${MYSQLDATA}"/.tmp
+    rm -rf /root/.walg_mysql_binlogs_cache
+}
+
+mysql_set_gtid_purged() {
+    gtids=$(tr -d '\n' < /var/lib/mysql/xtrabackup_binlog_info | awk '{print $3}')
+    echo "Gtids from backup $gtids"
+    mysql -e "RESET MASTER; SET @@GLOBAL.GTID_PURGED='$gtids';"
+}
+
+sysbench() {
+    /usr/bin/sysbench --verbosity=0 /usr/share/sysbench/oltp_insert.lua "$@"
+}
+
+date3339() {
+    date --rfc-3339=ns | sed 's/ /T/'
+}
+

docker/mysql_tests/scripts/run_integration_tests.sh

@@ -10,6 +10,7 @@ for i in /tmp/tests/*; do
   echo
   echo "===== RUNNING $i ====="
   set -x
+  chmod a+x "$i"
   "$i"
   set +x
   echo "===== SUCCESS $i ====="

docker/mysql_tests/scripts/tests/binlog_push_fetch_test.sh

@@ -6,12 +6,11 @@ set -e -x
 export WALE_S3_PREFIX=s3://mysqlbinlogpushfetchbucket
 export WALG_MYSQL_BINLOG_DST=/tmp/binlogs
 
-
 mysqld --initialize --init-file=/etc/mysql/init.sql
 service mysql start
 
 # drop all before-dump binlogs, so SHOW BINNARY LOGS will show all binlogs we need to fetch
-current_binlog=`mysql -e "SHOW BINARY LOGS" | tail -n 1 | awk '{print $1}'`
+current_binlog=$(mysql -e "SHOW BINARY LOGS" | tail -n 1 | awk '{print $1}')
 mysql -e "PURGE BINARY LOGS TO '$current_binlog'";
 
 wal-g backup-push
@@ -27,8 +26,8 @@ mysql -e "FLUSH LOGS"
 wal-g binlog-push
 
 # last binlog waas not archived
-current_binlog=`mysql -e "SHOW BINARY LOGS" | tail -n 1 | awk '{print $1}'`
-mysql -N -e 'show binary logs' | awk '{print $1}' | grep -v $current_binlog > /tmp/proper_order
+current_binlog=$(mysql -e "SHOW BINARY LOGS" | tail -n 1 | awk '{print $1}')
+mysql -N -e 'show binary logs' | awk '{print $1}' | grep -v "$current_binlog" > /tmp/proper_order
 
 rm -rf /tmp/binlogs
 mkdir /tmp/binlogs
@@ -40,11 +39,11 @@ sleep 2
 wal-g binlog-fetch --since LATEST
 diff -u /tmp/proper_order /tmp/binlogs/binlogs_order
 while read -r binlog; do
-    test -f /tmp/binlogs/$binlog
-    ls -lah $MYSQLDATA/$binlog /tmp/binlogs/$binlog 
-    if ! cmp $MYSQLDATA/$binlog /tmp/binlogs/$binlog; then
-        mysqlbinlog -v $MYSQLDATA/$binlog > /tmp/proper.sql
-        mysqlbinlog -v /tmp/binlogs/$binlog > /tmp/fetched.sql
+    test -f /tmp/binlogs/"$binlog"
+    ls -lah "$MYSQLDATA"/"$binlog" /tmp/binlogs/"$binlog"
+    if ! cmp "$MYSQLDATA"/"$binlog" /tmp/binlogs/"$binlog"; then
+        mysqlbinlog -v "$MYSQLDATA"/"$binlog" > /tmp/proper.sql
+        mysqlbinlog -v /tmp/binlogs/"$binlog" > /tmp/fetched.sql
         diff -u /tmp/proper.sql /tmp/fetched.sql
     fi
 done < /tmp/binlogs/binlogs_order

docker/mysql_tests/scripts/tests/conf-only-test.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+set -e -x
+
+. /usr/local/export_test_funcs.sh
+
+# https://github.com/wal-g/wal-g/issues/790
+## ensure correct conf option will overwrite this trash
+#export AWS_ENDPOINT=we23i902309239
+
+cat > /root/conf.yaml <<EOH
+WALE_S3_PREFIX: s3://mysqlconfonly
+AWS_ENDPOINT: http://s3:9000
+AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
+AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+WALG_S3_MAX_PART_SIZE: 5242880
+
+WALG_STREAM_RESTORE_COMMAND: "xbstream -x -C ${MYSQLDATA}"
+WALG_MYSQL_BACKUP_PREPARE_COMMAND: "xtrabackup --prepare --target-dir=${MYSQLDATA}"
+WALG_MYSQL_BINLOG_REPLAY_COMMAND: 'mysqlbinlog --stop-datetime="$WALG_MYSQL_BINLOG_END_TS" "$WALG_MYSQL_CURRENT_BINLOG" | mysql'
+WALG_MYSQL_BINLOG_DST: /tmp
+WALG_MYSQL_DATASOURCE_NAME: sbtest:@/sbtest
+WALG_STREAM_CREATE_COMMAND: "xtrabackup --backup \
+    --stream=xbstream \
+    --user=sbtest \
+    --host=localhost \
+    --parallel=2 \
+    --datadir=${MYSQLDATA}"
+EOH
+
+mysqld --initialize --init-file=/etc/mysql/init.sql
+service mysql start
+mysql mysql -e 'create table testt1(i int)'
+
+export NAME
+NAME=$(wal-g backup-push --config=/root/conf.yaml 2>&1 | grep -oe 'stream_[0-9]*T[0-9]*Z')
+
+mysql_kill_and_clean_data
+
+wal-g backup-fetch "$NAME" --config=/root/conf.yaml
+
+chown -R mysql:mysql "$MYSQLDATA"
+service mysql start || (cat /var/log/mysql/error.log && false)
+mysql mysql -e 'show tables' | grep testt1

docker/mysql_tests/scripts/tests/copy_backup.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+set -e -x
+
+. /usr/local/export_common.sh
+
+mysqld --initialize --init-file=/etc/mysql/init.sql
+service mysql start
+mysql mysql -e 'create table testt1(i int)'
+
+cat > /root/from.yaml <<EOH
+WALE_S3_PREFIX: "s3://mysqlcopybackupfrom"
+AWS_ENDPOINT: "http://s3:9000"
+AWS_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE"
+AWS_SECRET_ACCESS_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+WALG_S3_MAX_PART_SIZE: 5242880
+EOH
+
+cat > /root/to.yaml <<EOH
+WALE_S3_PREFIX: "s3://mysqlcopybackupto"
+AWS_ENDPOINT: "http://s3:9000"
+AWS_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE"
+AWS_SECRET_ACCESS_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+WALG_S3_MAX_PART_SIZE: 5242880
+EOH
+
+wal-g delete everything FORCE --confirm --config=/root/from.yaml
+wal-g delete everything FORCE --confirm --config=/root/to.yaml
+
+export NAME
+NAME=$(wal-g backup-push --config=/root/from.yaml 2>&1 | grep -oe 'stream_[0-9]*T[0-9]*Z')
+
+sleep 1
+
+echo "$NAME"
+
+wal-g backup-list --config=/root/from.yaml
+wal-g backup-list --config=/root/to.yaml
+
+wal-g copy --from=/root/from.yaml --to=/root/to.yaml --backup "$NAME"
+
+mysql_kill_and_clean_data
+
+wal-g backup-fetch "$NAME" --config=/root/to.yaml
+
+chown -R mysql:mysql "$MYSQLDATA"
+service mysql start || (cat /var/log/mysql/error.log && false)
+mysql mysql -e 'show tables' | grep testt1

docker/mysql_tests/scripts/tests/copyBackup.sh → docker/mysql_tests/scripts/tests/copy_between_different_accounts_storage.sh

@@ -7,35 +7,48 @@ mysqld --initialize --init-file=/etc/mysql/init.sql
 service mysql start
 mysql mysql -e 'create table testt1(i int)'
 
-
 cat > /root/from.yaml <<EOH
 WALE_S3_PREFIX: s3://mysqlcopybackupfrom
+AWS_ENDPOINT: http://s3:9000
+AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
+AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
 WALG_S3_MAX_PART_SIZE: 5242880
 EOH
 
 cat > /root/to.yaml <<EOH
 WALE_S3_PREFIX: s3://mysqlcopybackupto
+AWS_ENDPOINT: http://s3-another:9000
+AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLEBUTANOHER
+AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEYBUTANOTHERTOO
 WALG_S3_MAX_PART_SIZE: 5242880
 EOH
 
-wal-g backup-list --config=/root/from.yaml
 wal-g delete everything FORCE --confirm --config=/root/from.yaml
-wal-g backup-list --config=/root/from.yaml
+wal-g delete everything FORCE --confirm --config=/root/to.yaml
+
+export NAME
+NAME=$(wal-g backup-push --config=/root/from.yaml 2>&1 | grep -oe 'stream_[0-9]*T[0-9]*Z')
 
-export name=$(wal-g backup-push --config=/root/from.yaml 2>&1 | grep -oe 'stream_[0-9]*T[0-9]*Z')
 sleep 1
 
-echo $name
+echo "$NAME"
 
 wal-g backup-list --config=/root/from.yaml
 wal-g backup-list --config=/root/to.yaml
 
-wal-g copy --from=/root/from.yaml --to=/root/to.yaml --backup "$name" --config=/root/from.yaml
+wal-g copy --from=/root/from.yaml --to=/root/to.yaml --backup "$NAME"
 
 mysql_kill_and_clean_data
 
-wal-g backup-fetch "$name" --config=/root/to.yaml
+unset AWS_ENDPOINT
+unset AWS_SECRET_ACCESS_KEY
+unset AWS_ACCESS_KEY_ID
 
-chown -R mysql:mysql $MYSQLDATA
+wal-g backup-fetch "$NAME" --config=/root/to.yaml
+
+chown -R mysql:mysql "$MYSQLDATA"
 service mysql start || (cat /var/log/mysql/error.log && false)
 mysql mysql -e 'show tables' | grep testt1
+
+wal-g backup-list --config=/root/from.yaml
+wal-g backup-list --config=/root/to.yaml

docker/mysql_tests/scripts/tests/delete_end_to_end_test.sh

@@ -11,7 +11,6 @@ service mysql start
 sysbench --table-size=10 prepare
 mysql -e "FLUSH LOGS"
 
-
 # first backup
 sysbench --time=3 run
 wal-g backup-push
@@ -21,6 +20,7 @@ wal-g binlog-push
 sleep 1
 
 mysqldump sbtest > /tmp/dump_1.sql
+wal-g backup-list
 test "2" -eq "$(wal-g backup-list | wc -l)"
 FIRST_BACKUP=$(wal-g backup-list | awk 'NR==2{print $1}')
 DT1=$(date3339)
@@ -55,17 +55,17 @@ DT3=$(date3339)
 
 
 # delete first backup
-wal-g delete before FIND_FULL $SECOND_BACKUP --confirm
+wal-g delete before FIND_FULL "$SECOND_BACKUP" --confirm
 test "3" -eq "$(wal-g backup-list | wc -l)"
 
 
 # test restore second
 mysql_kill_and_clean_data
-wal-g backup-fetch $SECOND_BACKUP
-chown -R mysql:mysql $MYSQLDATA
+wal-g backup-fetch "$SECOND_BACKUP"
+chown -R mysql:mysql "$MYSQLDATA"
 service mysql start || (cat /var/log/mysql/error.log && false)
 mysql_set_gtid_purged
-wal-g binlog-replay --since $SECOND_BACKUP --until "$DT2"
+wal-g binlog-replay --since "$SECOND_BACKUP" --until "$DT2"
 mysqldump sbtest > /tmp/dump_2_restored.sql
 diff -u /tmp/dump_2.sql /tmp/dump_2_restored.sql
 
@@ -77,10 +77,10 @@ test "2" -eq "$(wal-g backup-list | wc -l)"
 
 # test restore third backup
 mysql_kill_and_clean_data
-wal-g backup-fetch $THIRD_BACKUP
-chown -R mysql:mysql $MYSQLDATA
+wal-g backup-fetch "$THIRD_BACKUP"
+chown -R mysql:mysql "$MYSQLDATA"
 service mysql start || (cat /var/log/mysql/error.log && false)
 mysql_set_gtid_purged
-wal-g binlog-replay --since $THIRD_BACKUP --until "$DT3"
+wal-g binlog-replay --since "$THIRD_BACKUP" --until "$DT3"
 mysqldump sbtest > /tmp/dump_3_restored.sql
 diff -u /tmp/dump_3.sql /tmp/dump_3_restored.sql