Add 6.2 support, add redis_cli.sh workaround (wal-g#1022)

* Add 6.2 support, add redis_cli.sh workaround * Update redis_cli.sh * Update redis_cli.sh
citusdata · Jun 28, 2021 · ea6a5ad · ea6a5ad
1 parent f6805f9
commit ea6a5ad
Show file tree

Hide file tree

Showing 8 changed files with 56 additions and 3 deletions.
diff --git a/.github/workflows/dockertests.yml b/.github/workflows/dockertests.yml
@@ -106,8 +106,10 @@ jobs:
                    'make redis_test',
                    'make REDIS_VERSION="5.0.8" redis_features',
                    'make REDIS_VERSION="6.0.8" redis_features',
+                   'make REDIS_VERSION="6.2.4" redis_features',
                    'make FEATURE="backup" REDIS_VERSION="5.0.8" redis_features',
                    'make FEATURE="backup" REDIS_VERSION="6.0.8" redis_features',
+                   'make FEATURE="backup" REDIS_VERSION="6.2.4" redis_features',
                    'make TEST="pg_wale_tablespace_compatibility_test" pg_integration_test',
                    'make TEST="pg_tablespace_support_test" pg_integration_test',
                    'make TEST="pg_wal_perftest" pg_integration_test',

diff --git a/docs/Redis.md b/docs/Redis.md
@@ -116,6 +116,11 @@ Typical configurations
 
 Here's typical wal-g configuration for that case:
 ```bash
-WALG_STREAM_CREATE_COMMAND:  'redis-cli --rdb /dev/stdout'
+WALG_STREAM_CREATE_COMMAND:  'redis_cli --rdb /dev/stdout'
 WALG_STREAM_RESTORE_COMMAND: 'cat > /var/lib/redis/dump.rdb'
 ```
+
+### Why we made redis_cli.sh
+redis-cli fails with error when redis version >= 6.2, so we made this workaround
+
+If you use redis >= 6.2, use [redis_cli.sh](https://github.com/wal-g/wal-g/blob/master/redis_cli.sh) and replace redis-cli in WALG_STREAM_CREATE_COMMAND
diff --git a/redis_cli.sh b/redis_cli.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+#
+# All args passed to this script will be passed to "redis-cli .. --rdb -"
+#
+# This script - workaround with redis-cli >= 6.2 bug, when redis-cli tries to fsync /dev/stdout and exit with error
+# see https://github.com/redis/redis/pull/9135
+#
+
+FILENAME=$(mktemp --suffix=redis-cli-stderr)
+redis-cli $@ --rdb - 2>$FILENAME
+exit_code=$?
+cat $FILENAME >&2
+grep "Fail to fsync" $FILENAME | grep -q "Invalid argument"
+FAILED=$?
+rm $FILENAME;
+if [[ $exit_code -ne 0 ]] ; then
+    test 0 -eq $FAILED || exit $exit_code
+fi
+exit 0
diff --git a/tests_func/Dockerfile.redis b/tests_func/Dockerfile.redis
@@ -89,6 +89,8 @@ RUN mkdir -p /var/log/wal-g/ ${TMP_DIR} && \
     ln --force -s /config/gpg-key.armor ${WALG_CONF_DIR}/gpg-key.armor
 
 COPY staging/wal-g ${WALG_REPO}/
+COPY staging/wal-g/redis_cli.sh /usr/bin/redis_cli.sh
+
 WORKDIR ${WALG_REPO}
 
 RUN make link_external_deps

diff --git a/tests_func/images/redis/config/wal-g-5.0.8.json b/tests_func/images/redis/config/wal-g-5.0.8.json
@@ -5,7 +5,7 @@
   "AWS_ENDPOINT": "http://minio:9000",
   "AWS_S3_FORCE_PATH_STYLE": "true",
 
-  "WALG_STREAM_CREATE_COMMAND": "redis-cli --rdb /dev/stdout",
+  "WALG_STREAM_CREATE_COMMAND": "/usr/bin/redis_cli.sh --rdb /dev/stdout",
   "WALG_STREAM_RESTORE_COMMAND": "cat > /data/dump.rdb",
 
   "WALG_SENTINEL_USER_DATA":    "{\"key\": \"value\"}",

diff --git a/tests_func/images/redis/config/wal-g-6.0.8.json b/tests_func/images/redis/config/wal-g-6.0.8.json
@@ -5,7 +5,7 @@
   "AWS_ENDPOINT": "http://minio:9000",
   "AWS_S3_FORCE_PATH_STYLE": "true",
 
-  "WALG_STREAM_CREATE_COMMAND": "redis-cli --rdb /dev/stdout",
+  "WALG_STREAM_CREATE_COMMAND": "/usr/bin/redis_cli.sh --rdb /dev/stdout",
   "WALG_STREAM_RESTORE_COMMAND": "cat > /data/dump.rdb",
 
   "WALG_SENTINEL_USER_DATA":    "{\"key\": \"value\"}",

diff --git a/tests_func/images/redis/config/wal-g-6.2.4.json b/tests_func/images/redis/config/wal-g-6.2.4.json
@@ -0,0 +1,21 @@
+{
+  "WALG_S3_PREFIX": "s3://dbaas/redis-backup/test_uuid/test_redis",
+  "AWS_ACCESS_KEY_ID": "S3_ACCESS_KEY",
+  "AWS_SECRET_ACCESS_KEY": "S3_SECRET_KEY",
+  "AWS_ENDPOINT": "http://minio:9000",
+  "AWS_S3_FORCE_PATH_STYLE": "true",
+
+  "WALG_STREAM_CREATE_COMMAND": "/usr/bin/redis_cli.sh --rdb /dev/stdout",
+  "WALG_STREAM_RESTORE_COMMAND": "cat > /data/dump.rdb",
+
+  "WALG_SENTINEL_USER_DATA":    "{\"key\": \"value\"}",
+
+  "WALG_PGP_KEY_PATH": "/etc/wal-g/gpg-key.armor",
+  "WALG_COMPRESSION_METHOD": "brotli",
+  "WALG_DISK_RATE_LIMIT": "41943040",
+  "WALG_NETWORK_RATE_LIMIT": "10485760",
+  "WALG_LOG_LEVEL": "DEVEL",
+  "WALG_UPLOAD_CONCURRENCY" : 1,
+
+  "WALG_REDIS_PASSWORD": "password"
+}
diff --git a/tests_func/redis_steps.go b/tests_func/redis_steps.go
@@ -129,6 +129,10 @@ func (tctx *TestContext) testEqualRedisDataAtHosts(host1, host2 string) error {
 		return keys2.Err()
 	}
 
+	if len(keys1.Val()) == 0 || len(keys2.Val()) == 0 {
+		return fmt.Errorf("keys1 or keys2 is empty - broken backup")
+	}
+
 	if !utils.IsArraysEqual(keys1.Val(), keys2.Val()) {
 		return fmt.Errorf("keys from redis1/redis2 aren't equal")
 	}