diff --git a/.github/workflows/dockertests.yml b/.github/workflows/dockertests.yml index 420a507b9..dca8f6cb3 100644 --- a/.github/workflows/dockertests.yml +++ b/.github/workflows/dockertests.yml @@ -106,8 +106,10 @@ jobs: 'make redis_test', 'make REDIS_VERSION="5.0.8" redis_features', 'make REDIS_VERSION="6.0.8" redis_features', + 'make REDIS_VERSION="6.2.4" redis_features', 'make FEATURE="backup" REDIS_VERSION="5.0.8" redis_features', 'make FEATURE="backup" REDIS_VERSION="6.0.8" redis_features', + 'make FEATURE="backup" REDIS_VERSION="6.2.4" redis_features', 'make TEST="pg_wale_tablespace_compatibility_test" pg_integration_test', 'make TEST="pg_tablespace_support_test" pg_integration_test', 'make TEST="pg_wal_perftest" pg_integration_test', diff --git a/docs/Redis.md b/docs/Redis.md index daf6aba2b..9aeb18ba1 100644 --- a/docs/Redis.md +++ b/docs/Redis.md @@ -116,6 +116,11 @@ Typical configurations Here's typical wal-g configuration for that case: ```bash -WALG_STREAM_CREATE_COMMAND: 'redis-cli --rdb /dev/stdout' +WALG_STREAM_CREATE_COMMAND: 'redis_cli --rdb /dev/stdout' WALG_STREAM_RESTORE_COMMAND: 'cat > /var/lib/redis/dump.rdb' ``` + +### Why we made redis_cli.sh +redis-cli fails with error when redis version >= 6.2, so we made this workaround + +If you use redis >= 6.2, use [redis_cli.sh](https://github.com/wal-g/wal-g/blob/master/redis_cli.sh) and replace redis-cli in WALG_STREAM_CREATE_COMMAND \ No newline at end of file diff --git a/redis_cli.sh b/redis_cli.sh new file mode 100755 index 000000000..fd861f940 --- /dev/null +++ b/redis_cli.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# All args passed to this script will be passed to "redis-cli .. --rdb -" +# +# This script - workaround with redis-cli >= 6.2 bug, when redis-cli tries to fsync /dev/stdout and exit with error +# see https://github.com/redis/redis/pull/9135 +# + +FILENAME=$(mktemp --suffix=redis-cli-stderr) +redis-cli $@ --rdb - 2>$FILENAME +exit_code=$? +cat $FILENAME >&2 +grep "Fail to fsync" $FILENAME | grep -q "Invalid argument" +FAILED=$? +rm $FILENAME; +if [[ $exit_code -ne 0 ]] ; then + test 0 -eq $FAILED || exit $exit_code +fi +exit 0 diff --git a/tests_func/Dockerfile.redis b/tests_func/Dockerfile.redis index d50886a33..4e9a9a9c3 100644 --- a/tests_func/Dockerfile.redis +++ b/tests_func/Dockerfile.redis @@ -89,6 +89,8 @@ RUN mkdir -p /var/log/wal-g/ ${TMP_DIR} && \ ln --force -s /config/gpg-key.armor ${WALG_CONF_DIR}/gpg-key.armor COPY staging/wal-g ${WALG_REPO}/ +COPY staging/wal-g/redis_cli.sh /usr/bin/redis_cli.sh + WORKDIR ${WALG_REPO} RUN make link_external_deps diff --git a/tests_func/images/redis/config/wal-g-5.0.8.json b/tests_func/images/redis/config/wal-g-5.0.8.json index ab175f5a1..0267afa6e 100644 --- a/tests_func/images/redis/config/wal-g-5.0.8.json +++ b/tests_func/images/redis/config/wal-g-5.0.8.json @@ -5,7 +5,7 @@ "AWS_ENDPOINT": "http://minio:9000", "AWS_S3_FORCE_PATH_STYLE": "true", - "WALG_STREAM_CREATE_COMMAND": "redis-cli --rdb /dev/stdout", + "WALG_STREAM_CREATE_COMMAND": "/usr/bin/redis_cli.sh --rdb /dev/stdout", "WALG_STREAM_RESTORE_COMMAND": "cat > /data/dump.rdb", "WALG_SENTINEL_USER_DATA": "{\"key\": \"value\"}", diff --git a/tests_func/images/redis/config/wal-g-6.0.8.json b/tests_func/images/redis/config/wal-g-6.0.8.json index ab175f5a1..0267afa6e 100644 --- a/tests_func/images/redis/config/wal-g-6.0.8.json +++ b/tests_func/images/redis/config/wal-g-6.0.8.json @@ -5,7 +5,7 @@ "AWS_ENDPOINT": "http://minio:9000", "AWS_S3_FORCE_PATH_STYLE": "true", - "WALG_STREAM_CREATE_COMMAND": "redis-cli --rdb /dev/stdout", + "WALG_STREAM_CREATE_COMMAND": "/usr/bin/redis_cli.sh --rdb /dev/stdout", "WALG_STREAM_RESTORE_COMMAND": "cat > /data/dump.rdb", "WALG_SENTINEL_USER_DATA": "{\"key\": \"value\"}", diff --git a/tests_func/images/redis/config/wal-g-6.2.4.json b/tests_func/images/redis/config/wal-g-6.2.4.json new file mode 100644 index 000000000..0267afa6e --- /dev/null +++ b/tests_func/images/redis/config/wal-g-6.2.4.json @@ -0,0 +1,21 @@ +{ + "WALG_S3_PREFIX": "s3://dbaas/redis-backup/test_uuid/test_redis", + "AWS_ACCESS_KEY_ID": "S3_ACCESS_KEY", + "AWS_SECRET_ACCESS_KEY": "S3_SECRET_KEY", + "AWS_ENDPOINT": "http://minio:9000", + "AWS_S3_FORCE_PATH_STYLE": "true", + + "WALG_STREAM_CREATE_COMMAND": "/usr/bin/redis_cli.sh --rdb /dev/stdout", + "WALG_STREAM_RESTORE_COMMAND": "cat > /data/dump.rdb", + + "WALG_SENTINEL_USER_DATA": "{\"key\": \"value\"}", + + "WALG_PGP_KEY_PATH": "/etc/wal-g/gpg-key.armor", + "WALG_COMPRESSION_METHOD": "brotli", + "WALG_DISK_RATE_LIMIT": "41943040", + "WALG_NETWORK_RATE_LIMIT": "10485760", + "WALG_LOG_LEVEL": "DEVEL", + "WALG_UPLOAD_CONCURRENCY" : 1, + + "WALG_REDIS_PASSWORD": "password" +} diff --git a/tests_func/redis_steps.go b/tests_func/redis_steps.go index 632caaec9..2b215cde7 100644 --- a/tests_func/redis_steps.go +++ b/tests_func/redis_steps.go @@ -129,6 +129,10 @@ func (tctx *TestContext) testEqualRedisDataAtHosts(host1, host2 string) error { return keys2.Err() } + if len(keys1.Val()) == 0 || len(keys2.Val()) == 0 { + return fmt.Errorf("keys1 or keys2 is empty - broken backup") + } + if !utils.IsArraysEqual(keys1.Val(), keys2.Val()) { return fmt.Errorf("keys from redis1/redis2 aren't equal") }