diff --git a/benchmarks/copy-composer/copy-composer.md b/benchmarks/copy-composer/copy-composer.md index aa42d788a..ff6f393b0 100644 --- a/benchmarks/copy-composer/copy-composer.md +++ b/benchmarks/copy-composer/copy-composer.md @@ -11,6 +11,9 @@ Backup storage: Yandex Object Storage Initial database size 100GB + +# Experiment without intermediate updates + ## Test DB and backup creation log ``` #!/bin/bash @@ -78,4 +81,83 @@ Tar size 500MB Tar size 1GB ![1GB](resources/net-1000000000.png) Tar size 5GB -![5GB](resources/net-5000000000.png) \ No newline at end of file +![5GB](resources/net-5000000000.png) + +# Experiment intermediate updates +## Test DB and backup creation log +``` +#!/bin/bash + +get_time () { + echo "$(date +%s)" +} + +PGUSER=postgres +SCALE=6690 + +for size in 100000000 500000000 1000000000 5000000000 +do +cat > /var/lib/postgresql/.walg.json << EOF +{ + "WALG_S3_PREFIX": "s3://walg/", + "AWS_ACCESS_KEY_ID": "yourkey", + "AWS_SECRET_ACCESS_KEY": "yourkey, + "AWS_ENDPOINT": "https://storage.yandexcloud.net", + "WALG_COMPRESSION_METHOD": "lz4", + "WALG_DELTA_MAX_STEPS": "5", + "WALG_TAR_SIZE_THRESHOLD": "$size", + "PGDATA": "/var/lib/postgresql/12/main", + "PGHOST": "/var/run/postgresql/.s.PGSQL.5432", + "AWS_REGION": "yourregion", + "PGUSER": "postgres", + "PGPASSWORD": "postgres" +} +EOF + chown postgres: /var/lib/postgresql/.walg.json + start_time=$(get_time) + log_name=log-start-${start_time}-scale-$SCALE-tar-$size-all + ./main/pg/wal-g --config=/var/lib/postgresql/.walg.json delete everything FORCE --confirm + echo "Restart postgres $(get_time)" >> $log_name.txt 2>&1 + service postgresql restart + echo "Generate tables start $(get_time)" >> $log_name.txt 2>&1 + pgbench -U $PGUSER -i -s $SCALE -n + echo "Generate tables finish $(get_time)" >> $log_name.txt 2>&1 + sleep 600 + echo "Plain push start $(get_time)" >> $log_name.txt 2>&1 + ./main/pg/wal-g --config=/var/lib/postgresql/.walg.json backup-push /var/lib/postgresql/12/main --full + echo "Plain push finish $(get_time)" >> $log_name.txt 2>&1 + sleep 600 + for transactions in 10 20 50 100 + do + pgbench -U $PGUSER -t $transactions + echo "Running $transactions transactions" >> $log_name.txt 2>&1 + echo "Copy push start $(get_time)" >> $log_name.txt 2>&1 + ./main/pg/wal-g --config=/var/lib/postgresql/.walg.json backup-push /var/lib/postgresql/12/main --full --copy-composer + echo "Copy push finish $(get_time)" >> $log_name.txt 2>&1 + sleep 600 + done +done +chmod 777 $log_name.txt +``` + +**CPU usage** + +Tar size 100MB +![100MB](resources/cpu-100000000-u.png) +Tar size 500MB +![500MB](resources/cpu-500000000-u.png) +Tar size 1GB +![1GB](resources/cpu-1000000000-u.png) +Tar size 5GB +![5GB](resources/cpu-5000000000-u.png) + +**Network usage** + +Tar size 100MB +![100MB](resources/net-100000000-u.png) +Tar size 500MB +![500MB](resources/net-500000000-u.png) +Tar size 1GB +![1GB](resources/net-1000000000-u.png) +Tar size 5GB +![5GB](resources/net-5000000000-u.png) \ No newline at end of file diff --git a/benchmarks/copy-composer/resources/cpu-100000000-u.png b/benchmarks/copy-composer/resources/cpu-100000000-u.png new file mode 100644 index 000000000..0b4f802ae Binary files /dev/null and b/benchmarks/copy-composer/resources/cpu-100000000-u.png differ diff --git a/benchmarks/copy-composer/resources/cpu-1000000000-u.png b/benchmarks/copy-composer/resources/cpu-1000000000-u.png new file mode 100644 index 000000000..c8e421b9c Binary files /dev/null and b/benchmarks/copy-composer/resources/cpu-1000000000-u.png differ diff --git a/benchmarks/copy-composer/resources/cpu-500000000-u.png b/benchmarks/copy-composer/resources/cpu-500000000-u.png new file mode 100644 index 000000000..1d8682e7b Binary files /dev/null and b/benchmarks/copy-composer/resources/cpu-500000000-u.png differ diff --git a/benchmarks/copy-composer/resources/cpu-5000000000-u.png b/benchmarks/copy-composer/resources/cpu-5000000000-u.png new file mode 100644 index 000000000..f47e2fcee Binary files /dev/null and b/benchmarks/copy-composer/resources/cpu-5000000000-u.png differ diff --git a/benchmarks/copy-composer/resources/net-100000000-u.png b/benchmarks/copy-composer/resources/net-100000000-u.png new file mode 100644 index 000000000..d7afb7774 Binary files /dev/null and b/benchmarks/copy-composer/resources/net-100000000-u.png differ diff --git a/benchmarks/copy-composer/resources/net-1000000000-u.png b/benchmarks/copy-composer/resources/net-1000000000-u.png new file mode 100644 index 000000000..fb6edf431 Binary files /dev/null and b/benchmarks/copy-composer/resources/net-1000000000-u.png differ diff --git a/benchmarks/copy-composer/resources/net-500000000-u.png b/benchmarks/copy-composer/resources/net-500000000-u.png new file mode 100644 index 000000000..a8c8da6e3 Binary files /dev/null and b/benchmarks/copy-composer/resources/net-500000000-u.png differ diff --git a/benchmarks/copy-composer/resources/net-5000000000-u.png b/benchmarks/copy-composer/resources/net-5000000000-u.png new file mode 100644 index 000000000..31bf3975c Binary files /dev/null and b/benchmarks/copy-composer/resources/net-5000000000-u.png differ