-
Notifications
You must be signed in to change notification settings - Fork 44
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore: support to continuous and incremental backup with the wal-g to…
…ol (#1456)
- Loading branch information
Showing
12 changed files
with
427 additions
and
127 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| #!/bin/bash | ||
| backup_base_path="$(dirname $DP_BACKUP_BASE_PATH)/wal-g" | ||
| export WALG_DATASAFED_CONFIG="" | ||
| export PATH="$PATH:$DP_DATASAFED_BIN_PATH" | ||
| export DATASAFED_BACKEND_BASE_PATH="$backup_base_path" | ||
|
|
||
| base_backup_list=$(datasafed list /basebackups_005 -d) | ||
| if [[ -z ${base_backup_list} ]]; then | ||
| datasafed rm -r /wal_005 | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| #!/bin/bash | ||
| set -e | ||
| backup_base_path="$(dirname $DP_BACKUP_BASE_PATH)/wal-g/" | ||
|
|
||
| function config_wal_g_for_fetch_wal_log() { | ||
| walg_dir=${VOLUME_DATA_DIR}/wal-g | ||
| walg_env=${walg_dir}/restore-env | ||
| mkdir -p ${walg_dir}/restore-env | ||
| cp /etc/datasafed/datasafed.conf ${walg_dir}/datasafed.conf | ||
| cp /usr/bin/wal-g ${walg_dir}/wal-g | ||
| datasafed_base_path=${1:?missing datasafed_base_path} | ||
| # config wal-g env | ||
| # config WALG_PG_WAL_SIZE with wal_segment_size which fetched by psql | ||
| # echo "" > ${walg_env}/WALG_PG_WAL_SIZE | ||
| echo "${walg_dir}/datasafed.conf" > ${walg_env}/WALG_DATASAFED_CONFIG | ||
| echo "${datasafed_base_path}" > ${walg_env}/DATASAFED_BACKEND_BASE_PATH | ||
| echo "zstd" > ${walg_env}/WALG_COMPRESSION_METHOD | ||
| } | ||
|
|
||
| # 1. config restore script | ||
| touch ${DATA_DIR}/recovery.signal; | ||
| mkdir -p ${RESTORE_SCRIPT_DIR} && chmod 777 -R ${RESTORE_SCRIPT_DIR}; | ||
| echo "#!/bin/bash" > ${RESTORE_SCRIPT_DIR}/kb_restore.sh; | ||
| echo "[[ -d '${DATA_DIR}.old' ]] && mv -f ${DATA_DIR}.old/* ${DATA_DIR}/;" >> ${RESTORE_SCRIPT_DIR}/kb_restore.sh; | ||
| echo "sync;" >> ${RESTORE_SCRIPT_DIR}/kb_restore.sh; | ||
| chmod +x ${RESTORE_SCRIPT_DIR}/kb_restore.sh; | ||
|
|
||
| # 2. config wal-g to fetch wal logs | ||
| config_wal_g_for_fetch_wal_log "${backup_base_path}" | ||
|
|
||
| # 3. config restore command | ||
| mkdir -p ${CONF_DIR} && chmod 777 -R ${CONF_DIR}; | ||
| WALG_DIR=/home/postgres/pgdata/wal-g | ||
|
|
||
| restore_command_str="envdir ${WALG_DIR}/restore-env ${WALG_DIR}/wal-g wal-fetch %f %p >> ${RESTORE_SCRIPT_DIR}/wal-g.log 2>&1" | ||
| cat << EOF > "${CONF_DIR}/recovery.conf" | ||
| restore_command='${restore_command_str}' | ||
| recovery_target_time='$( date -d "@${DP_RESTORE_TIMESTAMP}" '+%F %T%::z' )' | ||
| recovery_target_action='promote' | ||
| recovery_target_timeline='latest' | ||
| EOF | ||
| # this step is necessary, data dir must be empty for patroni | ||
| mv ${DATA_DIR} ${DATA_DIR}.old | ||
| sync |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| #!/bin/bash | ||
| backup_base_path="$(dirname $DP_BACKUP_BASE_PATH)/wal-g/wal_005" | ||
| export WALG_DATASAFED_CONFIG="" | ||
| export PATH="$PATH:$DP_DATASAFED_BIN_PATH" | ||
| export WALG_COMPRESSION_METHOD=zstd | ||
| export PGPASSWORD=${DP_DB_PASSWORD} | ||
| export DATASAFED_BACKEND_BASE_PATH=${backup_base_path} | ||
| export KB_BACKUP_WORKDIR=${VOLUME_DATA_DIR}/kb-backup | ||
| GLOBAL_OLD_SIZE=0 | ||
|
|
||
| # get start time of the wal log | ||
| function get_wal_log_start_time() { | ||
| local file="${1:?missing wal log name to analyze}" | ||
| local START_TIME=$(pg_waldump $file --rmgr=Transaction 2>/dev/null | grep 'desc: COMMIT' |head -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') | ||
| if [[ ! -z ${START_TIME} ]];then | ||
| START_TIME=$(date -d "${START_TIME}" -u '+%Y-%m-%dT%H:%M:%SZ') | ||
| echo $START_TIME | ||
| fi | ||
| } | ||
|
|
||
| # pull wal log and decompress to KB_BACKUP_WORKDIR dir | ||
| function pull_wal_log() { | ||
| file="${1:?missing file name to pull}" | ||
| # pull and decompress | ||
| fileName=$(basename ${file}) | ||
| datasafed pull -d zstd ${file} "$(DP_get_file_name_without_ext ${fileName})" | ||
| } | ||
|
|
||
| function get_wal_log_end_time() { | ||
| wal_file="${1:?missing file name to pull}" | ||
| mkdir -p ${KB_BACKUP_WORKDIR} && cd ${KB_BACKUP_WORKDIR} | ||
| pull_wal_log ${wal_file} | ||
| wal_file_name=$(DP_get_file_name_without_ext `basename ${wal_file}`) | ||
| local END_TIME=$(pg_waldump $wal_file_name --rmgr=Transaction 2>/dev/null | grep 'desc: COMMIT' |tail -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') | ||
| if [[ ! -z ${END_TIME} ]];then | ||
| END_TIME=$(date -d "${END_TIME}" -u '+%Y-%m-%dT%H:%M:%SZ') | ||
| echo $END_TIME | ||
| fi | ||
| rm -rf $wal_file_name | ||
| } | ||
|
|
||
|
|
||
| # save backup status info to sync file | ||
| function save_backup_status() { | ||
| local TOTAL_SIZE=$(datasafed stat / | grep TotalSize | awk '{print $2}') | ||
| # if no size changes, return | ||
| if [[ -z ${TOTAL_SIZE} || ${TOTAL_SIZE} -eq 0 || ${TOTAL_SIZE} == ${GLOBAL_OLD_SIZE} ]];then | ||
| return | ||
| fi | ||
| GLOBAL_OLD_SIZE=${TOTAL_SIZE} | ||
| local wal_files=$(datasafed list -f --recursive / -o json | jq -s -r '.[] | sort_by(.mtime) |.[] |.path') | ||
| local OLDEST_FILE=$(echo $wal_files | tr ' ' '\n' |head -n 1) | ||
| local LATEST_FILE=$(echo $wal_files | tr ' ' '\n' |tail -n 1) | ||
| local START_TIME= | ||
| local END_TIME= | ||
| if [ ! -z ${OLDEST_FILE} ]; then | ||
| START_TIME=$(DP_analyze_start_time_from_datasafed ${OLDEST_FILE} get_wal_log_start_time pull_wal_log) | ||
| fi | ||
| if [ ! -z ${LATEST_FILE} ]; then | ||
| END_TIME=$(get_wal_log_end_time ${LATEST_FILE}) | ||
| fi | ||
| DP_log "start time of the oldest wal: ${START_TIME}, end time of the latest wal: ${END_TIME}, total size: ${TOTAL_SIZE}" | ||
| DP_save_backup_status_info "${TOTAL_SIZE}" "${START_TIME}" "${END_TIME}" | ||
| } | ||
|
|
||
| # trap term signal | ||
| trap "echo 'Terminating...' && exit 0" TERM | ||
| DP_log "start to collect wal infos" | ||
| while true; do | ||
| save_backup_status | ||
| sleep ${LOG_ARCHIVE_SECONDS} | ||
| done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,34 +1,27 @@ | ||
| # shellcheck disable=SC2148 | ||
| #!/bin/bash | ||
| set -e | ||
|
|
||
| postgres_log_dir="${VOLUME_DATA_DIR}/logs" | ||
| postgres_scripts_log_file="${postgres_log_dir}/scripts.log" | ||
| setup_logging WALG_CONFIG "${postgres_scripts_log_file}" | ||
|
|
||
| if [[ ! -f /etc/datasafed/datasafed.conf ]]; then | ||
| echo "ERROR: backupRepo should use Tool accessMode" | ||
| exit 1 | ||
| echo "ERROR: backupRepo should use Tool accessMode" | ||
| exit 1 | ||
| fi | ||
|
|
||
| function config_wal_g() { | ||
| walg_dir="${VOLUME_DATA_DIR}/wal-g" | ||
| walg_env="${walg_dir}/env" | ||
| mkdir -p "${walg_env}" | ||
| cp /etc/datasafed/datasafed.conf "${walg_dir}/datasafed.conf" | ||
| cp /usr/bin/wal-g "${walg_dir}/wal-g" | ||
| walg_dir=${VOLUME_DATA_DIR}/wal-g | ||
| walg_env=${walg_dir}/env | ||
| mkdir -p ${walg_dir}/env | ||
| cp /etc/datasafed/datasafed.conf ${walg_dir}/datasafed.conf | ||
| cp /usr/bin/wal-g ${walg_dir}/wal-g | ||
| datasafed_base_path=${1:?missing datasafed_base_path} | ||
| # config wal-g env | ||
| # config WALG_PG_WAL_SIZE with wal_segment_size which fetched by psql | ||
| # echo "" > ${walg_env}/WALG_PG_WAL_SIZE | ||
| echo "${walg_dir}/datasafed.conf" > "${walg_env}/WALG_DATASAFED_CONFIG" | ||
| echo "${datasafed_base_path}" > "${walg_env}/DATASAFED_BACKEND_BASE_PATH" | ||
| echo "true" > "${walg_env}/PG_READY_RENAME" | ||
| echo "zstd" > "${walg_env}/WALG_COMPRESSION_METHOD" | ||
| echo "${walg_dir}/datasafed.conf" > ${walg_env}/WALG_DATASAFED_CONFIG | ||
| echo "${datasafed_base_path}" > ${walg_env}/DATASAFED_BACKEND_BASE_PATH | ||
| echo "true" > ${walg_env}/PG_READY_RENAME | ||
| echo "zstd" > ${walg_env}/WALG_COMPRESSION_METHOD | ||
| } | ||
|
|
||
| echo "Configure WAL-G : BEGIN" | ||
| config_wal_g "$(dirname "$(dirname "${DP_BACKUP_BASE_PATH}")")/wal-g" | ||
| echo "Configure WAL-G : DONE" | ||
|
|
||
| config_wal_g "$(dirname $(dirname $DP_BACKUP_BASE_PATH))/wal-g" | ||
| echo "{}" >"${DP_BACKUP_INFO_FILE}" | ||
| sync | ||
| sync |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,54 +1,45 @@ | ||
| # shellcheck disable=SC2148 | ||
|
|
||
| postgres_log_dir="${VOLUME_DATA_DIR}/logs" | ||
| postgres_scripts_log_file="${postgres_log_dir}/scripts.log" | ||
| setup_logging WALG_DELETE "${postgres_scripts_log_file}" | ||
|
|
||
| #!/bin/bash | ||
| export WALG_DATASAFED_CONFIG="" | ||
| export PATH="$PATH:$DP_DATASAFED_BIN_PATH" | ||
| export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH" | ||
|
|
||
| function getWalGSentinelInfo() { | ||
| local sentinelFile | ||
| local out | ||
| sentinelFile=${1} | ||
| out=$(datasafed list "${sentinelFile}") | ||
| local sentinelFile=${1} | ||
| local out=$(datasafed list ${sentinelFile}) | ||
| if [ "${out}" == "${sentinelFile}" ]; then | ||
| datasafed pull "${sentinelFile}" "${sentinelFile}" | ||
| cat "${sentinelFile}" | ||
| datasafed pull "${sentinelFile}" ${sentinelFile} | ||
| echo "$(cat ${sentinelFile})" | ||
| return | ||
| fi | ||
| } | ||
|
|
||
| # 1. get backup repo path of wal-g | ||
| backupRepoPath=$(getWalGSentinelInfo "wal-g-backup-repo.path") | ||
| if [[ -z "${backupRepoPath}" ]]; then | ||
| if [[ -z ${backupRepoPath} ]]; then | ||
| echo "INFO: nothing to delete." | ||
| exit 0 | ||
| fi | ||
|
|
||
| # 2. get backup name of this backup | ||
| backupName=$(getWalGSentinelInfo "wal-g-backup-name") | ||
| if [[ -z "${backupName}" ]]; then | ||
| if [[ -z ${backupName} ]]; then | ||
| echo "INFO: delete unsuccessfully backup files and outdated WAL archive." | ||
| export DATASAFED_BACKEND_BASE_PATH=${backupRepoPath} | ||
| wal-g delete garbage --confirm | ||
| exit 0 | ||
| fi | ||
|
|
||
| # 3. cleanup outdated wal logs, only effective when existing at least one full backup | ||
| echo "cleanup garbage archives in backup repo" | ||
| export DATASAFED_BACKEND_BASE_PATH="${backupRepoPath}" | ||
| export DATASAFED_BACKEND_BASE_PATH=${backupRepoPath} | ||
| wal-g delete garbage ARCHIVES | ||
|
|
||
| # 4. delete wal-g | ||
| echo "cleanup garbage full backups in backup repo" | ||
| dpBackupFilesCount=$(datasafed list --name "${backupName}_dp_*" /basebackups_005 | wc -l) | ||
| if [[ "${dpBackupFilesCount}" -le 1 ]]; then | ||
| if [[ ${dpBackupFilesCount} -le 1 ]]; then | ||
| # if this base backup only belongs to a backup CR, delete it. | ||
| echo "INFO: delete ${backupName}" | ||
| wal-g delete target "${backupName}" --confirm && wal-g delete garbage ARCHIVES --confirm | ||
| echo "INFO: delete ${backupName}, backupRepo: ${backupRepo}" | ||
| wal-g delete target ${backupName} --confirm && wal-g delete garbage ARCHIVES --confirm | ||
| fi | ||
| datasafed rm "/basebackups_005/${backupName}_dp_${DP_BACKUP_NAME}" | ||
| echo "backup cleanup DONE" | ||
| sync | ||
|
|
||
|
|
Oops, something went wrong.