feat: postgresql support pg_dump and pg_dumpall

addons/postgresql/dataprotection/pgdump.sh

@@ -0,0 +1,138 @@
+# shellcheck disable=SC2148
+# pg_dump extracts a PostgreSQL database into a script file or other archive file
+# more info: https://www.postgresql.org/docs/current/app-pgdump.html
+set -e
+set -o pipefail
+export PATH="$PATH:$DP_DATASAFED_BIN_PATH"
+export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH"
+export PGPASSWORD=${DP_DB_PASSWORD}
+
+trap handle_exit EXIT
+
+# Construct pg_dump options string based on input parameters.
+construct_pg_dump_options() {
+  PG_DUMP_OPTIONS=""
+
+  # schemas, comma-separated
+  if [ -n "${schemas}" ]; then
+    for schema in ${schemas//,/ }; do
+      PG_DUMP_OPTIONS+=" --schema=${schema}"
+    done
+  fi
+  # exclude schemas, comma-separated
+  if [ -n "${excludeSchemas}" ]; then
+    for schema in ${excludeSchemas//,/ }; do
+      PG_DUMP_OPTIONS+=" --exclude-schema=${schema}"
+    done
+  fi
+  # tables, comma-separated
+  if [ -n "${tables}" ]; then
+    for table in ${tables//,/ }; do
+      PG_DUMP_OPTIONS+=" --table=${table}"
+    done
+  fi
+  # exclude tables, comma-separated
+  if [ -n "${excludeTables}" ]; then
+    for table in ${excludeTables//,/ }; do
+      PG_DUMP_OPTIONS+=" --exclude-table=${table}"
+    done
+  fi
+
+    # Format, the dafault is tar format
+  if [ "${format}" = "p" ] || [ "${format}" = "plain" ]; then
+    PG_DUMP_OPTIONS+=" -Fp"
+  else
+    PG_DUMP_OPTIONS+=" -Ft"
+  fi
+  if [ -n "${dataOnly}" ] && [ "${dataOnly}" = "true" ]; then
+    # boolean, whether to dump only data
+    PG_DUMP_OPTIONS+=" --data-only"
+  fi
+  if [ -n "${schemaOnly}" ] && [ "${schemaOnly}" = "true" ]; then
+    # boolean, whether to dump only schema
+    PG_DUMP_OPTIONS+=" --schema-only"
+  fi
+  if [ -n "${jobs}" ]; then
+    # number of jobs to run in parallel
+    PG_DUMP_OPTIONS+=" --jobs=${jobs}"
+  fi
+  if [ -n "${setRole}" ]; then
+    # role to set before excuting
+    PG_DUMP_OPTIONS+=" --role=${setRole}"
+  fi
+  if [ -n "${disableTriggers}" ] && [ "${disableTriggers}" = "true" ]; then
+    # boolean, whether to disable triggers
+    PG_DUMP_OPTIONS+=" --disable-triggers"
+  fi
+  if [ -n "${useInserts}" ] && [ "${useInserts}" = "true" ]; then
+    # boolean, whether to use INSERT statements
+    PG_DUMP_OPTIONS+=" --inserts"
+  fi
+  if [ -n "${columnInserts}" ] && [ "${columnInserts}" = "true" ]; then
+    # boolean, whether to use column names in INSERT statements
+    PG_DUMP_OPTIONS+=" --column-inserts"
+  fi
+  if [ -n "${onConflictDoNothing}" ] && [ "${onConflictDoNothing}" = "true" ]; then
+    # boolean, whether to use ON CONFLICT DO NOTHING
+    PG_DUMP_OPTIONS+=" --on-conflict-do-nothing"
+  fi
+  if [ -n "${loadViaPartitionRoot}" ] && [ "${loadViaPartitionRoot}" = "true" ]; then
+    # boolean, whether to load via partition root
+    PG_DUMP_OPTIONS+=" --load-via-partition-root"
+  fi
+  if [ -n "${noComments}" ] && [ "${noComments}" = "true" ]; then
+    # boolean, whether to exclude comments
+    PG_DUMP_OPTIONS+=" --no-comments"
+  fi
+  if [ -n "${noTablespaces}" ] && [ "${noTablespaces}" = "true" ]; then
+    # boolean, whether to exclude tablespaces
+    PG_DUMP_OPTIONS+=" --no-tablespaces"
+  fi
+  if [ -n "${noUnloggedTableData}" ] && [ "${noUnloggedTableData}" = "true" ]; then
+    # boolean, whether to exclude unlogged table data
+    PG_DUMP_OPTIONS+=" --no-unlogged-table-data"
+  fi
+  if [ -n "${noBlobs}" ] && [ "${noBlobs}" = "true" ]; then
+    # boolean, whether to exclude blobs
+    PG_DUMP_OPTIONS+=" --no-blobs"
+  fi
+  if [ -n "${encoding}" ] && [ "${encoding}" != "UTF-8" ]; then
+    # encoding to use
+    PG_DUMP_OPTIONS+=" --encoding=${encoding}"
+  fi
+  PG_DUMP_OPTIONS+=" --clean"
+  PG_DUMP_OPTIONS+=" --if-exists"
+  PG_DUMP_OPTIONS+=" --no-owner"
+  PG_DUMP_OPTIONS+=" --no-privileges"
+  PG_DUMP_OPTIONS+=" --verbose"
+  echo "${PG_DUMP_OPTIONS}"
+}
+
+# Construct a file name based on $format environment variable
+file_name() {
+  local prefix=${DP_BACKUP_NAME}
+  if [ "${format}" = "p" ] || [ "${format}" = "plain" ]; then
+    echo "${prefix}.sql"
+  else
+    echo "${prefix}.tar"
+  fi
+}
+
+
+START_TIME=`get_current_time`
+
+if [ -z "${database}" ]; then
+  echo "no database specified"
+  echo "use the default database: postgres"
+  database="postgres"
+fi
+
+PG_DUMP_OPTIONS="-d ${database}$(construct_pg_dump_options)"
+# print options
+echo "pg_dump options: ${PG_DUMP_OPTIONS}"
+pg_dump -U ${DP_DB_USER} -h ${DP_DB_HOST} -p ${DP_DB_PORT} ${PG_DUMP_OPTIONS} | datasafed push -z zstd-fastest - "/$(file_name).zst"
+# log database info
+echo "dbname: ${database}" | datasafed push - "/dump.info"
+# stat and save the backup information
+stat_and_save_backup_info "$START_TIME"
+echo "backup done!";

addons/postgresql/dataprotection/pgdumpall-backup.sh

@@ -0,0 +1,147 @@
+# shellcheck disable=SC2148
+# pg_dumpall extracts the entire PostgreSQL database instance or multiple databases into a script file
+# more info: https://www.postgresql.org/docs/current/app-pgdumpall.html
+set -e
+set -o pipefail
+export PATH="$PATH:$DP_DATASAFED_BIN_PATH"
+export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH"
+export PGPASSWORD=${DP_DB_PASSWORD}
+
+trap handle_exit EXIT
+
+# Construct pg_dump options string based on input parameters.
+construct_pg_dumpall_options() {
+  PG_DUMPALL_OPTIONS=""
+
+  # General options
+  if [ -n "${lockWaitTimeout}" ]; then
+    PG_DUMPALL_OPTIONS+=" --lock-wait-timeout=${lockWaitTimeout}"
+  fi
+
+  # Options controlling the output content
+  if [ -n "${dataOnly}" ] && [ "${dataOnly}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --data-only"
+  fi
+  if [ -n "${schemaOnly}" ] && [ "${schemaOnly}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --schema-only"
+  fi
+  if [ -n "${clean}" ] && [ "${clean}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --clean"
+  fi
+  if [ -n "${encoding}" ]; then
+    PG_DUMPALL_OPTIONS+=" --encoding=${encoding}"
+  fi
+  if [ -n "${globalsOnly}" ] && [ "${globalsOnly}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --globals-only"
+  fi
+  if [ -n "${noOwner}" ] && [ "${noOwner}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --no-owner"
+  fi
+  if [ -n "${rolesOnly}" ] && [ "${rolesOnly}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --roles-only"
+  fi
+  if [ -n "${superuser}" ]; then
+    PG_DUMPALL_OPTIONS+=" --superuser=${superuser}"
+  fi
+  if [ -n "${tablespacesOnly}" ] && [ "${tablespacesOnly}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --tablespaces-only"
+  fi
+  if [ -n "${noPrivileges}" ] && [ "${noPrivileges}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --no-privileges"
+  fi
+
+  # Additional options
+  if [ -n "${binaryUpgrade}" ] && [ "${binaryUpgrade}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --binary-upgrade"
+  fi
+  if [ -n "${columnInserts}" ] && [ "${columnInserts}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --column-inserts"
+  fi
+  if [ -n "${disableDollarQuoting}" ] && [ "${disableDollarQuoting}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --disable-dollar-quoting"
+  fi
+  if [ -n "${disableTriggers}" ] && [ "${disableTriggers}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --disable-triggers"
+  fi
+  if [ -n "${excludeDatabase}" ]; then
+    for db in ${excludeDatabase//,/ }; do
+      PG_DUMPALL_OPTIONS+=" --exclude-database=${db}"
+    done
+  fi
+  if [ -n "${extraFloatDigits}" ]; then
+    PG_DUMPALL_OPTIONS+=" --extra-float-digits=${extraFloatDigits}"
+  fi
+  if [ -n "${ifExists}" ] && [ "${ifExists}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --if-exists"
+  fi
+  if [ -n "${inserts}" ] && [ "${inserts}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --inserts"
+  fi
+  if [ -n "${loadViaPartitionRoot}" ] && [ "${loadViaPartitionRoot}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --load-via-partition-root"
+  fi
+  if [ -n "${noComments}" ] && [ "${noComments}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --no-comments"
+  fi
+  if [ -n "${noPublications}" ] && [ "${noPublications}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --no-publications"
+  fi
+  if [ -n "${noRolePasswords}" ] && [ "${noRolePasswords}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --no-role-passwords"
+  fi
+  if [ -n "${noSecurityLabels}" ] && [ "${noSecurityLabels}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --no-security-labels"
+  fi
+  if [ -n "${noSubscriptions}" ] && [ "${noSubscriptions}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --no-subscriptions"
+  fi
+  if [ -n "${noSync}" ] && [ "${noSync}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --no-sync"
+  fi
+    if [ -n "${noTableAccessMethod}" ] && [ "${noTableAccessMethod}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --no-table-access-method"
+  fi
+  if [ -n "${noTablespaces}" ] && [ "${noTablespaces}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --no-tablespaces"
+  fi
+  if [ -n "${noToastCompression}" ] && [ "${noToastCompression}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --no-toast-compression"
+  fi
+  if [ -n "${noUnloggedTableData}" ] && [ "${noUnloggedTableData}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --no-unlogged-table-data"
+  fi
+  if [ -n "${onConflictDoNothing}" ] && [ "${onConflictDoNothing}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --on-conflict-do-nothing"
+  fi
+  if [ -n "${quoteAllIdentifiers}" ] && [ "${quoteAllIdentifiers}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --quote-all-identifiers"
+  fi
+  if [ -n "${rowsPerInsert}" ]; then
+    PG_DUMPALL_OPTIONS+=" --rows-per-insert=${rowsPerInsert}"
+  fi
+  if [ -n "${useSetSessionAuthorization}" ] && [ "${useSetSessionAuthorization}" = "true" ]; then
+    PG_DUMPALL_OPTIONS+=" --use-set-session-authorization"
+  fi
+
+  # Connection options
+  if [ -n "${dbname}" ]; then
+    PG_DUMPALL_OPTIONS+=" --dbname=${dbname}"
+  fi
+  if [ -n "${database}" ]; then
+    PG_DUMPALL_OPTIONS+=" --database=${database}"
+  fi
+  if [ -n "${role}" ]; then
+    PG_DUMPALL_OPTIONS+=" --role=${role}"
+  fi
+  PG_DUMPALL_OPTIONS+=" --verbose"
+  echo "${PG_DUMPALL_OPTIONS}"
+}
+
+START_TIME=`get_current_time`
+PG_DUMPALL_OPTIONS=$(construct_pg_dumpall_options)
+# print options
+echo "pg_dumpall options: ${PG_DUMPALL_OPTIONS}"
+pg_dumpall -U ${DP_DB_USER} -h ${DP_DB_HOST} -p ${DP_DB_PORT} ${PG_DUMPALL_OPTIONS} | datasafed push -z zstd-fastest - "/${DP_BACKUP_NAME}.sql.zst"
+# stat and save the backup information
+stat_and_save_backup_info "$START_TIME"
+echo "backup done!";

addons/postgresql/dataprotection/pgdumpall-restore.sh

@@ -0,0 +1,21 @@
+# shellcheck disable=SC2148
+# use psql to restore databses from a script files created by pg_dumpall
+set -e
+set -o pipefail
+export PATH="$PATH:$DP_DATASAFED_BIN_PATH"
+export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH"
+export PGPASSWORD=${DP_DB_PASSWORD}
+function remote_file_exists() {
+    local out=$(datasafed list $1)
+    if [ "${out}" == "$1" ]; then
+        echo "true"
+        return
+    fi
+    echo "false"
+}
+
+if [ $(remote_file_exists "${DP_BACKUP_NAME}.sql.zst") == "true" ]; then
+  datasafed pull -d zstd-fastest "${DP_BACKUP_NAME}.sql.zst" - | psql -U ${DP_DB_USER} -h ${DP_DB_HOST} -p ${DP_DB_PORT}
+  echo "restore complete!";
+  exit 0
+fi