From de332999c7b7c9a19264acda342e911f72f652fa Mon Sep 17 00:00:00 2001 From: Evan Blaudy Date: Wed, 9 Aug 2023 15:15:10 +0200 Subject: [PATCH 1/2] [qa][cli] for dump_database add a flag to store or not --- zou/app/services/backup_service.py | 9 +++------ zou/app/utils/commands.py | 19 +++++++++++++++++-- zou/cli.py | 5 +++-- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/zou/app/services/backup_service.py b/zou/app/services/backup_service.py index 16dc94a22f..839bf0fe50 100644 --- a/zou/app/services/backup_service.py +++ b/zou/app/services/backup_service.py @@ -27,12 +27,10 @@ ) -def generate_db_backup(host, port, user, password, database): +def generate_db_backup(host, port, user, password, database, filename): """ Generate a Postgres dump file from the database. """ - now = datetime.datetime.now().strftime("%Y-%m-%d") - filename = "%s-zou-db-backup.sql.gz" % now cmd = ["pg_dump", "-h", host, "-p", port, "-U", user, database] with gzip.open(filename, "wb") as f: popen = subprocess.Popen( @@ -48,11 +46,10 @@ def generate_db_backup(host, port, user, password, database): popen.stdout.close() popen.wait() - print(f"Postgres dump created ({filename}).") return filename -def store_db_backup(filename): +def store_db_backup(filename, path): """ Store given file located in the same directory, inside the files bucket using the `dbbackup` prefix. @@ -60,7 +57,7 @@ def store_db_backup(filename): from zou.app import app with app.app_context(): - file_store.add_file("dbbackup", filename, filename) + file_store.add_file("dbbackup", filename, path) def upload_preview_files_to_storage(days=None): diff --git a/zou/app/utils/commands.py b/zou/app/utils/commands.py index 3207aba2ba..a149191724 100644 --- a/zou/app/utils/commands.py +++ b/zou/app/utils/commands.py @@ -3,6 +3,7 @@ import os import json import datetime +import tempfile from ldap3 import Server, Connection, ALL, NTLM, SIMPLE @@ -525,15 +526,29 @@ def download_file_from_storage(): sync_service.download_preview_files_from_storage() -def dump_database(): +def dump_database(store=False): + now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + filename = f"zou-db-backup-{now}.sql.gz" + if store: + filename = os.path.join(tempfile.gettempdir(), filename) + filename = backup_service.generate_db_backup( app.config["DATABASE"]["host"], app.config["DATABASE"]["port"], app.config["DATABASE"]["username"], app.config["DATABASE"]["password"], app.config["DATABASE"]["database"], + filename, ) - backup_service.store_db_backup(filename) + + if store: + backup_service.store_db_backup(os.path.basename(filename), filename) + os.remove(filename) + print( + f"Postgres dump added to store (dbbackup/{os.path.basename(filename)})." + ) + else: + print(f"Postgres dump created ({os.path.realpath(filename)}).") def upload_files_to_cloud_storage(days): diff --git a/zou/cli.py b/zou/cli.py index 0bc17b298f..fa80f139ae 100755 --- a/zou/cli.py +++ b/zou/cli.py @@ -395,12 +395,13 @@ def download_storage_files(): @cli.command() -def dump_database(): +@click.option("--store", is_flag=True) +def dump_database(store=False): """ Dump database described in Zou environment variables and save it to configured object storage. """ - commands.dump_database() + commands.dump_database(store) @cli.command() From 0d10a4ab397ce876bed8e5efcf7dd5bfa8d81345 Mon Sep 17 00:00:00 2001 From: Evan Blaudy Date: Wed, 9 Aug 2023 15:15:38 +0200 Subject: [PATCH 2/2] [qa] use utcnow() instead of utc() --- zou/app/utils/date_helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zou/app/utils/date_helpers.py b/zou/app/utils/date_helpers.py index f2600ff6d6..47e9b59a88 100644 --- a/zou/app/utils/date_helpers.py +++ b/zou/app/utils/date_helpers.py @@ -66,7 +66,7 @@ def get_year_interval(year): Get a tuple containing start date and end date for given year. """ year = int(year) - if year > datetime.now().year or year < 2010: + if year > datetime.utcnow().year or year < 2010: raise WrongDateFormatException start = datetime(year, 1, 1) @@ -80,7 +80,7 @@ def get_month_interval(year, month): """ year = int(year) month = int(month) - if year > datetime.now().year or year < 2010 or month < 1 or month > 12: + if year > datetime.utcnow().year or year < 2010 or month < 1 or month > 12: raise WrongDateFormatException start = datetime(year, month, 1) @@ -94,7 +94,7 @@ def get_week_interval(year, week): """ year = int(year) week = int(week) - if year > datetime.now().year or year < 2010 or week < 1 or week > 52: + if year > datetime.utcnow().year or year < 2010 or week < 1 or week > 52: raise WrongDateFormatException start = isoweek.Week(year, week).monday() end = start + relativedelta.relativedelta(days=7) @@ -109,7 +109,7 @@ def get_day_interval(year, month, day): month = int(month) day = int(day) if ( - year > datetime.now().year + year > datetime.utcnow().year or year < 2010 or month < 1 or month > 12