From dfaa34b92bf5071232854d27de319b39419e0d24 Mon Sep 17 00:00:00 2001 From: Evan Blaudy Date: Fri, 28 Jun 2024 16:20:05 +0200 Subject: [PATCH] [cli] add a command to renormalize a movie preview file --- zou/app/blueprints/previews/resources.py | 2 - zou/app/services/preview_files_service.py | 12 ++- zou/app/utils/commands.py | 91 ++++++++++++++++++++++- zou/cli.py | 44 +++++++++-- 4 files changed, 137 insertions(+), 12 deletions(-) diff --git a/zou/app/blueprints/previews/resources.py b/zou/app/blueprints/previews/resources.py index 353d948492..756ff02832 100644 --- a/zou/app/blueprints/previews/resources.py +++ b/zou/app/blueprints/previews/resources.py @@ -338,8 +338,6 @@ def save_movie_preview( preview_files_service.prepare_and_store_movie( preview_file_id, uploaded_movie_path, normalize=normalize ) - preview_file = files_service.get_preview_file(preview_file_id) - tasks_service.update_preview_file_info(preview_file) return preview_file_id def save_file_preview(self, instance_id, uploaded_file, extension): diff --git a/zou/app/services/preview_files_service.py b/zou/app/services/preview_files_service.py index a3e4fc4e04..5f3f4d2ec3 100644 --- a/zou/app/services/preview_files_service.py +++ b/zou/app/services/preview_files_service.py @@ -163,7 +163,10 @@ def set_preview_file_as_ready(preview_file_id): def prepare_and_store_movie( - preview_file_id, uploaded_movie_path, normalize=True + preview_file_id, + uploaded_movie_path, + normalize=True, + add_source_to_file_store=True, ): """ Prepare movie preview, normalize the movie as a .mp4, build the thumbnails @@ -196,13 +199,14 @@ def prepare_and_store_movie( if normalize: current_app.logger.info("start normalization") try: + if add_source_to_file_store: + file_store.add_movie( + "source", preview_file_id, uploaded_movie_path + ) if ( config.ENABLE_JOB_QUEUE_REMOTE and len(config.JOB_QUEUE_NOMAD_NORMALIZE_JOB) > 0 ): - file_store.add_movie( - "source", preview_file_id, uploaded_movie_path - ) result = _run_remote_normalize_movie( current_app, preview_file_id, fps, width, height ) diff --git a/zou/app/utils/commands.py b/zou/app/utils/commands.py index 1ebf7b4640..0031dc6009 100644 --- a/zou/app/utils/commands.py +++ b/zou/app/utils/commands.py @@ -3,11 +3,13 @@ import os import datetime import tempfile +import sys +import shutil from ldap3 import Server, Connection, ALL, NTLM, SIMPLE from zou.app.utils import thumbnail as thumbnail_utils, auth -from zou.app.stores import auth_tokens_store, file_store +from zou.app.stores import auth_tokens_store, file_store, queue_store from zou.app.services import ( assets_service, backup_service, @@ -23,6 +25,8 @@ tasks_service, ) from zou.app.models.person import Person +from zou.app.models.preview_file import PreviewFile +from zou.app.models.task import Task from sqlalchemy.sql.expression import not_ from zou.app.services.exception import ( @@ -717,3 +721,88 @@ def create_bot( is_bot=True, ) print(bot["access_token"]) + + +def renormalize_movie_preview_files( + preview_file_id=None, project_id=None, all_broken=None, all_processing=None +): + with app.app_context(): + if preview_file_id is None and not all_broken and not all_processing: + print("You must specify at least one option.") + sys.exit(1) + else: + query = PreviewFile.query.filter( + PreviewFile.extension == "mp4" + ).order_by(PreviewFile.created_at.asc()) + + if preview_file_id is not None: + query = query.filter(PreviewFile.id == preview_file_id) + + if project_id is not None: + query = query.join(Task).filter( + PreviewFile.project_id == project_id + ) + + if all_broken and all_processing: + query = query.filter( + PreviewFile.status.in_(("broken", "processing")) + ) + elif all_broken: + query = query.filter(PreviewFile.status == "broken") + elif all_processing: + query = query.filter(PreviewFile.status == "processing") + + preview_files = query.all() + len_preview_files = len(preview_files) + if len_preview_files == 0: + print("No preview files found.") + sys.exit(1) + else: + for i, preview_file in enumerate(preview_files): + try: + preview_file_id = str(preview_file.id) + print( + f"Renormalizing preview file {preview_file_id} ({i+1}/{len_preview_files})." + ) + extension = preview_file.extension + uploaded_movie_path = os.path.join( + config.TMP_DIR, + f"{preview_file_id}.{extension}.tmp", + ) + if config.FS_BACKEND == "local": + shutil.copyfile( + file_store.get_local_movie_path( + "source", preview_file_id + ), + uploaded_movie_path, + ) + else: + sync_service.download_file( + uploaded_movie_path, + "source", + file_store.open_movie, + str(preview_file_id), + ) + if config.ENABLE_JOB_QUEUE: + queue_store.job_queue.enqueue( + preview_files_service.prepare_and_store_movie, + args=( + preview_file_id, + uploaded_movie_path, + True, + False, + ), + job_timeout=int(config.JOB_QUEUE_TIMEOUT), + ) + else: + preview_files_service.prepare_and_store_movie( + preview_file_id, + uploaded_movie_path, + normalize=True, + add_source_to_file_store=False, + ) + except Exception as e: + print( + f"Renormalization of preview file {preview_file_id} failed: {e}" + ) + continue diff --git a/zou/cli.py b/zou/cli.py index 179bfd8970..60ae91f07d 100755 --- a/zou/cli.py +++ b/zou/cli.py @@ -456,13 +456,13 @@ def upload_files_to_cloud_storage(days): @cli.command() -@click.option("--projectid") -def clean_tasks_data(projectid): +@click.option("--project-id") +def clean_tasks_data(project_id): """ Reset task models data (retake count, wip start date and end date) """ - if projectid is not None: - commands.reset_tasks_data(projectid) + if project_id is not None: + commands.reset_tasks_data(project_id) @cli.command() @@ -569,7 +569,7 @@ def reset_breakdown_data(): @click.option("--email", required=True) @click.option("--name", required=True) @click.option( - "--expiration_date", + "--expiration-date", required=False, default=None, show_default=True, @@ -593,5 +593,39 @@ def create_bot( ) +@cli.command() +@click.option( + "--preview-file-id", + required=False, + default=None, + show_default=True, +) +@click.option( + "--project-id", + required=False, + default=None, + show_default=True, +) +@click.option("--all-broken", is_flag=True, default=False, show_default=True) +@click.option( + "--all-processing", is_flag=True, default=False, show_default=True +) +def renormalize_movie_preview_files( + preview_file_id, + project_id, + all_broken, + all_processing, +): + """ + Renormalize all preview files. + """ + commands.renormalize_movie_preview_files( + preview_file_id, + project_id, + all_broken, + all_processing, + ) + + if __name__ == "__main__": cli()