Merge pull request #707 from EvanBldy/master

Add a command to generate tiles and update preview files metadata in the same time

tests/user/test_route_context.py

@@ -313,9 +313,7 @@ def test_get_filter_groups(self):
         self.assertEqual(len(result["asset"][project_id]), 1)
         self.assertEqual(len(result["shot"][project_id]), 1)
         self.assertEqual(len(result["all"][project_id]), 1)
-        self.assertEqual(
-            result["asset"][project_id][0]["name"], "g1"
-        )
+        self.assertEqual(result["asset"][project_id][0]["name"], "g1")
         self.assertEqual(result["shot"][project_id][0]["name"], "g2")
         self.assertEqual(result["all"][project_id][0]["name"], "g3")
 
@@ -331,10 +329,12 @@ def test_update_filter_group(self):
         search_filter_group = self.post(path, filter_group_1)
         result = self.get(path)
         self.assertTrue("asset" in result)
-        self.put("%s/%s" % (path, search_filter_group["id"]),
-                 {"name": "updated"})
+        self.put(
+            "%s/%s" % (path, search_filter_group["id"]), {"name": "updated"}
+        )
         result = self.get(
-            "data/search-filter-groups/%s" % search_filter_group["id"])
+            "data/search-filter-groups/%s" % search_filter_group["id"]
+        )
         self.assertEqual(result["name"], "updated")
 
     def test_remove_filter_group(self):

zou/app/models/search_filter.py

@@ -19,7 +19,7 @@ class SearchFilter(db.Model, BaseMixin, SerializerMixin):
     search_filter_group_id = db.Column(
         UUIDType(binary=False),
         db.ForeignKey("search_filter_group.id"),
-        nullable=True
+        nullable=True,
     )
     person_id = db.Column(
         UUIDType(binary=False), db.ForeignKey("person.id"), index=True

zou/app/services/preview_files_service.py

@@ -650,20 +650,18 @@ def generate_tiles_for_movie_previews():
             path = extract_tile_from_preview_file(preview_file.serialize())
             file_store.add_picture("tiles", str(preview_file.id), path)
             print(
-                f"Tile generated preview file for {preview_file.id}",
+                f"Tile generated for preview file {preview_file.id}",
             )
         except Exception as e:
             print(
-                "Failed to generate tile for preview file %s: %s",
-                str(preview_file.id),
-                e,
+                f"Failed to generate tile for preview file {preview_file.id}: {e}"
             )
     return preview_files
 
 
-def reset_movie_file_metadata():
+def reset_movie_files_metadata():
     """
-    Reset preview file size information of open projects.
+    Reset preview files size informations of open projects.
     """
     preview_files = (
         PreviewFile.query.join(Task)
@@ -683,9 +681,8 @@ def reset_movie_file_metadata():
                 str(preview_file.id),
                 "mp4",
             )
-            size = movie.get_movie_size(preview_file_path)
             file_size = os.path.getsize(preview_file_path)
-            width, height = size
+            width, height = movie.get_movie_size(preview_file_path)
             update_preview_file_raw(
                 preview_file,
                 {
@@ -695,19 +692,17 @@ def reset_movie_file_metadata():
                 },
             )
             print(
-                f"Size information stored for {preview_file.id}",
+                f"Size information stored preview file {preview_file.id}",
             )
         except Exception as e:
             print(
-                "Failed to store information for preview file %s: %s",
-                str(preview_file.id),
-                e,
+                f"Failed to store information for preview file {preview_file.id}: {e}"
             )
 
 
-def reset_picture_file_metadata():
+def reset_picture_files_metadata():
     """
-    Reset preview file size information of open projects.
+    Reset preview files size informations of open projects.
     """
     preview_files = (
         PreviewFile.query.join(Task)
@@ -738,11 +733,104 @@ def reset_picture_file_metadata():
                 },
             )
             print(
-                f"Size information stored for {preview_file.id}",
+                f"Size information stored for preview file {preview_file.id}",
             )
         except Exception as e:
             print(
-                "Failed to store information for preview file %s: %s",
-                str(preview_file.id),
-                e,
+                f"Failed to store information for preview file {preview_file.id}: {e}"
+            )
+
+
+def generate_tiles_and_reset_preview_files_metadata():
+    """
+    Generate tiles for all movie previews and reset previews file size
+    informations of open projects.
+    """
+    preview_files = (
+        PreviewFile.query.join(Task)
+        .join(Project)
+        .join(ProjectStatus)
+        .filter(ProjectStatus.name.in_(("Active", "open", "Open")))
+        .filter(PreviewFile.status.not_in(("broken", "processing")))
+        .filter(PreviewFile.extension.in_(("mp4", "png")))
+    )
+    preview_file_already_in_cache = False
+    for preview_file in preview_files:
+        preview_file_id = str(preview_file.id)
+        prefix = "previews" if preview_file.extension == "mp4" else "original"
+        if config.FS_BACKEND != "local":
+            preview_file_already_in_cache = os.path.isfile(
+                os.path.join(
+                    config.TMP_DIR,
+                    "cache-%s-%s.%s"
+                    % (prefix, preview_file_id, preview_file.extension),
+                )
             )
+        try:
+            try:
+                preview_file_path = fs.get_file_path_and_file(
+                    config,
+                    file_store.get_local_movie_path
+                    if preview_file.extension == "mp4"
+                    else file_store.get_local_picture_path,
+                    file_store.open_movie
+                    if preview_file.extension == "mp4"
+                    else file_store.open_picture,
+                    prefix,
+                    preview_file_id,
+                    preview_file.extension,
+                )
+            except Exception as e:
+                print(f"Failed to get preview file {preview_file_id}: {e}")
+                continue
+            try:
+                if preview_file.extension == "mp4":
+                    project = get_project_from_preview_file(preview_file_id)
+                    fps = get_preview_file_fps(project)
+                    extracted_tile_path = movie.generate_tile(
+                        preview_file_path, fps
+                    )
+                    file_store.add_picture(
+                        "tiles", preview_file_id, extracted_tile_path
+                    )
+                    print(
+                        f"Tile generated for preview file {preview_file_id}",
+                    )
+            except Exception as e:
+                print(
+                    f"Failed to generate tile for preview file {preview_file_id}: {e}"
+                )
+            try:
+                if preview_file.extension == "mp4":
+                    width, height = movie.get_movie_size(preview_file_path)
+                else:
+                    width, height = thumbnail_utils.get_dimensions(
+                        preview_file_path
+                    )
+                file_size = os.path.getsize(preview_file_path)
+                update_preview_file_raw(
+                    preview_file,
+                    {
+                        "width": width,
+                        "height": height,
+                        "file_size": file_size,
+                    },
+                )
+                print(
+                    f"Size information stored for preview file {preview_file_id}",
+                )
+            except Exception as e:
+                print(
+                    f"Failed to store information for preview file {preview_file_id}: {e}",
+                )
+        finally:
+            if (
+                config.FS_BACKEND != "local"
+                and not preview_file_already_in_cache
+            ):
+                try:
+                    os.remove(preview_file_path)
+                except:
+                    pass
+
+    return preview_files

zou/app/services/user_service.py

@@ -896,8 +896,9 @@ def get_user_filter_groups(current_user_id):
     result = {}
 
     filter_groups = (
-        SearchFilterGroup.query
-        .join(Project, Project.id == SearchFilterGroup.project_id)
+        SearchFilterGroup.query.join(
+            Project, Project.id == SearchFilterGroup.project_id
+        )
         .join(ProjectStatus, ProjectStatus.id == Project.project_status_id)
         .filter(SearchFilterGroup.person_id == current_user_id)
         .filter(build_open_project_filter())
@@ -906,8 +907,9 @@ def get_user_filter_groups(current_user_id):
 
     filter_groups = (
         filter_groups
-        + SearchFilterGroup.query
-        .filter(SearchFilterGroup.person_id == current_user_id)
+        + SearchFilterGroup.query.filter(
+            SearchFilterGroup.person_id == current_user_id
+        )
         .filter(SearchFilterGroup.project_id == None)
         .all()
     )
@@ -931,11 +933,7 @@ def get_user_filter_groups(current_user_id):
 
 
 def create_filter_group(
-    list_type,
-    name,
-    color,
-    project_id=None,
-    entity_type=None
+    list_type, name, color, project_id=None, entity_type=None
 ):
     """
     Add a new search filter group to the database.

zou/app/utils/commands.py

@@ -631,11 +631,16 @@ def generate_tiles():
         preview_files_service.generate_tiles_for_movie_previews()
 
 
-def reset_movie_file_metadata():
+def reset_movie_files_metadata():
     with app.app_context():
-        preview_files_service.reset_movie_file_metadata()
+        preview_files_service.reset_movie_files_metadata()
 
 
-def reset_picture_file_metadata():
+def reset_picture_files_metadata():
     with app.app_context():
-        preview_files_service.reset_picture_file_metadata()
+        preview_files_service.reset_picture_files_metadata()
+
+
+def generate_tiles_and_reset_preview_files_metadata():
+    with app.app_context():
+        preview_files_service.generate_tiles_and_reset_preview_files_metadata()

zou/cli.py

@@ -469,19 +469,28 @@ def generate_tiles():
 
 
 @cli.command()
-def reset_movie_file_metadata():
+def reset_movie_files_metadata():
     """
     Store height and width metadata for all movie previews in the database.
     """
-    commands.reset_movie_file_metadata()
+    commands.reset_movie_files_metadata()
 
 
 @cli.command()
-def reset_picture_file_metadata():
+def reset_picture_files_metadata():
     """
     Store height and width metadata for all picture previews in the database.
     """
-    commands.reset_picture_file_metadata()
+    commands.reset_picture_files_metadata()
+
+
+@cli.command()
+def generate_tiles_and_reset_preview_files_metadata():
+    """
+    Generate tiles and store height and width metadata for all pictures/movies
+    previews in the database.
+    """
+    commands.generate_tiles_and_reset_preview_files_metadata()
 
 
 if __name__ == "__main__":

zou/utils/movie.py

@@ -94,30 +94,6 @@ def extract_frame_from_movie(movie_path, frame_number, movie_fps):
     return file_target_path
 
 
-def get_all_frames(movie_path):
-    """
-    Generate thumbnails to represent the movie given at movie path. It
-    takes a picture at each frame of the movie.
-    """
-    folder_path = os.path.join(os.getcwd(), "movie_frames")
-    os.makedirs(folder_path, exist_ok=True)
-    file_source_name = os.path.basename(movie_path)
-    file_target_name = f"{file_source_name[:-4]}_%d.png"
-    file_target_path = os.path.join(folder_path, file_target_name)
-
-    try:
-        (
-            ffmpeg.input(movie_path)
-            .output(file_target_path, vsync=0)
-            .run(quiet=True)
-        )
-    except ffmpeg._run.Error as e:
-        print(f"Error generating thumbnails: {e}")
-        raise e
-
-    return folder_path
-
-
 def generate_tile(movie_path, movie_fps):
     """
     Generates a tile from a movie.
@@ -126,7 +102,6 @@ def generate_tile(movie_path, movie_fps):
     file_source_name = os.path.basename(movie_path)
     file_target_name = f"{file_source_name[:-4]}_tile.png"
     file_target_path = os.path.join(folder_path, file_target_name)
-
     probe = ffmpeg.probe(movie_path)
     duration_in_seconds = float(probe["streams"][0]["duration"])
     float_movie_fps = float(movie_fps)
@@ -139,7 +114,7 @@ def generate_tile(movie_path, movie_fps):
     try:
         ffmpeg.input(movie_path).output(
             file_target_path, vf=f"scale={width}:{height},tile=8x{rows}"
-        ).run(quiet=True)
+        ).overwrite_output().run(quiet=True)
     except ffmpeg._run.Error as e:
         log_ffmpeg_error(e, "An error occured while generating the tile.")
         raise e