From 19b706aba46e12813cecb6368b9e880cffaa6cd3 Mon Sep 17 00:00:00 2001 From: Brice PARENT Date: Tue, 12 Jul 2022 00:16:06 +0200 Subject: [PATCH 1/9] feat: Added logger and argparse --- .pylintrc | 3 +- backend/src/crispy/AI/network.py | 29 ++++++++++++++----- backend/src/crispy/AI/trainer.py | 28 +++++++++++++----- backend/src/crispy/main.py | 12 +++++++- backend/src/crispy/utils/arguments.py | 21 ++++++++++++++ backend/src/crispy/utils/create_dataset.py | 8 +++-- .../src/crispy/utils/video/ffmpeg_utils.py | 4 ++- 7 files changed, 85 insertions(+), 20 deletions(-) create mode 100644 backend/src/crispy/utils/arguments.py diff --git a/.pylintrc b/.pylintrc index 24f847d..7afa557 100644 --- a/.pylintrc +++ b/.pylintrc @@ -15,4 +15,5 @@ disable= E1101, R0914, C0200, - W0614 + W0614, + W1203 diff --git a/backend/src/crispy/AI/network.py b/backend/src/crispy/AI/network.py index d0fc105..6760d5b 100644 --- a/backend/src/crispy/AI/network.py +++ b/backend/src/crispy/AI/network.py @@ -2,11 +2,12 @@ import numpy as np import scipy.special -import scipy.ndimage class NeuralNetwork: - """Neural network to predict if a kill is on the image""" + """ + Neural network to predict if a kill is on the image + """ def __init__(self, nodes: List[int], learning_rate: float) -> None: self.nodes = nodes @@ -15,7 +16,9 @@ def __init__(self, nodes: List[int], learning_rate: float) -> None: self.activation_function = lambda x: scipy.special.expit(x) def initialize_weights(self) -> None: - """Initialize the weights of the neural network""" + """ + Initialize the weights of the neural network + """ for i in range(len(self.nodes) - 1): w = np.random.normal(0.0, pow(self.nodes[i], -0.5), (self.nodes[i + 1], self.nodes[i])) @@ -23,7 +26,9 @@ def initialize_weights(self) -> None: def _train(self, inputs: List[float], targets: Any) -> Tuple[int, int, int]: - """Train the neural network""" + """ + Train the neural network + """ inputs = np.array(inputs, ndmin=2).T targets = np.array(targets, ndmin=2).T @@ -65,12 +70,16 @@ def _train(self, inputs: List[float], return expected == got, expected, got def mean_weights(self, N: "NeuralNetwork") -> None: - """Mitigate the weights of the current network with the weights of N""" + """ + Mitigate the weights of the current network with the weights of N + """ for i in range(len(self.weights)): self.weights[i] = (self.weights[i] + N.weights[i]) / 2 def query(self, inputs: List[float]) -> List[float]: - """Query the neural network on a given input""" + """ + Query the neural network on a given input + """ inputs = np.array(inputs, ndmin=2).T outputs = [] @@ -84,12 +93,16 @@ def query(self, inputs: List[float]) -> List[float]: return outputs[-1] def save(self, filename: str) -> None: - """Save the weights of the neural network in numpy format""" + """ + Save the weights of the neural network in numpy format + """ print(f"Saving weights to {filename}.npy") np.save(filename, self.weights) def load(self, filename: str) -> None: - """Load the weights of the neural network from numpy format""" + """ + Load the weights of the neural network from numpy format + """ self.weights = np.load(filename, allow_pickle=True) def __str__(self) -> str: diff --git a/backend/src/crispy/AI/trainer.py b/backend/src/crispy/AI/trainer.py index c5e3af1..e186806 100644 --- a/backend/src/crispy/AI/trainer.py +++ b/backend/src/crispy/AI/trainer.py @@ -18,7 +18,9 @@ class Trainer(NeuralNetwork): - """Trainer for the neural network""" + """ + Trainer for the neural network + """ def __init__(self, nodes: List[int], learning_rate: float) -> None: super().__init__(nodes, learning_rate) @@ -28,7 +30,9 @@ def __init__(self, nodes: List[int], learning_rate: float) -> None: @staticmethod def move_images_from_histogram(histogram: List[int], path: str) -> None: - """Debugging method to move images to the folder `issues`""" + """ + Debugging method to move images to the folder `issues` + """ assert DEBUG maximum = max(histogram) @@ -43,7 +47,9 @@ def move_images_from_histogram(histogram: List[int], path: str) -> None: def train(self, epochs: int, inputs: List[List[float]], targets: List[Any]) -> None: - """Train the neural network for a given number of epochs""" + """ + Train the neural network for a given number of epochs + """ for epoch in range(epochs): print("===\n\tEpoch:", epoch) progress_bar = progressbar.ProgressBar(max_value=len(inputs)) @@ -70,7 +76,9 @@ def train(self, epochs: int, inputs: List[List[float]], self.save("./outputs/trained_network_" + self.hash + "_end") def test(self, inputs: List[List[float]], targets: List[Any]) -> bool: - """Test the neural network""" + """ + Test the neural network + """ print("Testing...") accuracy_score = 0 failed = [] @@ -102,7 +110,9 @@ def __str__(self) -> str: def get_inputs_targets(path: str) -> Tuple[List[List[float]], List[Any]]: - """Read the path csv file and return the inputs and targets""" + """ + Read the path csv file and return the inputs and targets + """ with open(path, 'r') as f: test_data_list = f.readlines() @@ -123,13 +133,17 @@ def get_inputs_targets(path: str) -> Tuple[List[List[float]], List[Any]]: def test(trainer: Trainer, path: str) -> bool: - """Wrapper for the test method""" + """ + Wrapper for the test method + """ final_inputs, final_targets = get_inputs_targets(path) return trainer.test(final_inputs, final_targets) def train(epoch: int, trainer: Trainer, path: str) -> None: - """Wrapper for the train method""" + """ + Wrapper for the train method + """ final_inputs, final_targets = get_inputs_targets(path) trainer.train(epoch, final_inputs, final_targets) diff --git a/backend/src/crispy/main.py b/backend/src/crispy/main.py index fd03dd7..2651869 100644 --- a/backend/src/crispy/main.py +++ b/backend/src/crispy/main.py @@ -1 +1,11 @@ -print("Hello World!") +import logging + +from utils.arguments import args + +l = logging.getLogger() + +print("Welcome to crispy!") + +l.info("Starting the program crispy") + +l.debug(f"Arguments: {args}") diff --git a/backend/src/crispy/utils/arguments.py b/backend/src/crispy/utils/arguments.py new file mode 100644 index 0000000..5cdf54a --- /dev/null +++ b/backend/src/crispy/utils/arguments.py @@ -0,0 +1,21 @@ +import argparse + +import logging + +_parser = argparse.ArgumentParser() + +_parser.add_argument("-d", "--debug", help="Debug mode", action="store_true") + +_parser.add_argument("-v", + "--verbose", + help="increase output verbosity", + action="store_true") + +args = _parser.parse_args() + +if args.debug: + logging.basicConfig(level=logging.DEBUG) +elif args.verbose: + logging.basicConfig(level=logging.INFO) +else: + logging.basicConfig(level=logging.WARNING) diff --git a/backend/src/crispy/utils/create_dataset.py b/backend/src/crispy/utils/create_dataset.py index c4d5950..0cb91ca 100644 --- a/backend/src/crispy/utils/create_dataset.py +++ b/backend/src/crispy/utils/create_dataset.py @@ -12,7 +12,9 @@ def to_csv(folder: str, file: str, values: dict, save: bool = False) -> None: - """Convert the images to a csv file""" + """ + Convert the images to a csv file + """ global INDEX if not file in values: @@ -76,7 +78,9 @@ def to_csv(folder: str, file: str, values: dict, save: bool = False) -> None: def concat_csv(folder: str) -> None: - """Merge all the csv files into one""" + """ + Merge all the csv files into one + """ result = [] files = os.listdir(folder) files.sort() diff --git a/backend/src/crispy/utils/video/ffmpeg_utils.py b/backend/src/crispy/utils/video/ffmpeg_utils.py index f5ed507..c8c87e7 100644 --- a/backend/src/crispy/utils/video/ffmpeg_utils.py +++ b/backend/src/crispy/utils/video/ffmpeg_utils.py @@ -34,7 +34,9 @@ def __apply_filter_and_do_operations(im: Image, def extract_images(video_path: str, save_path: str) -> None: - """Extract the images from the video""" + """ + Extract the images from the video + """ if not os.path.exists(save_path): os.makedirs(save_path) From 11939acf9ba7d2e7ce0c7b5aa555be5799b0f71a Mon Sep 17 00:00:00 2001 From: Brice PARENT Date: Tue, 12 Jul 2022 00:25:54 +0200 Subject: [PATCH 2/9] feat: Added doublequote test --- .pylintrc | 5 +++++ backend/requirements.txt | 1 + backend/src/app.py | 2 +- backend/src/crispy/AI/trainer.py | 4 ++-- backend/src/crispy/utils/create_dataset.py | 4 ++-- backend/src/crispy/utils/video/ffmpeg_utils.py | 6 +++--- scripts/pylint.sh | 2 +- 7 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.pylintrc b/.pylintrc index 7afa557..f75957b 100644 --- a/.pylintrc +++ b/.pylintrc @@ -17,3 +17,8 @@ disable= C0200, W0614, W1203 + + +string-quote=double +triple-quote=double +docstring-quote=double diff --git a/backend/requirements.txt b/backend/requirements.txt index a05fd5e..1788273 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -17,3 +17,4 @@ ffmpeg-python==0.2.0 yapf==0.32.0 mypy==0.961 pylint==2.14.3 +pylint-quotes==0.2.3 diff --git a/backend/src/app.py b/backend/src/app.py index 1b7c9b8..78452a5 100644 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -79,7 +79,7 @@ def new_json() -> None: save_json() -@app.get('/') +@app.get("/") async def main_root(): return JSON_INFO diff --git a/backend/src/crispy/AI/trainer.py b/backend/src/crispy/AI/trainer.py index e186806..b24f60f 100644 --- a/backend/src/crispy/AI/trainer.py +++ b/backend/src/crispy/AI/trainer.py @@ -113,14 +113,14 @@ def get_inputs_targets(path: str) -> Tuple[List[List[float]], List[Any]]: """ Read the path csv file and return the inputs and targets """ - with open(path, 'r') as f: + with open(path, "r") as f: test_data_list = f.readlines() final_inputs = [] final_targets = [] for record in test_data_list: - all_values = record.split(',') + all_values = record.split(",") inputs = (np.asfarray(all_values[1:]) / 255.0 * 0.99) + 0.01 targets = np.zeros(2) + 0.01 diff --git a/backend/src/crispy/utils/create_dataset.py b/backend/src/crispy/utils/create_dataset.py index 0cb91ca..a4c770c 100644 --- a/backend/src/crispy/utils/create_dataset.py +++ b/backend/src/crispy/utils/create_dataset.py @@ -86,7 +86,7 @@ def concat_csv(folder: str) -> None: files.sort() for file in files: if file.split(".")[-1] == "csv": - if file in ('result.csv', 'test.csv'): + if file in ("result.csv", "test.csv"): continue with open(os.path.join(folder, file), "r") as f: lines = f.readlines() @@ -113,7 +113,7 @@ def main(ext: bool, csv: bool) -> None: for video in videos: print("Doing:", video) - video_no_ext = video.split('.', maxsplit=1)[0] + video_no_ext = video.split(".", maxsplit=1)[0] if ext: ffmpeg_utils.extract_images( os.path.join(VIDEOS_PATH, video), diff --git a/backend/src/crispy/utils/video/ffmpeg_utils.py b/backend/src/crispy/utils/video/ffmpeg_utils.py index c8c87e7..5894de4 100644 --- a/backend/src/crispy/utils/video/ffmpeg_utils.py +++ b/backend/src/crispy/utils/video/ffmpeg_utils.py @@ -33,7 +33,7 @@ def __apply_filter_and_do_operations(im: Image, return final -def extract_images(video_path: str, save_path: str) -> None: +def extract_images(video_path: str, save_path: str, fps: int = 4) -> None: """ Extract the images from the video """ @@ -43,10 +43,10 @@ def extract_images(video_path: str, save_path: str) -> None: ( ffmpeg .input(video_path) - .filter('fps', fps='1/0.25') + .filter("fps", fps=f"1/{round(1 / fps), 5}") .crop(x=899, y=801, width=122, height=62) # .overlay(ffmpeg.input(DOT_PATH)) - .output(os.path.join(save_path, "%5d.bmp"), start_number=0) + .output(os.path.join(save_path, "%8d.bmp"), start_number=0) .overwrite_output() .run(quiet=True) ) # yapf: disable diff --git a/scripts/pylint.sh b/scripts/pylint.sh index 5d0b2c7..ea84e55 100755 --- a/scripts/pylint.sh +++ b/scripts/pylint.sh @@ -1,3 +1,3 @@ #!/bin/sh -find backend/src -name '*.py' -print0 | xargs -0 pylint +find backend/src -name '*.py' -print0 | xargs -0 pylint --load-plugins pylint_quotes From 3db0af384231066e18f71231ec66c097637bcd94 Mon Sep 17 00:00:00 2001 From: Brice PARENT Date: Tue, 12 Jul 2022 14:42:59 +0200 Subject: [PATCH 3/9] feat: Added query array --- backend/src/crispy/AI/trainer.py | 10 +-- backend/src/crispy/main.py | 45 +++++++++++- backend/src/crispy/utils/IO/io.py | 73 +++++++++++++++++++ backend/src/crispy/utils/arguments.py | 5 ++ backend/src/crispy/utils/constants.py | 30 +++++++- backend/src/crispy/utils/create_dataset.py | 8 +- .../crispy/utils/{video => }/ffmpeg_utils.py | 10 +-- backend/src/crispy/video/video.py | 72 ++++++++++++++++++ 8 files changed, 229 insertions(+), 24 deletions(-) create mode 100644 backend/src/crispy/utils/IO/io.py rename backend/src/crispy/utils/{video => }/ffmpeg_utils.py (84%) create mode 100644 backend/src/crispy/video/video.py diff --git a/backend/src/crispy/AI/trainer.py b/backend/src/crispy/AI/trainer.py index b24f60f..4187129 100644 --- a/backend/src/crispy/AI/trainer.py +++ b/backend/src/crispy/AI/trainer.py @@ -150,9 +150,8 @@ def train(epoch: int, trainer: Trainer, path: str) -> None: if __name__ == "__main__": t = Trainer([4000, 120, 15, 2], 0.01) - - csv_path = "./backend/dataset/result.csv" - csv_test_path = "./backend/dataset/test.csv" + csv_path = os.path.join("backend", "dataset", "result.csv") + csv_test_path = os.path.join("backend", "dataset", "test.csv") parser = argparse.ArgumentParser() parser.add_argument("--train", @@ -166,10 +165,7 @@ def train(epoch: int, trainer: Trainer, path: str) -> None: parser.add_argument("--load", help="Load a trained network", action="store_true") - parser.add_argument("--path", - help="Path to the network", - type=str, - default="./backend/assets/trained_network_latest.npy") + parser.add_argument("--path", help="Path to the network", type=str) parser.add_argument("--debug", help="Debug mode", action="store_true") diff --git a/backend/src/crispy/main.py b/backend/src/crispy/main.py index 2651869..72555fb 100644 --- a/backend/src/crispy/main.py +++ b/backend/src/crispy/main.py @@ -1,11 +1,48 @@ +from typing import List + import logging from utils.arguments import args +from utils.constants import NEURAL_NETWORK_PATH +from utils.IO.io import generate_tmp_architecture +import video.video as vid +from AI.network import NeuralNetwork + +logging.getLogger("PIL").setLevel(logging.ERROR) + + +def main(videos: List[str]) -> None: + nn = NeuralNetwork([4000, 120, 15, 2], 0.01) + nn.load(NEURAL_NETWORK_PATH) + l.debug(f"Neural network: {nn}") + + generate_tmp_architecture(not args.no_extract) + + for video in videos: + l.info(f"Currently processing {video}") + + if not args.no_extract: + images_path = vid.extract_frames_from_video(video, 8) + else: + images_path = vid.get_saving_path(video) + + query_array = vid.get_query_array_from_video(nn, images_path) + + print(query_array) + + +if __name__ == "__main__": + l = logging.getLogger() + + print("Welcome to crispy!") + + l.info("Starting the program crispy") -l = logging.getLogger() + l.debug(f"Arguments: {args}") -print("Welcome to crispy!") + videos_path = ["0.mp4"] -l.info("Starting the program crispy") + # FIXME: should be sort with the frontend ? + videos_path.sort() -l.debug(f"Arguments: {args}") + main(videos_path) diff --git a/backend/src/crispy/utils/IO/io.py b/backend/src/crispy/utils/IO/io.py new file mode 100644 index 0000000..50ac7c2 --- /dev/null +++ b/backend/src/crispy/utils/IO/io.py @@ -0,0 +1,73 @@ +import os +import shutil + +from utils.constants import TMP_PATH, IMAGE, CUT + + +def generate_tmp_architecture(overwrite: bool) -> None: + """ + Generate the .tmp directory + """ + if not os.path.exists(TMP_PATH): + os.makedirs(TMP_PATH) + elif overwrite: + clear_directory(TMP_PATH) + os.makedirs(TMP_PATH) + + +def generate_folder_clip(name: str, overwrite: bool = True) -> None: + """ + Generate the folder clip + Will generate: + .tmp/{name}/cut + .tmp/{name}/images + """ + path = os.path.join(TMP_PATH, name) + if not os.path.exists(path): + os.makedirs(path) + os.makedirs(os.path.join(path, CUT)) + os.makedirs(os.path.join(path, IMAGE)) + elif overwrite: + clear_directory(os.path.join(path, CUT)) + clear_directory(os.path.join(path, IMAGE)) + generate_folder_clip(name) + + +def clear_directory(path: str) -> None: + """ + Clear the given directory + """ + if os.path.exists(path): + shutil.rmtree(path) + + +def generate_clean_name(name: str) -> str: + """ + Generate the path from the name + Remove the delim characters from the file name + Add hash so (hello world) and (hello_world) are different + """ + + def custom_hash(string: str) -> str: + """ + Generate a hash from the name, same for each session + """ + h = 0 + for ch in string: + h = (h * 281 ^ ord(ch) * 997) & 0xFFFFFFFF + return str(h) + + init_name = name + name = name.replace(" ", "__") + name = name.replace("-", "__") + name = name.replace("/", "__") + name = name.replace("\\", "__") + + return name + "-" + custom_hash(init_name) + + +def remove_extension(name: str) -> str: + """ + Remove the extension from the file name + """ + return name.split(".")[0] diff --git a/backend/src/crispy/utils/arguments.py b/backend/src/crispy/utils/arguments.py index 5cdf54a..fe65184 100644 --- a/backend/src/crispy/utils/arguments.py +++ b/backend/src/crispy/utils/arguments.py @@ -11,6 +11,11 @@ help="increase output verbosity", action="store_true") +_parser.add_argument("--no-extract", + default=False, + help="Do not extract frames", + action="store_true") + args = _parser.parse_args() if args.debug: diff --git a/backend/src/crispy/utils/constants.py b/backend/src/crispy/utils/constants.py index 3e16ec8..4e9d88e 100644 --- a/backend/src/crispy/utils/constants.py +++ b/backend/src/crispy/utils/constants.py @@ -1,7 +1,31 @@ import os BACKEND = "backend" +OUTPUT = "output" + +### UTILS ### +VIDEO = "video" +MUSIC = "music" +IMAGE = "image" +CUT = "cut" +### UTILS ### + +### DATASET_PATH ### DATASET_PATH = os.path.join(BACKEND, "dataset") -VALUES_PATH = os.path.join(DATASET_PATH, "values.json") -VIDEOS_PATH = os.path.join(BACKEND, "resources", "video") -DOT_PATH = os.path.join(BACKEND, "assets", "dot.png") +ASSETS = os.path.join(BACKEND, "assets") + +DATASET_VALUES_PATH = os.path.join(DATASET_PATH, "values.json") +DOT_PATH = os.path.join(ASSETS, "dot.png") +### DATASET_PATH ### + +### CODE_PATH ### +GLOBAL_PATH = BACKEND #FIXME: This is a temporary solution + +TMP_PATH = os.path.join(GLOBAL_PATH, "tmp") + +RESOURCE_PATH = os.path.join(GLOBAL_PATH, "resources") +VIDEOS_PATH = os.path.join(RESOURCE_PATH, VIDEO) +MUSICS_PATH = os.path.join(RESOURCE_PATH, MUSIC) + +NEURAL_NETWORK_PATH = os.path.join(ASSETS, "trained_network_latest.npy") +### CODE_PATH ### diff --git a/backend/src/crispy/utils/create_dataset.py b/backend/src/crispy/utils/create_dataset.py index a4c770c..767639f 100644 --- a/backend/src/crispy/utils/create_dataset.py +++ b/backend/src/crispy/utils/create_dataset.py @@ -4,9 +4,9 @@ from PIL import Image -from video import ffmpeg_utils +import ffmpeg_utils -from constants import * # pylint: disable=wildcard-import +from constants import DATASET_PATH, DATASET_VALUES_PATH, VIDEOS_PATH INDEX = 0 @@ -100,13 +100,11 @@ def main(ext: bool, csv: bool) -> None: if not os.path.exists(DATASET_PATH): os.makedirs(DATASET_PATH) - with open(VALUES_PATH, "r") as f: + with open(DATASET_VALUES_PATH, "r") as f: values = json.load(f) videos = os.listdir(VIDEOS_PATH) videos.sort() - videos = ["test.mp4"] - print(videos) if not os.path.exists(os.path.join(DATASET_PATH, "result")): os.makedirs(os.path.join(DATASET_PATH, "result")) diff --git a/backend/src/crispy/utils/video/ffmpeg_utils.py b/backend/src/crispy/utils/ffmpeg_utils.py similarity index 84% rename from backend/src/crispy/utils/video/ffmpeg_utils.py rename to backend/src/crispy/utils/ffmpeg_utils.py index 5894de4..7d5bff3 100644 --- a/backend/src/crispy/utils/video/ffmpeg_utils.py +++ b/backend/src/crispy/utils/ffmpeg_utils.py @@ -8,8 +8,8 @@ DOT_PATH = os.path.join(BACKEND, "assets", "dot.png") -def __apply_filter_and_do_operations(im: Image, - im_filter: Optional[Any]) -> Image: +def _apply_filter_and_do_operations(im: Image, + im_filter: Optional[Any]) -> Image: if im_filter is not None: im = im.filter(im_filter) @@ -43,7 +43,7 @@ def extract_images(video_path: str, save_path: str, fps: int = 4) -> None: ( ffmpeg .input(video_path) - .filter("fps", fps=f"1/{round(1 / fps), 5}") + .filter("fps", fps=f"1/{round(1 / fps, 5)}") .crop(x=899, y=801, width=122, height=62) # .overlay(ffmpeg.input(DOT_PATH)) .output(os.path.join(save_path, "%8d.bmp"), start_number=0) @@ -60,8 +60,8 @@ def extract_images(video_path: str, save_path: str, fps: int = 4) -> None: im = ImageOps.grayscale(im) - edges = __apply_filter_and_do_operations(im, ImageFilter.FIND_EDGES) - enhanced = __apply_filter_and_do_operations( + edges = _apply_filter_and_do_operations(im, ImageFilter.FIND_EDGES) + enhanced = _apply_filter_and_do_operations( im, ImageFilter.EDGE_ENHANCE_MORE) # im = __apply_filter_and_do_operation(im, None) diff --git a/backend/src/crispy/video/video.py b/backend/src/crispy/video/video.py new file mode 100644 index 0000000..2a2d554 --- /dev/null +++ b/backend/src/crispy/video/video.py @@ -0,0 +1,72 @@ +from typing import List +import os + +from PIL import Image +import numpy as np + +from utils.constants import TMP_PATH, IMAGE, RESOURCE_PATH, VIDEO +import utils.ffmpeg_utils as ff +from utils.IO import io +from AI.network import NeuralNetwork + + +def get_saving_path(video: str) -> str: + """ + Get the saving path for the video + """ + video_no_ext = io.remove_extension(video) + video_clean_name = io.generate_clean_name(video_no_ext) + return os.path.join(TMP_PATH, video_clean_name, IMAGE) + + +def extract_frames_from_video(video: str, fps: int = 6) -> str: + """ + Extract frames from the video + return: saving location + """ + video_no_ext = io.remove_extension(video) + video_clean_name = io.generate_clean_name(video_no_ext) + + io.generate_folder_clip(video_clean_name) + + loading_path = os.path.join(RESOURCE_PATH, VIDEO, video) + saving_path = os.path.join(TMP_PATH, video_clean_name, IMAGE) + ff.extract_images(loading_path, saving_path, fps) + + return saving_path + + +def _image_to_list_format(path: str) -> List[int]: + """ + Convert the image to list format + """ + im = Image.open(path) + pixel_values = list(im.getchannel("R").getdata()) + return pixel_values + + +def get_query_array_from_video(neural_network: NeuralNetwork, + images_path: str) -> List[int]: + """ + Query the neural network on a given input + """ + images = os.listdir(images_path) + images.sort() + query_array = [] + + for i, image in enumerate(images): + image_path = os.path.join(images_path, image) + + list_format = _image_to_list_format(image_path) + inputs = (np.asfarray(list_format) / 255.0 * 0.99) + 0.01 + + query_result = neural_network.query(inputs) + + # FIXME: confidence is not used + # Should be used instead of np.argmax + result = np.argmax(query_result) + + if result == 1: + query_array.append(i) + + return query_array From b24601a5b8d1190b8d202ee104fb71d2825564d4 Mon Sep 17 00:00:00 2001 From: Brice PARENT Date: Tue, 12 Jul 2022 15:33:33 +0200 Subject: [PATCH 4/9] fix: Poorly named function and overwrite bug --- backend/src/crispy/main.py | 4 ++-- backend/src/crispy/utils/IO/io.py | 11 +++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/backend/src/crispy/main.py b/backend/src/crispy/main.py index 72555fb..94831f9 100644 --- a/backend/src/crispy/main.py +++ b/backend/src/crispy/main.py @@ -4,7 +4,7 @@ from utils.arguments import args from utils.constants import NEURAL_NETWORK_PATH -from utils.IO.io import generate_tmp_architecture +from utils.IO.io import generate_tmp_folder import video.video as vid from AI.network import NeuralNetwork @@ -16,7 +16,7 @@ def main(videos: List[str]) -> None: nn.load(NEURAL_NETWORK_PATH) l.debug(f"Neural network: {nn}") - generate_tmp_architecture(not args.no_extract) + generate_tmp_folder(not args.no_extract) for video in videos: l.info(f"Currently processing {video}") diff --git a/backend/src/crispy/utils/IO/io.py b/backend/src/crispy/utils/IO/io.py index 50ac7c2..33b0ea4 100644 --- a/backend/src/crispy/utils/IO/io.py +++ b/backend/src/crispy/utils/IO/io.py @@ -4,9 +4,9 @@ from utils.constants import TMP_PATH, IMAGE, CUT -def generate_tmp_architecture(overwrite: bool) -> None: +def generate_tmp_folder(overwrite: bool) -> None: """ - Generate the .tmp directory + Generate the tmp directory """ if not os.path.exists(TMP_PATH): os.makedirs(TMP_PATH) @@ -19,8 +19,8 @@ def generate_folder_clip(name: str, overwrite: bool = True) -> None: """ Generate the folder clip Will generate: - .tmp/{name}/cut - .tmp/{name}/images + tmp/{name}/cut + tmp/{name}/images """ path = os.path.join(TMP_PATH, name) if not os.path.exists(path): @@ -28,8 +28,7 @@ def generate_folder_clip(name: str, overwrite: bool = True) -> None: os.makedirs(os.path.join(path, CUT)) os.makedirs(os.path.join(path, IMAGE)) elif overwrite: - clear_directory(os.path.join(path, CUT)) - clear_directory(os.path.join(path, IMAGE)) + clear_directory(path) generate_folder_clip(name) From e22b9d88efaf9feb7a6ed6cb4325a6fbdb655d88 Mon Sep 17 00:00:00 2001 From: Brice PARENT Date: Fri, 15 Jul 2022 19:06:33 +0200 Subject: [PATCH 5/9] feat: Added video segmentation --- backend/src/crispy/main.py | 24 ++++++++++++------ backend/src/crispy/utils/IO/io.py | 10 ++++++++ backend/src/crispy/utils/arguments.py | 5 ++++ backend/src/crispy/utils/ffmpeg_utils.py | 22 ++++++++++++++++- backend/src/crispy/video/video.py | 31 +++++++++++++++++------- 5 files changed, 74 insertions(+), 18 deletions(-) diff --git a/backend/src/crispy/main.py b/backend/src/crispy/main.py index 94831f9..a624192 100644 --- a/backend/src/crispy/main.py +++ b/backend/src/crispy/main.py @@ -1,10 +1,9 @@ -from typing import List - import logging +from typing import List from utils.arguments import args from utils.constants import NEURAL_NETWORK_PATH -from utils.IO.io import generate_tmp_folder +from utils.IO import io import video.video as vid from AI.network import NeuralNetwork @@ -12,23 +11,32 @@ def main(videos: List[str]) -> None: + io.generate_tmp_folder(not args.no_extract) + nn = NeuralNetwork([4000, 120, 15, 2], 0.01) nn.load(NEURAL_NETWORK_PATH) l.debug(f"Neural network: {nn}") - generate_tmp_folder(not args.no_extract) - for video in videos: l.info(f"Currently processing {video}") + video_no_ext = io.remove_extension(video) + video_clean_name = io.generate_clean_name(video_no_ext) + l.debug(f"Clean name: {video_clean_name}") if not args.no_extract: + io.generate_folder_clip(video_clean_name) images_path = vid.extract_frames_from_video(video, 8) else: - images_path = vid.get_saving_path(video) + images_path = vid.get_saving_path(video_clean_name) + + if not args.no_segmentation: + io.clean_cuts(video_clean_name) - query_array = vid.get_query_array_from_video(nn, images_path) + query_array = vid.get_query_array_from_video(nn, images_path) + print(query_array) + kill_array = [(0, 0)] - print(query_array) + vid.segment_video_with_kill_array(video, kill_array, 8) if __name__ == "__main__": diff --git a/backend/src/crispy/utils/IO/io.py b/backend/src/crispy/utils/IO/io.py index 33b0ea4..40d3c0f 100644 --- a/backend/src/crispy/utils/IO/io.py +++ b/backend/src/crispy/utils/IO/io.py @@ -70,3 +70,13 @@ def remove_extension(name: str) -> str: Remove the extension from the file name """ return name.split(".")[0] + + +def clean_cuts(folder: str) -> None: + """ + Clean the cuts folder + """ + path = os.path.join(TMP_PATH, folder, CUT) + if os.path.exists(path): + shutil.rmtree(path) + os.makedirs(path) diff --git a/backend/src/crispy/utils/arguments.py b/backend/src/crispy/utils/arguments.py index fe65184..f976182 100644 --- a/backend/src/crispy/utils/arguments.py +++ b/backend/src/crispy/utils/arguments.py @@ -16,6 +16,11 @@ help="Do not extract frames", action="store_true") +_parser.add_argument("--no-segmentation", + default=False, + help="Do not extract frames", + action="store_true") + args = _parser.parse_args() if args.debug: diff --git a/backend/src/crispy/utils/ffmpeg_utils.py b/backend/src/crispy/utils/ffmpeg_utils.py index 7d5bff3..8e6e700 100644 --- a/backend/src/crispy/utils/ffmpeg_utils.py +++ b/backend/src/crispy/utils/ffmpeg_utils.py @@ -1,5 +1,5 @@ import os -from typing import Optional, Any +from typing import Optional, Any, List, Tuple import ffmpeg from PIL import Image, ImageFilter, ImageOps @@ -73,3 +73,23 @@ def extract_images(video_path: str, save_path: str, fps: int = 4) -> None: final.paste(enhanced, (0, 40)) final.save(im_path) + + +def segment_video(video_path: str, save_path: str, + frames: List[Tuple[int, int]], frame_duration: int) -> None: + """ + Segment a video on multiple smaller video using the frames array + """ + for frame in frames: + start = frame[0] / frame_duration + end = frame[1] / frame_duration + # print(start, end, frame_duration, video_path, save_path) + ( + ffmpeg + .input(video_path) + .output(os.path.join(save_path, f"{frame[0]}-{frame[1]}.mp4"), + ss=f"{start}", + to=f"{end}") + .overwrite_output() + .run(quiet=True) + ) # yapf: disable diff --git a/backend/src/crispy/video/video.py b/backend/src/crispy/video/video.py index 2a2d554..768aa44 100644 --- a/backend/src/crispy/video/video.py +++ b/backend/src/crispy/video/video.py @@ -1,10 +1,10 @@ -from typing import List +from typing import List, Tuple import os from PIL import Image import numpy as np -from utils.constants import TMP_PATH, IMAGE, RESOURCE_PATH, VIDEO +from utils.constants import TMP_PATH, IMAGE, RESOURCE_PATH, VIDEO, CUT import utils.ffmpeg_utils as ff from utils.IO import io from AI.network import NeuralNetwork @@ -14,9 +14,7 @@ def get_saving_path(video: str) -> str: """ Get the saving path for the video """ - video_no_ext = io.remove_extension(video) - video_clean_name = io.generate_clean_name(video_no_ext) - return os.path.join(TMP_PATH, video_clean_name, IMAGE) + return os.path.join(TMP_PATH, video, IMAGE) def extract_frames_from_video(video: str, fps: int = 6) -> str: @@ -24,13 +22,12 @@ def extract_frames_from_video(video: str, fps: int = 6) -> str: Extract frames from the video return: saving location """ + loading_path = os.path.join(RESOURCE_PATH, VIDEO, video) + video_no_ext = io.remove_extension(video) video_clean_name = io.generate_clean_name(video_no_ext) + saving_path = os.path.join(TMP_PATH, video_clean_name, CUT) - io.generate_folder_clip(video_clean_name) - - loading_path = os.path.join(RESOURCE_PATH, VIDEO, video) - saving_path = os.path.join(TMP_PATH, video_clean_name, IMAGE) ff.extract_images(loading_path, saving_path, fps) return saving_path @@ -70,3 +67,19 @@ def get_query_array_from_video(neural_network: NeuralNetwork, query_array.append(i) return query_array + + +def segment_video_with_kill_array(video: str, + kill_array: List[Tuple[int, int]], + frame_duration: int = 4) -> None: + """ + Segment the video with the given kill array + """ + + loading_path = os.path.join(RESOURCE_PATH, VIDEO, video) + + video_no_ext = io.remove_extension(video) + video_clean_name = io.generate_clean_name(video_no_ext) + save_path = os.path.join(TMP_PATH, video_clean_name, CUT) + + ff.segment_video(loading_path, save_path, kill_array, frame_duration) From 4eaba3578b400646df43e598039daf71e03f30c3 Mon Sep 17 00:00:00 2001 From: Brice PARENT Date: Sat, 16 Jul 2022 14:39:31 +0200 Subject: [PATCH 6/9] feat: Added kill array --- backend/src/crispy/main.py | 16 ++++++++--- backend/src/crispy/utils/ffmpeg_utils.py | 7 ++--- backend/src/crispy/video/video.py | 34 +++++++++++++++++++++--- 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/backend/src/crispy/main.py b/backend/src/crispy/main.py index a624192..4f333d0 100644 --- a/backend/src/crispy/main.py +++ b/backend/src/crispy/main.py @@ -11,6 +11,14 @@ def main(videos: List[str]) -> None: + ### FIXME: should be settings in settings.json + framerate = 8 + second_before = 2.5 + second_after = 1.5 + #### FIXME: should not be parameters to function + frames_before = int(second_before * framerate) + frames_after = int(second_after * framerate) + io.generate_tmp_folder(not args.no_extract) nn = NeuralNetwork([4000, 120, 15, 2], 0.01) @@ -25,7 +33,7 @@ def main(videos: List[str]) -> None: if not args.no_extract: io.generate_folder_clip(video_clean_name) - images_path = vid.extract_frames_from_video(video, 8) + images_path = vid.extract_frames_from_video(video, framerate) else: images_path = vid.get_saving_path(video_clean_name) @@ -33,10 +41,10 @@ def main(videos: List[str]) -> None: io.clean_cuts(video_clean_name) query_array = vid.get_query_array_from_video(nn, images_path) - print(query_array) - kill_array = [(0, 0)] + kill_array = vid.get_kill_array_from_query_array( + query_array, frames_before, frames_after) - vid.segment_video_with_kill_array(video, kill_array, 8) + vid.segment_video_with_kill_array(video, kill_array, framerate) if __name__ == "__main__": diff --git a/backend/src/crispy/utils/ffmpeg_utils.py b/backend/src/crispy/utils/ffmpeg_utils.py index 8e6e700..d64e443 100644 --- a/backend/src/crispy/utils/ffmpeg_utils.py +++ b/backend/src/crispy/utils/ffmpeg_utils.py @@ -33,17 +33,18 @@ def _apply_filter_and_do_operations(im: Image, return final -def extract_images(video_path: str, save_path: str, fps: int = 4) -> None: +def extract_images(video_path: str, + save_path: str, + framerate: int = 4) -> None: """ Extract the images from the video """ if not os.path.exists(save_path): os.makedirs(save_path) - ( ffmpeg .input(video_path) - .filter("fps", fps=f"1/{round(1 / fps, 5)}") + .filter("framerate", framerate=f"1/{round(1 / framerate, 5)}") .crop(x=899, y=801, width=122, height=62) # .overlay(ffmpeg.input(DOT_PATH)) .output(os.path.join(save_path, "%8d.bmp"), start_number=0) diff --git a/backend/src/crispy/video/video.py b/backend/src/crispy/video/video.py index 768aa44..087d142 100644 --- a/backend/src/crispy/video/video.py +++ b/backend/src/crispy/video/video.py @@ -17,7 +17,7 @@ def get_saving_path(video: str) -> str: return os.path.join(TMP_PATH, video, IMAGE) -def extract_frames_from_video(video: str, fps: int = 6) -> str: +def extract_frames_from_video(video: str, framerate: int = 6) -> str: """ Extract frames from the video return: saving location @@ -26,9 +26,9 @@ def extract_frames_from_video(video: str, fps: int = 6) -> str: video_no_ext = io.remove_extension(video) video_clean_name = io.generate_clean_name(video_no_ext) - saving_path = os.path.join(TMP_PATH, video_clean_name, CUT) + saving_path = os.path.join(TMP_PATH, video_clean_name, IMAGE) - ff.extract_images(loading_path, saving_path, fps) + ff.extract_images(loading_path, saving_path, framerate) return saving_path @@ -83,3 +83,31 @@ def segment_video_with_kill_array(video: str, save_path = os.path.join(TMP_PATH, video_clean_name, CUT) ff.segment_video(loading_path, save_path, kill_array, frame_duration) + + +# FIXME: Add post processing +def get_kill_array_from_query_array( + query_array: List[int], frames_before: int, + frames_after: int) -> List[Tuple[int, int]]: + """ + Get the kill array from the query array + """ + kill_array: List[List[int]] = [] + current_kill: List[int] = [] + for q in query_array: + if len(current_kill) == 0: + current_kill.append(q) + elif q - current_kill[-1] == 1: + current_kill.append(q) + else: + kill_array.append(current_kill) + current_kill = [q] + kill_array.append(current_kill) + + result = [] + for kill in kill_array: + start = kill[0] - frames_before + end = kill[-1] + frames_after + result.append((start, end)) + + return result From 8f111fdaf4dc10ccc1db5bb83881a6344a63e222 Mon Sep 17 00:00:00 2001 From: Brice PARENT Date: Sun, 17 Jul 2022 01:34:39 +0200 Subject: [PATCH 7/9] feat: Added settings.json --- .gitignore | 2 + .pylintrc | 3 +- backend/src/crispy/main.py | 22 ++++------ backend/src/crispy/utils/constants.py | 8 ++++ backend/src/crispy/utils/ffmpeg_utils.py | 4 +- backend/src/crispy/video/video.py | 54 +++++++++++++++++------- settings_template.json | 11 +++++ 7 files changed, 72 insertions(+), 32 deletions(-) create mode 100644 settings_template.json diff --git a/.gitignore b/.gitignore index b65f2b8..2960527 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ build/ tmp/ outputs/ issues/ + +settings.json diff --git a/.pylintrc b/.pylintrc index f75957b..3cd34c6 100644 --- a/.pylintrc +++ b/.pylintrc @@ -16,7 +16,8 @@ disable= R0914, C0200, W0614, - W1203 + W1203, + C3001 string-quote=double diff --git a/backend/src/crispy/main.py b/backend/src/crispy/main.py index 4f333d0..edfb1d4 100644 --- a/backend/src/crispy/main.py +++ b/backend/src/crispy/main.py @@ -11,14 +11,6 @@ def main(videos: List[str]) -> None: - ### FIXME: should be settings in settings.json - framerate = 8 - second_before = 2.5 - second_after = 1.5 - #### FIXME: should not be parameters to function - frames_before = int(second_before * framerate) - frames_after = int(second_after * framerate) - io.generate_tmp_folder(not args.no_extract) nn = NeuralNetwork([4000, 120, 15, 2], 0.01) @@ -33,7 +25,7 @@ def main(videos: List[str]) -> None: if not args.no_extract: io.generate_folder_clip(video_clean_name) - images_path = vid.extract_frames_from_video(video, framerate) + images_path = vid.extract_frames_from_video(video) else: images_path = vid.get_saving_path(video_clean_name) @@ -41,10 +33,12 @@ def main(videos: List[str]) -> None: io.clean_cuts(video_clean_name) query_array = vid.get_query_array_from_video(nn, images_path) - kill_array = vid.get_kill_array_from_query_array( - query_array, frames_before, frames_after) - - vid.segment_video_with_kill_array(video, kill_array, framerate) + l.debug(query_array) + kill_array = vid.get_kill_array_from_query_array(query_array) + l.debug(kill_array) + kill_array = vid.post_processing_kill_array(kill_array) + l.debug(kill_array) + vid.segment_video_with_kill_array(video, kill_array) if __name__ == "__main__": @@ -56,7 +50,7 @@ def main(videos: List[str]) -> None: l.debug(f"Arguments: {args}") - videos_path = ["0.mp4"] + videos_path = ["4.mp4"] # FIXME: should be sort with the frontend ? videos_path.sort() diff --git a/backend/src/crispy/utils/constants.py b/backend/src/crispy/utils/constants.py index 4e9d88e..972785e 100644 --- a/backend/src/crispy/utils/constants.py +++ b/backend/src/crispy/utils/constants.py @@ -1,4 +1,5 @@ import os +import json BACKEND = "backend" OUTPUT = "output" @@ -29,3 +30,10 @@ NEURAL_NETWORK_PATH = os.path.join(ASSETS, "trained_network_latest.npy") ### CODE_PATH ### + +### SETTINGS ### +SETTINGS_PATH = "settings.json" +_f = open(SETTINGS_PATH, "r") +SETTINGS = json.load(_f) +_f.close() +### SETTINGS ### diff --git a/backend/src/crispy/utils/ffmpeg_utils.py b/backend/src/crispy/utils/ffmpeg_utils.py index d64e443..cbf686c 100644 --- a/backend/src/crispy/utils/ffmpeg_utils.py +++ b/backend/src/crispy/utils/ffmpeg_utils.py @@ -44,7 +44,7 @@ def extract_images(video_path: str, ( ffmpeg .input(video_path) - .filter("framerate", framerate=f"1/{round(1 / framerate, 5)}") + .filter("framerate", fps=f"1/{round(1 / framerate, 5)}") .crop(x=899, y=801, width=122, height=62) # .overlay(ffmpeg.input(DOT_PATH)) .output(os.path.join(save_path, "%8d.bmp"), start_number=0) @@ -92,5 +92,5 @@ def segment_video(video_path: str, save_path: str, ss=f"{start}", to=f"{end}") .overwrite_output() - .run(quiet=True) + .run(quiet=False) ) # yapf: disable diff --git a/backend/src/crispy/video/video.py b/backend/src/crispy/video/video.py index 087d142..1bdcefb 100644 --- a/backend/src/crispy/video/video.py +++ b/backend/src/crispy/video/video.py @@ -4,7 +4,7 @@ from PIL import Image import numpy as np -from utils.constants import TMP_PATH, IMAGE, RESOURCE_PATH, VIDEO, CUT +from utils.constants import TMP_PATH, IMAGE, RESOURCE_PATH, VIDEO, CUT, SETTINGS import utils.ffmpeg_utils as ff from utils.IO import io from AI.network import NeuralNetwork @@ -17,7 +17,7 @@ def get_saving_path(video: str) -> str: return os.path.join(TMP_PATH, video, IMAGE) -def extract_frames_from_video(video: str, framerate: int = 6) -> str: +def extract_frames_from_video(video: str) -> str: """ Extract frames from the video return: saving location @@ -28,7 +28,7 @@ def extract_frames_from_video(video: str, framerate: int = 6) -> str: video_clean_name = io.generate_clean_name(video_no_ext) saving_path = os.path.join(TMP_PATH, video_clean_name, IMAGE) - ff.extract_images(loading_path, saving_path, framerate) + ff.extract_images(loading_path, saving_path, SETTINGS["clip"]["framerate"]) return saving_path @@ -50,6 +50,7 @@ def get_query_array_from_video(neural_network: NeuralNetwork, images = os.listdir(images_path) images.sort() query_array = [] + confidence = SETTINGS["neural-network"]["confidence"] for i, image in enumerate(images): image_path = os.path.join(images_path, image) @@ -59,19 +60,14 @@ def get_query_array_from_video(neural_network: NeuralNetwork, query_result = neural_network.query(inputs) - # FIXME: confidence is not used - # Should be used instead of np.argmax - result = np.argmax(query_result) - - if result == 1: + if query_result[1] >= confidence: query_array.append(i) return query_array def segment_video_with_kill_array(video: str, - kill_array: List[Tuple[int, int]], - frame_duration: int = 4) -> None: + kill_array: List[Tuple[int, int]]) -> None: """ Segment the video with the given kill array """ @@ -82,18 +78,44 @@ def segment_video_with_kill_array(video: str, video_clean_name = io.generate_clean_name(video_no_ext) save_path = os.path.join(TMP_PATH, video_clean_name, CUT) - ff.segment_video(loading_path, save_path, kill_array, frame_duration) + ff.segment_video(loading_path, save_path, kill_array, + SETTINGS["clip"]["framerate"]) + + +def post_processing_kill_array( + kill_array: List[Tuple[int, int]]) -> List[Tuple[int, int]]: + """ + Post processing the kill array + """ + found = True + offset = SETTINGS["clip"]["second-between-kills"] * SETTINGS["clip"][ + "framerate"] + while found: + found = False + for i in range(len(kill_array) - 1): + if kill_array[i][1] + offset >= kill_array[i + 1][0]: + found = True + kill_array[i] = (kill_array[i][0], kill_array[i + 1][1]) + kill_array.pop(i + 1) + break + + return kill_array -# FIXME: Add post processing def get_kill_array_from_query_array( - query_array: List[int], frames_before: int, - frames_after: int) -> List[Tuple[int, int]]: + query_array: List[int]) -> List[Tuple[int, int]]: """ Get the kill array from the query array """ kill_array: List[List[int]] = [] current_kill: List[int] = [] + + framerate = SETTINGS["clip"]["framerate"] + mul = lambda x: int(x * framerate) + + frames_before = mul(SETTINGS["clip"]["second-before"]) + frames_after = mul(SETTINGS["clip"]["second-after"]) + for q in query_array: if len(current_kill) == 0: current_kill.append(q) @@ -106,8 +128,10 @@ def get_kill_array_from_query_array( result = [] for kill in kill_array: + if len(kill) < framerate // 2: + continue + start = kill[0] - frames_before end = kill[-1] + frames_after result.append((start, end)) - return result diff --git a/settings_template.json b/settings_template.json new file mode 100644 index 0000000..2f3f055 --- /dev/null +++ b/settings_template.json @@ -0,0 +1,11 @@ +{ + "neural-network": { + "confidence": 0.8 + }, + "clip": { + "framerate": 8, + "second-before": 3, + "second-after": 2, + "second-between-kills": 1 + } +} From 2afde5cbd08248b592bf6f30ef7cba903283069d Mon Sep 17 00:00:00 2001 From: Brice PARENT Date: Sun, 17 Jul 2022 01:48:44 +0200 Subject: [PATCH 8/9] fix: Typo on refactor --- backend/src/crispy/utils/ffmpeg_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/crispy/utils/ffmpeg_utils.py b/backend/src/crispy/utils/ffmpeg_utils.py index 68b2426..782edd7 100644 --- a/backend/src/crispy/utils/ffmpeg_utils.py +++ b/backend/src/crispy/utils/ffmpeg_utils.py @@ -46,7 +46,7 @@ def extract_images(video_path: str, ( ffmpeg .input(video_path) - .filter("framerate", fps=f"1/{round(1 / framerate, 5)}") + .filter("fps", fps=f"1/{round(1 / framerate, 5)}") .crop(x=899, y=801, width=122, height=62) # .overlay(ffmpeg.input(DOT_PATH)) .output(os.path.join(save_path, "%8d.bmp"), start_number=0) @@ -94,7 +94,7 @@ def segment_video(video_path: str, save_path: str, ss=f"{start}", to=f"{end}") .overwrite_output() - .run(quiet=False) + .run(quiet=True) ) # yaPf: disable From e6137c7841abbbdd559e5afc2a479b8b9093a9a7 Mon Sep 17 00:00:00 2001 From: Brice PARENT Date: Sun, 17 Jul 2022 12:27:25 +0200 Subject: [PATCH 9/9] fix: Fixed a bug on fake positives --- backend/src/crispy/main.py | 5 ++++- backend/src/crispy/utils/arguments.py | 5 +++++ backend/src/crispy/utils/ffmpeg_utils.py | 21 +++++++++++++++++++++ backend/src/crispy/video/video.py | 21 ++++++++++++++++++++- 4 files changed, 50 insertions(+), 2 deletions(-) diff --git a/backend/src/crispy/main.py b/backend/src/crispy/main.py index edfb1d4..767f454 100644 --- a/backend/src/crispy/main.py +++ b/backend/src/crispy/main.py @@ -40,6 +40,9 @@ def main(videos: List[str]) -> None: l.debug(kill_array) vid.segment_video_with_kill_array(video, kill_array) + if not args.no_merge: + vid.merge_cuts() + if __name__ == "__main__": l = logging.getLogger() @@ -50,7 +53,7 @@ def main(videos: List[str]) -> None: l.debug(f"Arguments: {args}") - videos_path = ["4.mp4"] + videos_path = ["4.mp4", "quadra chrlie mice.mp4"] # FIXME: should be sort with the frontend ? videos_path.sort() diff --git a/backend/src/crispy/utils/arguments.py b/backend/src/crispy/utils/arguments.py index f976182..df0c13b 100644 --- a/backend/src/crispy/utils/arguments.py +++ b/backend/src/crispy/utils/arguments.py @@ -21,6 +21,11 @@ help="Do not extract frames", action="store_true") +_parser.add_argument("--no-merge", + default=False, + help="Do not merge final videos", + action="store_true") + args = _parser.parse_args() if args.debug: diff --git a/backend/src/crispy/utils/ffmpeg_utils.py b/backend/src/crispy/utils/ffmpeg_utils.py index 782edd7..2408aea 100644 --- a/backend/src/crispy/utils/ffmpeg_utils.py +++ b/backend/src/crispy/utils/ffmpeg_utils.py @@ -1,6 +1,7 @@ import os import random import string +import shutil from typing import Optional, Any, List, Tuple import ffmpeg @@ -148,3 +149,23 @@ def create_new_path(video_path: str) -> str: res = os.path.join(drive, cur_name + ext) return res + + +# FIXME: audio +def merge_videos(videos_path: List[str], save_path: str) -> None: + """ + Merge videos together. + """ + if len(videos_path) > 1: + videos: List[Any] = [] + for video_path in videos_path: + videos.append(ffmpeg.input(video_path)) + ( + ffmpeg + .concat(*videos) + .output(save_path) + .overwrite_output() + .run(quiet=True) + ) # yapf: disable + else: + shutil.copyfile(videos_path[0], save_path) diff --git a/backend/src/crispy/video/video.py b/backend/src/crispy/video/video.py index 1bdcefb..051a9d6 100644 --- a/backend/src/crispy/video/video.py +++ b/backend/src/crispy/video/video.py @@ -128,10 +128,29 @@ def get_kill_array_from_query_array( result = [] for kill in kill_array: - if len(kill) < framerate // 2: + # Remove fake-positives + if len(kill) <= 2: continue start = kill[0] - frames_before end = kill[-1] + frames_after result.append((start, end)) return result + + +def merge_cuts() -> None: + """ + Merge the cuts + """ + folders = os.listdir(TMP_PATH) + folders = [f for f in folders if os.path.isdir(os.path.join(TMP_PATH, f))] + folders.sort() + cuts: List[str] = [] + for folder in folders: + cut = os.listdir(os.path.join(TMP_PATH, folder, CUT)) + cut.sort() + for i in range(len(cut)): + cut[i] = os.path.join(TMP_PATH, folder, CUT, cut[i]) + cuts.extend(cut) + + ff.merge_videos(cuts, os.path.join(TMP_PATH, "merged.mp4"))