diff --git a/README.md b/README.md index ae69d8f..55d5284 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ It uses a neural network to detect highlights in the video-game frames.\ # Supported games -Currently it supports **[Valorant](https://playvalorant.com/)**, **[Overwatch](https://playoverwatch.com/)**. +Currently it supports **[Valorant](https://playvalorant.com/)**, **[Overwatch](https://playoverwatch.com/)** and **[CSGO2](https://www.counter-strike.net/cs2)**. # Usage @@ -59,7 +59,63 @@ The following settings are adjustable: - second-before: Seconds of gameplay included before the highlight. - second-after: Seconds of gameplay included after the highlight. - second-between-kills: Transition time between highlights. If the time between two highlights is less than this value, the both highlights will be merged. -- game: Chosen game (either "valorant" or "overwatch") +- game: Chosen game (either "valorant", "overwatch" or "csgo2") + +### Recommended settings + +I recommend you to use the trials and errors method to find the best settings for your videos.\ +Here are some settings that I found to work well for me: + +#### Valorant + +```json +{ + "neural-network": { + "confidence": 0.8 + }, + "clip": { + "framerate": 8, + "second-before": 4, + "second-after": 0.5, + "second-between-kills": 3 + }, + "game": "valorant" +} +``` + +#### Overwatch + +```json +{ + "neural-network": { + "confidence": 0.6 + }, + "clip": { + "framerate": 8, + "second-before": 4, + "second-after": 3, + "second-between-kills": 5 + }, + "game": "overwatch" +} +``` + +#### CSGO2 + +```json +{ + "neural-network": { + "confidence": 0.7 + }, + "clip": { + "framerate": 8, + "second-before": 4, + "second-after": 1, + "second-between-kills": 3 + }, + "game": "csgo2" +} +``` ## Run @@ -139,3 +195,7 @@ Now `pre-commit` will run on every `git commit`. - `cd crispy-frontend && yarn && yarn dev` - `cd crispy-backend && pip install -Ir requirements-dev.txt && python -m api` + +## Test + +- `cd crispy-api && pytest` diff --git a/crispy-api/api/__init__.py b/crispy-api/api/__init__.py index e408b18..33a2f49 100644 --- a/crispy-api/api/__init__.py +++ b/crispy-api/api/__init__.py @@ -11,7 +11,7 @@ from montydb import MontyClient, set_storage from pydantic.json import ENCODERS_BY_TYPE -from api.config import DATABASE_PATH, DEBUG, GAME, MUSICS, VIDEOS +from api.config import DATABASE_PATH, DEBUG, FRAMERATE, GAME, MUSICS, VIDEOS from api.tools.AI.network import NeuralNetwork from api.tools.enums import SupportedGames from api.tools.filters import apply_filters # noqa @@ -19,12 +19,16 @@ ENCODERS_BY_TYPE[ObjectId] = str +neural_network = NeuralNetwork(GAME) + if GAME == SupportedGames.OVERWATCH: - neural_network = NeuralNetwork([10000, 120, 15, 2]) neural_network.load("./assets/overwatch.npy") elif GAME == SupportedGames.VALORANT: - neural_network = NeuralNetwork([4000, 120, 15, 2], 0.01) neural_network.load("./assets/valorant.npy") +elif GAME == SupportedGames.CSGO2: + neural_network.load("./assets/csgo2.npy") +else: + raise ValueError(f"game {GAME} not supported") logging.getLogger("PIL").setLevel(logging.ERROR) @@ -62,7 +66,7 @@ def is_tool_installed(ffmpeg_tool: str) -> None: @app.on_event("startup") async def setup_crispy() -> None: await handle_musics(MUSICS) - await handle_highlights(VIDEOS, GAME, framerate=8) + await handle_highlights(VIDEOS, GAME, framerate=FRAMERATE) @app.exception_handler(HTTPException) diff --git a/crispy-api/api/__main__.py b/crispy-api/api/__main__.py index c1df1f0..3d33b96 100644 --- a/crispy-api/api/__main__.py +++ b/crispy-api/api/__main__.py @@ -1,15 +1,35 @@ import argparse import asyncio +import os +import sys import uvicorn from api import init_database -from api.config import DEBUG, HOST, PORT +from api.config import ( + DATASET_CSV_PATH, + DATASET_CSV_TEST_PATH, + DEBUG, + HOST, + NETWORK_OUTPUTS_PATH, + PORT, +) +from api.tools.AI.trainer import Trainer, test, train from api.tools.dataset import create_dataset from api.tools.enums import SupportedGames _parser = argparse.ArgumentParser() +# Dataset _parser.add_argument("--dataset", action="store_true") + +# Trainer +_parser.add_argument("--train", help="Train the network", action="store_true") +_parser.add_argument("--test", help="Test the network", action="store_true") +_parser.add_argument("--epoch", help="Number of epochs", type=int, default=1000) +_parser.add_argument("--load", help="Load a trained network", action="store_true") +_parser.add_argument("--path", help="Path to the network", type=str) + +# Game _parser.add_argument( "--game", type=str, choices=[game.value for game in SupportedGames] ) @@ -26,11 +46,34 @@ async def generate_dataset(game: SupportedGames) -> None: if __name__ == "__main__": - if not _args.dataset: + if not _args.dataset and not _args.train and not _args.test: uvicorn.run("api:app", host=HOST, port=PORT, reload=DEBUG, proxy_headers=True) else: game = SupportedGames(_args.game) - if not game: - raise ValueError("Game not supported") + if _args.dataset: + if not game: + raise ValueError("Game not supported") + + asyncio.run(generate_dataset(game)) + else: + trainer = Trainer(game, 0.01) + + if _args.load: + trainer.load(_args.path) + else: + trainer.initialize_weights() + + print(trainer) + if _args.train: + if not os.path.exists(NETWORK_OUTPUTS_PATH): + os.makedirs(NETWORK_OUTPUTS_PATH) + train( + _args.epoch, trainer, DATASET_CSV_PATH, True, NETWORK_OUTPUTS_PATH + ) + + if _args.test: + if not _args.load and not _args.train: + print("You need to load a trained network") + sys.exit(1) - asyncio.run(generate_dataset(game)) + sys.exit(not test(trainer, DATASET_CSV_TEST_PATH)) diff --git a/crispy-api/api/config.py b/crispy-api/api/config.py index 1dcd911..8efca99 100644 --- a/crispy-api/api/config.py +++ b/crispy-api/api/config.py @@ -14,7 +14,8 @@ ASSETS = "assets" SILENCE_PATH = os.path.join(ASSETS, "silence.mp3") -DOT_PATH = os.path.join(ASSETS, "dot.png") +VALORANT_MASK_PATH = os.path.join(ASSETS, "valorant-mask.png") +CSGO2_MASK_PATH = os.path.join(ASSETS, "csgo2-mask.png") BACKUP = "backup" @@ -24,7 +25,10 @@ MUSICS = os.path.join(RESOURCES, "musics") DATASET_PATH = "dataset" -DATASET_VALUES_PATH = "dataset-values.json" +DATASET_VALUES_PATH = os.path.join(DATASET_PATH, "dataset-values.json") +DATASET_CSV_PATH = os.path.join(DATASET_PATH, "result.csv") +DATASET_CSV_TEST_PATH = os.path.join(DATASET_PATH, "test.csv") +NETWORK_OUTPUTS_PATH = "outputs" DATABASE_PATH = ".data" diff --git a/crispy-api/api/models/highlight.py b/crispy-api/api/models/highlight.py index 50d3c7e..c9b0a34 100644 --- a/crispy-api/api/models/highlight.py +++ b/crispy-api/api/models/highlight.py @@ -7,7 +7,7 @@ from mongo_thingy import Thingy from PIL import Image, ImageFilter, ImageOps -from api.config import BACKUP, DOT_PATH +from api.config import BACKUP, CSGO2_MASK_PATH, VALORANT_MASK_PATH from api.models.filter import Filter from api.models.segment import Segment from api.tools.audio import silence_if_no_audio @@ -15,6 +15,8 @@ from api.tools.ffmpeg import merge_videos logger = logging.getLogger("uvicorn") +valorant_mask = Image.open(VALORANT_MASK_PATH) +csgo2_mask = Image.open(CSGO2_MASK_PATH) class Highlight(Thingy): @@ -130,9 +132,7 @@ def _apply_filter_and_do_operations( image = image.crop((1, 1, image.width - 2, image.height - 2)) - dot = Image.open(DOT_PATH) - - image.paste(dot, (0, 0), dot) + image.paste(valorant_mask, (0, 0), valorant_mask) left = image.crop((0, 0, 25, 60)) right = image.crop((95, 0, 120, 60)) @@ -162,6 +162,22 @@ def post_process(image: Image) -> Image: post_process, (899, 801, 122, 62), framerate=framerate ) + async def extract_csgo2_images(self, framerate: int = 4) -> bool: + def post_process(image: Image) -> Image: + image = ImageOps.grayscale( + image.filter(ImageFilter.FIND_EDGES).filter( + ImageFilter.EDGE_ENHANCE_MORE + ) + ) + final = Image.new("RGB", (100, 100)) + final.paste(image, (0, 0)) + final.paste(csgo2_mask, (0, 0), csgo2_mask) + return final + + return await self.extract_images( + post_process, (930, 925, 100, 100), framerate=framerate + ) + async def extract_images_from_game( self, game: SupportedGames, framerate: int = 4 ) -> bool: @@ -169,6 +185,8 @@ async def extract_images_from_game( return await self.extract_overwatch_images(framerate) elif game == SupportedGames.VALORANT: return await self.extract_valorant_images(framerate) + elif game == SupportedGames.CSGO2: + return await self.extract_csgo2_images(framerate) else: raise NotImplementedError diff --git a/crispy-api/api/tools/AI/network.py b/crispy-api/api/tools/AI/network.py index 519785a..e912284 100644 --- a/crispy-api/api/tools/AI/network.py +++ b/crispy-api/api/tools/AI/network.py @@ -3,14 +3,22 @@ import numpy as np import scipy.special +from api.tools.enums import SupportedGames + +NetworkResolution = { + SupportedGames.VALORANT: [4000, 120, 15, 2], + SupportedGames.OVERWATCH: [10000, 120, 15, 2], + SupportedGames.CSGO2: [10000, 120, 15, 2], +} + class NeuralNetwork: """ Neural network to predict if a kill is on the image """ - def __init__(self, nodes: List[int], learning_rate: float = 0.01) -> None: - self.nodes = nodes + def __init__(self, game: SupportedGames, learning_rate: float = 0.01) -> None: + self.nodes = NetworkResolution[game] self.learning_rate = learning_rate self.weights: List[Any] = [] self.activation_function = lambda x: scipy.special.expit(x) @@ -55,7 +63,7 @@ def _train(self, inputs: List[float], targets: Any) -> Tuple[int, int, int]: for i in range(len(self.nodes) - 1 - 1, 0, -1): errors.insert(0, np.dot(self.weights[i].T, errors[0])) - # ten times more likely to be not be kill + # five times more likely to be not be kill # so we mitigate the error if expected == 0: errors = [e / 5 for e in errors] diff --git a/crispy-api/api/tools/AI/trainer.py b/crispy-api/api/tools/AI/trainer.py index 2148846..e700401 100644 --- a/crispy-api/api/tools/AI/trainer.py +++ b/crispy-api/api/tools/AI/trainer.py @@ -1,6 +1,4 @@ -import argparse import os -import sys from datetime import datetime from typing import Any, List, Tuple @@ -8,6 +6,7 @@ import progressbar from api.tools.AI.network import NeuralNetwork +from api.tools.enums import SupportedGames class Trainer(NeuralNetwork): @@ -15,8 +14,8 @@ class Trainer(NeuralNetwork): Trainer for the neural network """ - def __init__(self, nodes: List[int], learning_rate: float) -> None: - super().__init__(nodes, learning_rate) + def __init__(self, game: SupportedGames, learning_rate: float) -> None: + super().__init__(game, learning_rate) value = hash(str(datetime.now())) value %= 1 << 12 self.hash = str(value) @@ -37,28 +36,28 @@ def train( progress_bar = progressbar.ProgressBar(max_value=len(inputs)) progress_bar.update(0) accuracy = 0 + failed = 0 - for j in range(len(inputs)): - res, _, _ = self._train(inputs[j], targets[j]) + for i in range(len(inputs)): + res, expected, got = self._train(inputs[i], targets[i]) accuracy += res if not res: + failed += 1 print( "Failed:", - j, + i, "Got:", - res, + got, "Expected:", - np.argmax(targets[j]), - "Inputs:", - inputs[j], + expected, ) - if j % 25 == 0: - progress_bar.update(j) + if i % 25 == 0: + progress_bar.update(i) progress_bar.finish() - print("Errors:", len(inputs) - accuracy) + print("Errors:", failed) print("Accuracy:", accuracy / len(inputs)) if epoch % 25 == 0 and save: @@ -77,9 +76,7 @@ def test(self, inputs: List[List[float]], targets: List[Any]) -> bool: """ Test the neural network """ - print("Testing...") accuracy_score = 0 - failed = [] confidence = 0 mini_confidence = 100 for j in range(len(inputs)): @@ -90,7 +87,6 @@ def test(self, inputs: List[List[float]], targets: List[Any]) -> bool: accuracy_score += result == np.argmax(targets[j]) if result != np.argmax(targets[j]): - failed.append(j) print( "--- Expected:", np.argmax(targets[j]), @@ -100,8 +96,6 @@ def test(self, inputs: List[List[float]], targets: List[Any]) -> bool: j, "confidence:", str(int(np.max(q) * 100)) + "%", - "inputs:", - inputs[j], ) acc = accuracy_score / len(inputs) @@ -155,41 +149,3 @@ def train( """ final_inputs, final_targets = get_inputs_targets(path) trainer.train(epoch, final_inputs, final_targets, save, output_path) - - -if __name__ == "__main__": # pragma: no cover - t = Trainer([4000, 120, 15, 2], 0.01) - csv_path = os.path.join("backend", "dataset", "result.csv") - csv_test_path = os.path.join("backend", "dataset", "test.csv") - - parser = argparse.ArgumentParser() - parser.add_argument("--train", help="Train the network", action="store_true") - parser.add_argument("--test", help="Test the network", action="store_true") - parser.add_argument("--epoch", help="Number of epochs", type=int, default=1000) - parser.add_argument("--load", help="Load a trained network", action="store_true") - parser.add_argument("--path", help="Path to the network", type=str) - - parser.add_argument("--debug", help="Debug mode", action="store_true") - - args = parser.parse_args() - - images = [] - if args.debug: - images = os.listdir("./backend/dataset/result") - images.sort(key=lambda x: int(x.split("_")[0])) - - if args.load: - t.load(args.path) - else: - t.initialize_weights() - - print(t) - if args.train: - train(args.epoch, t, csv_path, True, "outputs") - - if args.test: - if not args.load and not args.train: - print("You need to load a trained network") - sys.exit(1) - - sys.exit(not test(t, csv_test_path)) diff --git a/crispy-api/api/tools/enums.py b/crispy-api/api/tools/enums.py index e5c3549..d3e8c89 100644 --- a/crispy-api/api/tools/enums.py +++ b/crispy-api/api/tools/enums.py @@ -4,3 +4,4 @@ class SupportedGames(str, Enum): VALORANT = "valorant" OVERWATCH = "overwatch" + CSGO2 = "csgo2" diff --git a/crispy-api/assets/csgo2-mask.png b/crispy-api/assets/csgo2-mask.png new file mode 100644 index 0000000..7549527 Binary files /dev/null and b/crispy-api/assets/csgo2-mask.png differ diff --git a/crispy-api/assets/csgo2.npy b/crispy-api/assets/csgo2.npy new file mode 100644 index 0000000..53194eb Binary files /dev/null and b/crispy-api/assets/csgo2.npy differ diff --git a/crispy-api/assets/dot.png b/crispy-api/assets/valorant-mask.png similarity index 100% rename from crispy-api/assets/dot.png rename to crispy-api/assets/valorant-mask.png diff --git a/crispy-api/settings.json b/crispy-api/settings.json index 79c7b2e..59e000c 100644 --- a/crispy-api/settings.json +++ b/crispy-api/settings.json @@ -8,10 +8,5 @@ "second-after": 0.5, "second-between-kills": 1 }, - "supported-games-and-recommended-neural-network-confidence": [ - ["valorant", 0.8], - ["valorant-review", 0.6], - ["overwatch", 0.4] - ], "game": "valorant" } diff --git a/crispy-api/tests/assets b/crispy-api/tests/assets index c99daca..2f71a93 160000 --- a/crispy-api/tests/assets +++ b/crispy-api/tests/assets @@ -1 +1 @@ -Subproject commit c99daca30e211d43d80ab41e43f3ee19817a5f87 +Subproject commit 2f71a9352c02fbe4bbbb7f01723ec1abdfa6f569 diff --git a/crispy-api/tests/conftest.py b/crispy-api/tests/conftest.py index 4d60a51..4801ce9 100644 --- a/crispy-api/tests/conftest.py +++ b/crispy-api/tests/conftest.py @@ -17,6 +17,7 @@ from api.models.music import Music from api.models.segment import Segment from api.tools.AI.network import NeuralNetwork +from api.tools.enums import SupportedGames from api.tools.image import compare_image from api.tools.job_scheduler import JobScheduler from tests.constants import MAIN_MUSIC, MAIN_VIDEO, ROOT_ASSETS, VALORANT_NETWORK @@ -108,7 +109,7 @@ async def job_scheduler(): @pytest.fixture async def neural_network(): - neural_network = NeuralNetwork([4000, 120, 15, 2], 0.01) + neural_network = NeuralNetwork(SupportedGames.VALORANT, 0.01) neural_network.load(VALORANT_NETWORK) return neural_network @@ -173,8 +174,11 @@ def is_same_csv(self, file_path, expected_file_path): with open(file_path, "r") as f: reader = csv.reader(f) expected_reader = csv.reader(e) - for row, expected_row in zip(reader, expected_reader): - assert row == expected_row + assert len(list(reader)) == len(list(expected_reader)) + + for line, expected_line in zip(reader, expected_reader): + assert len(line) == len(expected_line) + assert line[0] == expected_line[0] def is_same_directory(self, folder, expected_folder): assert os.path.exists(expected_folder) diff --git a/crispy-api/tests/constants.py b/crispy-api/tests/constants.py index d4603ea..171ac89 100644 --- a/crispy-api/tests/constants.py +++ b/crispy-api/tests/constants.py @@ -11,6 +11,7 @@ MAIN_VIDEO_DOWNSCALED = os.path.join(VIDEOS_PATH, "main-video_downscaled.mp4") MAIN_VIDEO_NO_AUDIO = os.path.join(VIDEOS_PATH, "main-video-no-audio.mp4") MAIN_VIDEO_OVERWATCH = os.path.join(VIDEOS_PATH, "main-video-overwatch.mp4") +MAIN_VIDEO_CSGO2 = os.path.join(VIDEOS_PATH, "main-video-csgo2.mp4") MAIN_SEGMENT = os.path.join(VIDEOS_PATH, "main-video-segment.mp4") DATASET_VALUES_PATH = os.path.join(ROOT_ASSETS, "dataset-values.json") diff --git a/crispy-api/tests/models/highlight.py b/crispy-api/tests/models/highlight.py index 86f1b46..3d66a69 100644 --- a/crispy-api/tests/models/highlight.py +++ b/crispy-api/tests/models/highlight.py @@ -7,7 +7,7 @@ from api.models.highlight import Highlight from api.models.segment import Segment from api.tools.enums import SupportedGames -from tests.constants import MAIN_VIDEO_NO_AUDIO, MAIN_VIDEO_OVERWATCH +from tests.constants import MAIN_VIDEO_CSGO2, MAIN_VIDEO_NO_AUDIO, MAIN_VIDEO_OVERWATCH async def test_highlight(highlight): @@ -151,8 +151,9 @@ async def test_segment_video_segments_are_removed(highlight, tmp_path): [ (None, SupportedGames.VALORANT, 8), (MAIN_VIDEO_OVERWATCH, SupportedGames.OVERWATCH, 1.5), + (MAIN_VIDEO_CSGO2, SupportedGames.CSGO2, 1.5), ], - ids=["valorant", "overwatch"], + ids=["valorant", "overwatch", "csgo2"], ) async def test_extract_game_images(highlight, highlight_path, game, rate): if highlight_path is not None: diff --git a/crispy-api/tests/tools/AI/network.py b/crispy-api/tests/tools/AI/network.py index 919e3a0..0a9b7ed 100644 --- a/crispy-api/tests/tools/AI/network.py +++ b/crispy-api/tests/tools/AI/network.py @@ -4,6 +4,7 @@ from api.tools.AI.network import NeuralNetwork from api.tools.AI.trainer import get_inputs_targets +from api.tools.enums import SupportedGames from tests.constants import CSV_PATH_OVERWATCH, OVERWATCH_NETWORK numpy.random.seed(2) @@ -11,7 +12,7 @@ def test_network(tmp_path, capsys): with capsys.disabled(): - network = NeuralNetwork([10000, 120, 15, 2], 0.01) + network = NeuralNetwork(SupportedGames.OVERWATCH, 0.01) assert network.weights == [] network.initialize_weights() assert len(network.weights) == 3 @@ -25,7 +26,7 @@ def test_network(tmp_path, capsys): network.save(os.path.join(tmp_path, "test"), False) - network2 = NeuralNetwork([10000, 120, 15, 2], 0.01) + network2 = NeuralNetwork(SupportedGames.OVERWATCH, 0.01) network2.load(os.path.join(tmp_path, "test.npy")) assert numpy.allclose(network.weights[0], network2.weights[0]) diff --git a/crispy-api/tests/tools/AI/trainer.py b/crispy-api/tests/tools/AI/trainer.py index 20be29c..1c03b77 100644 --- a/crispy-api/tests/tools/AI/trainer.py +++ b/crispy-api/tests/tools/AI/trainer.py @@ -3,16 +3,21 @@ import numpy +from api.tools.AI.network import NetworkResolution from api.tools.AI.trainer import Trainer from api.tools.AI.trainer import test as trainer_test from api.tools.AI.trainer import train +from api.tools.enums import SupportedGames from tests.constants import CSV_PATH_XOR +SupportedGames.XOR = "xor" +NetworkResolution[SupportedGames.XOR] = [2, 4, 2] + numpy.random.seed(2) def test_trainer(capsys, tmp_path): - trainer = Trainer([2, 4, 2], 0.1) + trainer = Trainer(SupportedGames.XOR, 0.1) trainer.initialize_weights() assert not trainer_test(trainer, CSV_PATH_XOR) @@ -30,7 +35,7 @@ def test_trainer(capsys, tmp_path): def test_print(capsys): - trainer = Trainer([2, 4, 2], 0.1) + trainer = Trainer(SupportedGames.XOR, 0.1) trainer.hash = "test" print(trainer)