From 33a18d6e253d597656ee56402eaa25ac53dc3d9d Mon Sep 17 00:00:00 2001 From: x10102 Date: Sun, 8 Sep 2024 12:57:48 +0200 Subject: [PATCH 1/4] Fix word count not showing on original articles --- static/js/user.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/js/user.js b/static/js/user.js index 86f9002..e1fcc44 100644 --- a/static/js/user.js +++ b/static/js/user.js @@ -263,7 +263,7 @@ function addOriginalRow(article, hasAuth) { } else { template.find("#article-name").addClass("text-gray-500").text(article.name) } - template.find('#translation-words').text(article.words) + template.find('#article-words').text(article.words) if(article.corrector) { let link = $("", { From 88002e7828f482b97f3bc68a457263c3d1a34f1f Mon Sep 17 00:00:00 2001 From: x10102 Date: Sun, 8 Sep 2024 14:13:45 +0200 Subject: [PATCH 2/4] Make logger log the module name --- App.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/App.py b/App.py index 15424b7..b40a4be 100644 --- a/App.py +++ b/App.py @@ -33,6 +33,8 @@ app = Flask(__name__) +LOGGER_FORMAT_STR = '[%(asctime)s][%(module)s] %(levelname)s: %(message)s' + @app.route('/') def index(): sort = request.args.get('sort', type=str, default='points') @@ -45,10 +47,10 @@ def init_logger() -> None: Sets up logging """ - logging.basicConfig(filename='translatordb.log', filemode='a', format='[%(asctime)s] %(levelname)s: %(message)s', encoding='utf-8') + logging.basicConfig(filename='translatordb.log', filemode='a', format=LOGGER_FORMAT_STR, encoding='utf-8') logging.getLogger().setLevel(logging.INFO) handler_st = logging.StreamHandler() - handler_st.setFormatter(logging.Formatter('[%(asctime)s] %(levelname)s: %(message)s')) + handler_st.setFormatter(logging.Formatter(LOGGER_FORMAT_STR)) logging.getLogger().addHandler(handler_st) def fix_proxy() -> None: From 4cb4eb367dda884f27123a8d8ffa1a9b9cafdc50 Mon Sep 17 00:00:00 2001 From: x10102 Date: Sun, 8 Sep 2024 14:15:14 +0200 Subject: [PATCH 3/4] Move discord tasks to separate file, some refactoring --- App.py | 5 +++-- blueprints/debug.py | 2 +- db.py | 22 +++++----------------- tasks/__init__.py | 0 tasks/discord_tasks.py | 37 +++++++++++++++++++++++++++++++++++++ 5 files changed, 46 insertions(+), 20 deletions(-) create mode 100644 tasks/__init__.py create mode 100644 tasks/discord_tasks.py diff --git a/App.py b/App.py index b40a4be..1a68459 100644 --- a/App.py +++ b/App.py @@ -16,6 +16,7 @@ from utils import ensure_config from discord import DiscordClient from rss import RSSUpdateType +from tasks import discord_tasks # Blueprints from blueprints.auth import UserAuth @@ -109,8 +110,8 @@ def extensions_init() -> None: discord_token = app.config.get('DISCORD_TOKEN', None) if discord_token: DiscordClient.init_app(app) - sched.add_job('Download avatars', lambda: DiscordClient.download_avatars([u.discord for u in dbs.users()], './temp/avatar'), trigger='interval', days=3) - sched.add_job('Fetch nicknames', lambda: dbs.update_discord_nicknames(), trigger='interval', days=4) + sched.add_job('Download avatars', lambda: discord_tasks.download_avatars_task(), trigger='interval', days=3) + sched.add_job('Fetch nicknames', lambda: discord_tasks.update_nicknames_task(), trigger='interval', days=4) else: warning('Discord API token not set. Profiles won\'t be updated!') diff --git a/blueprints/debug.py b/blueprints/debug.py index cf250e6..f6dc690 100644 --- a/blueprints/debug.py +++ b/blueprints/debug.py @@ -14,7 +14,7 @@ def log_debug_access(): @DebugTools.route('/debug/nickupdate') @login_required def nickupdate(): - dbs.update_discord_nicknames() + sched.run_job('Fetch nicknames') return redirect(url_for('index')) @DebugTools.route('/debug/avupdate') diff --git a/db.py b/db.py index 7528a30..69420d4 100644 --- a/db.py +++ b/db.py @@ -243,7 +243,10 @@ def delete_article(self, aid: int) -> None: query = "DELETE FROM Article WHERE id=?" self.__tryexec(query, (aid, )) - def users(self) -> t.List: + def users(self) -> t.List[User]: + """ + Fetches all users from the database + """ query = "SELECT * FROM User" rows = self.__tryexec(query).fetchall() return [User(*row) for row in rows] @@ -277,22 +280,7 @@ def update_user(self, u: User) -> None: self.__tryexec(query, data) def rename_article(self, name: str, new_name: str): - ... # We need to update the link too - - # TODO: Calling an API adapter in a database class is absolutely horrible - def update_discord_nicknames(self) -> None: - query = "SELECT discord FROM User" - ids = self.__tryexec(query).fetchall() - users = dict() - for id_ in ids: - users[id_[0]] = DiscordClient.get_global_username(id_[0]) - time.sleep(0.2) # Wait a bit so the API doesn't 429 - for uid, nickname in users.items(): - self.__tryexec("UPDATE User SET display_name=? WHERE discord=?", (nickname, uid)) - - def update_nickname(self, uid) -> None: - nickname = DiscordClient.get_global_username(uid) - self.__tryexec("UPDATE User SET display_name=? WHERE discord=?", (nickname, uid)) + ... # TODO: We need to update the link too def translation_exists(self, name: str) -> bool: query = "SELECT * FROM Article WHERE name=? COLLATE NOCASE" diff --git a/tasks/__init__.py b/tasks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tasks/discord_tasks.py b/tasks/discord_tasks.py new file mode 100644 index 0000000..470cd89 --- /dev/null +++ b/tasks/discord_tasks.py @@ -0,0 +1,37 @@ +from discord import DiscordClient, DiscordException +from extensions import dbs +from PIL import Image +from io import BytesIO +from logging import warning +from os.path import join +import time + +def update_nicknames_task(): + users = dbs.users() + for user in users: + if not user.discord: + warning(f"Skipping nickname update for {user.nickname}") + continue + try: + new_nickname = DiscordClient.get_global_username(user.discord) + except DiscordException: + warning(f"Skipping nickname update for {user.nickname} (API error)") + continue + if new_nickname != None: + user.display_name = new_nickname + dbs.update_user(user) + time.sleep(0.2) # Wait a bit so the API doesn't 429 + +def download_avatars_task(path: str = './temp/avatar'): + ids = [user.discord for user in dbs.users()] + for user in ids: + if user is None or not DiscordClient._validate_user_id(user): + warning(f"Skipping profile update for {user} (Empty or invalid Discord ID)") + continue + avatar = DiscordClient.get_avatar(user) + if avatar is not None: + with open(join(path,f'{user}.png'), 'wb') as file: + file.write(avatar) + Image.open(BytesIO(avatar)).resize((64, 64), Image.Resampling.NEAREST).save(join(path,f'{user}_thumb.png')) # Create a 64x64 thumnail and save it as [ID]_thumb.png + + time.sleep(0.1) # Wait for a bit so we don't hit the rate limit \ No newline at end of file From dadc5bf169b4ba976e83d5e82cd23caf7391a69c Mon Sep 17 00:00:00 2001 From: x10102 Date: Sun, 8 Sep 2024 14:15:37 +0200 Subject: [PATCH 4/4] Retry discord API requests if rate limited --- discord.py | 82 +++++++++++++++++++++++++++++---------------------- extensions.py | 5 ++++ 2 files changed, 52 insertions(+), 35 deletions(-) diff --git a/discord.py b/discord.py index 8d03563..71ab810 100644 --- a/discord.py +++ b/discord.py @@ -3,17 +3,17 @@ import time import typing as t from logging import warning, error, info -from os.path import join -from io import BytesIO +from http import HTTPStatus # External import requests -from PIL import Image API_UA = "SCUTTLE Discord service (https://scp-wiki.cz, v1)" API_URL = "https://discord.com/api" CDN_URL = "https://cdn.discordapp.com" +RATELIMIT_RETRIES = 3 + class DiscordException(Exception): pass @@ -45,13 +45,21 @@ def _validate_user_id(uid: str): @staticmethod def _get_user(uid: int) -> t.Optional[dict]: - response = requests.get(API_URL + f'/users/{uid}', headers=DiscordClient.__request_headers) - - user = json.loads(response.content) - - if response.status_code == 200: - return user - elif response.status_code == 404: + + retry = 0 + while retry < RATELIMIT_RETRIES: + response = requests.get(API_URL + f'/users/{uid}', headers=DiscordClient.__request_headers) + if response.status_code == HTTPStatus.TOO_MANY_REQUESTS: + wait_sec = 2**retry + warning(f"Rate limited! Waiting for {wait_sec}") + time.sleep(wait_sec) + retry += 1 + else: + break + + if response.status_code == HTTPStatus.OK: + return json.loads(response.content) + elif response.status_code == HTTPStatus.NOT_FOUND: warning(f'Discord user API returned 404 for {uid}') return None else: @@ -60,6 +68,18 @@ def _get_user(uid: int) -> t.Optional[dict]: @staticmethod def get_global_username(uid: int) -> t.Optional[str]: + """Fetches the global nickname, if the user doesn't have any set, returns the username + + Args: + uid (int): The user's Discord ID + + Raises: + DiscordException: Raised when either of the API requests fails + + Returns: + str | None - The nickname / username. None is returned if the request succeeds but some other unexpected error occurs + """ + try: user = DiscordClient._get_user(uid) except DiscordException as e: @@ -93,39 +113,31 @@ def get_avatar(uid: int, size = 256) -> bytes: return None endpoint = CDN_URL + f"/avatars/{uid}/{user['avatar']}.png" - response = requests.get(endpoint, headers=DiscordClient.__request_headers, params={'size': str(size)}) - - if response.status_code == 200: + + retry = 0 + while retry < RATELIMIT_RETRIES: + response = requests.get(endpoint, headers=DiscordClient.__request_headers, params={'size': str(size)}) + if response.status_code == HTTPStatus.TOO_MANY_REQUESTS: + wait_sec = 2**retry + warning(f"Rate limited! Waiting for {wait_sec}") + time.sleep(wait_sec) + retry += 1 + else: + break + + if response.status_code == HTTPStatus.OK: return response.content - elif response.status_code == 404: + elif response.status_code == HTTPStatus.NOT_FOUND: warning(f"Discord CDN request returned 404 for {uid}") return None else: error(f"Discord CDN request failed for {uid}") raise DiscordException("CDN Request failed") - @staticmethod - def download_avatars(users, path: str = './temp/avatar') -> None: - """Downloads the avatars for multiple users - - Args: - users (List[int]): The User IDs - path (str): The Download directory - """ - for user in users: - if user is None or not DiscordClient._validate_user_id(user): - warning(f"Skipping profile update for {user}") - continue - avatar = DiscordClient.get_avatar(user) - if avatar is not None: - with open(join(path,f'{user}.png'), 'wb') as file: - file.write(avatar) - Image.open(BytesIO(avatar)).resize((64, 64), Image.Resampling.NEAREST).save(join(path,f'{user}_thumb.png')) - - time.sleep(0.1) # Wait for a bit so we don't hit the rate limit - class DiscordWebhook(): - + """ + Utility class for sending webhooks + """ def __init__(self, url: str = None, notify = 0) -> None: self.url = url self.notify = notify diff --git a/extensions.py b/extensions.py index c287051..29def0e 100644 --- a/extensions.py +++ b/extensions.py @@ -1,3 +1,8 @@ +""" +This file stores singletons for extensions, database and api services +Classes are instantiated here, but are ready for use only after init_app is called in app.py +""" + from db import Database from flask_apscheduler import APScheduler from flask_login import LoginManager