Merge pull request #63 from x10102/master

Bugfixes, refactoring, better handling of Discord API 429

App.py

@@ -16,6 +16,7 @@
 from utils import ensure_config
 from discord import DiscordClient
 from rss import RSSUpdateType
+from tasks import discord_tasks
 
 # Blueprints
 from blueprints.auth import UserAuth
@@ -33,6 +34,8 @@
 
 app = Flask(__name__)
 
+LOGGER_FORMAT_STR = '[%(asctime)s][%(module)s] %(levelname)s: %(message)s'
+
 @app.route('/')
 def index():
     sort = request.args.get('sort', type=str, default='points')
@@ -45,10 +48,10 @@ def init_logger() -> None:
     Sets up logging
     """
 
-    logging.basicConfig(filename='translatordb.log', filemode='a', format='[%(asctime)s] %(levelname)s: %(message)s', encoding='utf-8')
+    logging.basicConfig(filename='translatordb.log', filemode='a', format=LOGGER_FORMAT_STR, encoding='utf-8')
     logging.getLogger().setLevel(logging.INFO)
     handler_st = logging.StreamHandler()
-    handler_st.setFormatter(logging.Formatter('[%(asctime)s] %(levelname)s: %(message)s'))
+    handler_st.setFormatter(logging.Formatter(LOGGER_FORMAT_STR))
     logging.getLogger().addHandler(handler_st)
 
 def fix_proxy() -> None:
@@ -107,8 +110,8 @@ def extensions_init() -> None:
     discord_token = app.config.get('DISCORD_TOKEN', None)
     if discord_token:
         DiscordClient.init_app(app)
-        sched.add_job('Download avatars', lambda: DiscordClient.download_avatars([u.discord for u in dbs.users()], './temp/avatar'), trigger='interval', days=3)
-        sched.add_job('Fetch nicknames', lambda: dbs.update_discord_nicknames(), trigger='interval', days=4)
+        sched.add_job('Download avatars', lambda: discord_tasks.download_avatars_task(), trigger='interval', days=3)
+        sched.add_job('Fetch nicknames', lambda: discord_tasks.update_nicknames_task(), trigger='interval', days=4)
     else:
         warning('Discord API token not set. Profiles won\'t be updated!')
 

blueprints/debug.py

@@ -14,7 +14,7 @@ def log_debug_access():
 @DebugTools.route('/debug/nickupdate')
 @login_required
 def nickupdate():
-    dbs.update_discord_nicknames()
+    sched.run_job('Fetch nicknames')
     return redirect(url_for('index'))
 
 @DebugTools.route('/debug/avupdate')

db.py

@@ -243,7 +243,10 @@ def delete_article(self, aid: int) -> None:
         query = "DELETE FROM Article WHERE id=?"
         self.__tryexec(query, (aid, ))
 
-    def users(self) -> t.List:
+    def users(self) -> t.List[User]:
+        """
+        Fetches all users from the database
+        """
         query = "SELECT * FROM User"
         rows = self.__tryexec(query).fetchall()
         return [User(*row) for row in rows]
@@ -277,22 +280,7 @@ def update_user(self, u: User) -> None:
         self.__tryexec(query, data)
 
     def rename_article(self, name: str, new_name: str):
-        ... # We need to update the link too
-
-    # TODO: Calling an API adapter in a database class is absolutely horrible
-    def update_discord_nicknames(self) -> None:
-        query = "SELECT discord FROM User"
-        ids = self.__tryexec(query).fetchall()
-        users = dict()
-        for id_ in ids:
-            users[id_[0]] = DiscordClient.get_global_username(id_[0])
-            time.sleep(0.2) # Wait a bit so the API doesn't 429
-        for uid, nickname in users.items():
-            self.__tryexec("UPDATE User SET display_name=? WHERE discord=?", (nickname, uid))
-
-    def update_nickname(self, uid) -> None:
-        nickname = DiscordClient.get_global_username(uid)
-        self.__tryexec("UPDATE User SET display_name=? WHERE discord=?", (nickname, uid))
+        ... # TODO: We need to update the link too
 
     def translation_exists(self, name: str) -> bool:
         query = "SELECT * FROM Article WHERE name=? COLLATE NOCASE"

discord.py

@@ -3,17 +3,17 @@
 import time
 import typing as t
 from logging import warning, error, info
-from os.path import join
-from io import BytesIO
+from http import HTTPStatus
 
 # External
 import requests
-from PIL import Image
 
 API_UA = "SCUTTLE Discord service (https://scp-wiki.cz, v1)"
 API_URL = "https://discord.com/api"
 CDN_URL = "https://cdn.discordapp.com"
 
+RATELIMIT_RETRIES = 3
+
 class DiscordException(Exception):
     pass
 
@@ -45,13 +45,21 @@ def _validate_user_id(uid: str):
 
     @staticmethod
     def _get_user(uid: int) -> t.Optional[dict]:
-        response = requests.get(API_URL + f'/users/{uid}', headers=DiscordClient.__request_headers)
-
-        user = json.loads(response.content)
-
-        if response.status_code == 200:
-            return user
-        elif response.status_code == 404:
+
+        retry = 0
+        while retry < RATELIMIT_RETRIES:
+            response = requests.get(API_URL + f'/users/{uid}', headers=DiscordClient.__request_headers)
+            if response.status_code == HTTPStatus.TOO_MANY_REQUESTS:
+                wait_sec = 2**retry
+                warning(f"Rate limited! Waiting for {wait_sec}")
+                time.sleep(wait_sec)
+                retry += 1
+            else:
+                break
+
+        if response.status_code == HTTPStatus.OK:
+            return json.loads(response.content)
+        elif response.status_code == HTTPStatus.NOT_FOUND:
             warning(f'Discord user API returned 404 for {uid}')
             return None
         else:
@@ -60,6 +68,18 @@ def _get_user(uid: int) -> t.Optional[dict]:
 
     @staticmethod
     def get_global_username(uid: int) -> t.Optional[str]:
+        """Fetches the global nickname, if the user doesn't have any set, returns the username
+
+        Args:
+            uid (int): The user's Discord ID
+
+        Raises:
+            DiscordException: Raised when either of the API requests fails
+
+        Returns:
+            str | None - The nickname / username. None is returned if the request succeeds but some other unexpected error occurs
+        """
+
         try:
             user = DiscordClient._get_user(uid)
         except DiscordException as e:
@@ -93,39 +113,31 @@ def get_avatar(uid: int, size = 256) -> bytes:
             return None
 
         endpoint = CDN_URL + f"/avatars/{uid}/{user['avatar']}.png"
-        response = requests.get(endpoint, headers=DiscordClient.__request_headers, params={'size': str(size)})
-
-        if response.status_code == 200:
+
+        retry = 0
+        while retry < RATELIMIT_RETRIES:
+            response = requests.get(endpoint, headers=DiscordClient.__request_headers, params={'size': str(size)})
+            if response.status_code == HTTPStatus.TOO_MANY_REQUESTS:
+                wait_sec = 2**retry
+                warning(f"Rate limited! Waiting for {wait_sec}")
+                time.sleep(wait_sec)
+                retry += 1
+            else:
+                break
+
+        if response.status_code == HTTPStatus.OK:
             return response.content
-        elif response.status_code == 404:
+        elif response.status_code == HTTPStatus.NOT_FOUND:
             warning(f"Discord CDN request returned 404 for {uid}")
             return None
         else:
             error(f"Discord CDN request failed for {uid}")
             raise DiscordException("CDN Request failed")
 
-    @staticmethod
-    def download_avatars(users, path: str = './temp/avatar') -> None:
-        """Downloads the avatars for multiple users
-
-        Args:
-            users (List[int]): The User IDs
-            path (str): The Download directory
-        """
-        for user in users:
-            if user is None or not DiscordClient._validate_user_id(user):
-                warning(f"Skipping profile update for {user}")
-                continue
-            avatar = DiscordClient.get_avatar(user)
-            if avatar is not None:
-                with open(join(path,f'{user}.png'), 'wb') as file:
-                    file.write(avatar)
-                    Image.open(BytesIO(avatar)).resize((64, 64), Image.Resampling.NEAREST).save(join(path,f'{user}_thumb.png'))
-
-                time.sleep(0.1) # Wait for a bit so we don't hit the rate limit
-
 class DiscordWebhook():
-
+    """
+    Utility class for sending webhooks
+    """
     def __init__(self, url: str = None, notify = 0) -> None:
         self.url = url
         self.notify = notify

extensions.py

@@ -1,3 +1,8 @@
+"""
+This file stores singletons for extensions, database and api services
+Classes are instantiated here, but are ready for use only after init_app is called in app.py
+"""
+
 from db import Database
 from flask_apscheduler import APScheduler
 from flask_login import LoginManager

static/js/user.js

@@ -263,7 +263,7 @@ function addOriginalRow(article, hasAuth) {
     } else {
         template.find("#article-name").addClass("text-gray-500").text(article.name)
     }
-    template.find('#translation-words').text(article.words)
+    template.find('#article-words').text(article.words)
 
     if(article.corrector) {
         let link = $("<a>", {

tasks/discord_tasks.py

@@ -0,0 +1,37 @@
+from discord import DiscordClient, DiscordException
+from extensions import dbs
+from PIL import Image
+from io import BytesIO
+from logging import warning
+from os.path import join
+import time
+
+def update_nicknames_task():
+    users = dbs.users()
+    for user in users:
+        if not user.discord:
+            warning(f"Skipping nickname update for {user.nickname}")
+            continue
+        try:
+            new_nickname = DiscordClient.get_global_username(user.discord)
+        except DiscordException:
+            warning(f"Skipping nickname update for {user.nickname} (API error)")
+            continue
+        if new_nickname != None:
+            user.display_name = new_nickname
+            dbs.update_user(user)
+        time.sleep(0.2) # Wait a bit so the API doesn't 429
+
+def download_avatars_task(path: str = './temp/avatar'):
+    ids = [user.discord for user in dbs.users()]
+    for user in ids:
+        if user is None or not DiscordClient._validate_user_id(user):
+            warning(f"Skipping profile update for {user} (Empty or invalid Discord ID)")
+            continue
+        avatar = DiscordClient.get_avatar(user)
+        if avatar is not None:
+            with open(join(path,f'{user}.png'), 'wb') as file:
+                file.write(avatar)
+                Image.open(BytesIO(avatar)).resize((64, 64), Image.Resampling.NEAREST).save(join(path,f'{user}_thumb.png')) # Create a 64x64 thumnail and save it as [ID]_thumb.png
+
+            time.sleep(0.1) # Wait for a bit so we don't hit the rate limit