Merge remote-tracking branch 'origin/master'

.trunk/trunk.yaml

@@ -21,22 +21,22 @@ lint:
     - [email protected]
     - [email protected]
     - [email protected]
-    - [email protected].2
+    - [email protected].3
     - [email protected]
-    - [email protected].370
+    - [email protected].372
     - git-diff-check
     - [email protected]
     - [email protected]
     - [email protected]
-    - [email protected].3
+    - [email protected].4
     - [email protected]
-    - [email protected].6
+    - [email protected].7
     - [email protected]
     - [email protected]
     - [email protected]
     - [email protected]
     - [email protected]
-    - [email protected].8
+    - [email protected].12
     - [email protected]
   ignore:
     - linters: [ALL]

backend/alembic/versions/0034_virtual_collections_db_view.py

@@ -52,6 +52,8 @@ def upgrade() -> None:
                 SELECT c.id, rom_id::INT, NOW(), NOW()
                 FROM collections c,
                 LATERAL jsonb_array_elements_text(c.roms) AS rom_id
+                LEFT JOIN roms r ON rom_id::INT = r.id
+                WHERE r.id IS NOT NULL;
                 """
             )
         )
@@ -143,6 +145,8 @@ def upgrade() -> None:
                 SELECT c.id, jt.rom_id, NOW(), NOW()
                 FROM collections c
                 JOIN JSON_TABLE(c.roms, '$[*]' COLUMNS (rom_id INT PATH '$')) AS jt
+                LEFT JOIN roms r ON jt.rom_id = r.id
+                WHERE r.id IS NOT NULL;
                 """
             )
         )
@@ -263,18 +267,19 @@ def upgrade() -> None:
 
 def downgrade() -> None:
     with op.batch_alter_table("collections", schema=None) as batch_op:
-        batch_op.add_column(sa.Column("roms", CustomJSON(), nullable=False))
+        batch_op.add_column(sa.Column("roms", CustomJSON(), nullable=True))
 
     connection = op.get_bind()
     if is_postgresql(connection):
         connection.execute(
             sa.text(
                 """
                 UPDATE collections c
-                SET roms = (
-                    SELECT jsonb_agg(rom_id)
+                SET roms = COALESCE(
+                    (SELECT jsonb_agg(rom_id)
                     FROM collections_roms cr
-                    WHERE cr.collection_id = c.id
+                    WHERE cr.collection_id = c.id),
+                    '[]'::jsonb
                 );
                 """
             )
@@ -285,16 +290,20 @@ def downgrade() -> None:
                 """
                 UPDATE collections c
                 JOIN (
-                    SELECT collection_id, JSON_ARRAYAGG(rom_id) as roms
+                    SELECT collection_id, IFNULL(JSON_ARRAYAGG(rom_id), JSON_ARRAY()) AS roms
                     FROM collections_roms
                     GROUP BY collection_id
                 ) cr
                 ON c.id = cr.collection_id
                 SET c.roms = cr.roms;
+
                 """
             )
         )
 
+    with op.batch_alter_table("collections", schema=None) as batch_op:
+        batch_op.alter_column("roms", existing_type=CustomJSON(), nullable=False)
+
     op.drop_table("collections_roms")
 
     connection.execute(

backend/alembic/versions/0035_screenscraper.py

@@ -25,8 +25,8 @@ def upgrade() -> None:
     with op.batch_alter_table("roms", schema=None) as batch_op:
         batch_op.add_column(sa.Column("ss_id", sa.Integer(), nullable=True))
         batch_op.add_column(sa.Column("ss_metadata", CustomJSON(), nullable=True))
-        batch_op.add_column(sa.Column("url_manual", sa.Text(), nullable=True)),
-        batch_op.add_column(sa.Column("path_manual", sa.Text(), nullable=True)),
+        batch_op.add_column(sa.Column("url_manual", sa.Text(), nullable=True))
+        batch_op.add_column(sa.Column("path_manual", sa.Text(), nullable=True))
         batch_op.create_index("idx_roms_ss_id", ["ss_id"], unique=False)
     # ### end Alembic commands ###
 
@@ -39,7 +39,6 @@ def downgrade() -> None:
         batch_op.drop_column("ss_metadata")
         batch_op.drop_column("url_manual")
         batch_op.drop_column("path_manual")
-        batch_op.drop_index("idx_roms_ss_id")
 
     with op.batch_alter_table("platforms", schema=None) as batch_op:
         batch_op.drop_column("ss_id")

backend/config/__init__.py

@@ -51,20 +51,20 @@ def str_to_bool(value: str) -> bool:
 # IGDB
 IGDB_CLIENT_ID: Final = os.environ.get(
     "IGDB_CLIENT_ID", os.environ.get("CLIENT_ID", "")
-)
+).strip()
 IGDB_CLIENT_SECRET: Final = os.environ.get(
     "IGDB_CLIENT_SECRET", os.environ.get("CLIENT_SECRET", "")
-)
+).strip()
 
 # SCREENSCRAPER
 SCREENSCRAPER_USER: Final = os.environ.get("SCREENSCRAPER_USER", "")
 SCREENSCRAPER_PASSWORD: Final = os.environ.get("SCREENSCRAPER_PASSWORD", "")
 
 # STEAMGRIDDB
-STEAMGRIDDB_API_KEY: Final = os.environ.get("STEAMGRIDDB_API_KEY", "")
+STEAMGRIDDB_API_KEY: Final = os.environ.get("STEAMGRIDDB_API_KEY", "").strip()
 
 # MOBYGAMES
-MOBYGAMES_API_KEY: Final = os.environ.get("MOBYGAMES_API_KEY", "")
+MOBYGAMES_API_KEY: Final = os.environ.get("MOBYGAMES_API_KEY", "").strip()
 
 # AUTH
 ROMM_AUTH_SECRET_KEY: Final = os.environ.get(
@@ -81,8 +81,8 @@ def str_to_bool(value: str) -> bool:
 # OIDC
 OIDC_ENABLED: Final = str_to_bool(os.environ.get("OIDC_ENABLED", "false"))
 OIDC_PROVIDER: Final = os.environ.get("OIDC_PROVIDER", "")
-OIDC_CLIENT_ID: Final = os.environ.get("OIDC_CLIENT_ID", "")
-OIDC_CLIENT_SECRET: Final = os.environ.get("OIDC_CLIENT_SECRET", "")
+OIDC_CLIENT_ID: Final = os.environ.get("OIDC_CLIENT_ID", "").strip()
+OIDC_CLIENT_SECRET: Final = os.environ.get("OIDC_CLIENT_SECRET", "").strip()
 OIDC_REDIRECT_URI: Final = os.environ.get("OIDC_REDIRECT_URI", "")
 OIDC_SERVER_APPLICATION_URL: Final = os.environ.get("OIDC_SERVER_APPLICATION_URL", "")
 OIDC_TLS_CACERTFILE: Final = os.environ.get("OIDC_TLS_CACERTFILE", None)

backend/endpoints/responses/collection.py

@@ -18,7 +18,7 @@ class BaseCollectionSchema(BaseModel):
 
 class CollectionSchema(BaseCollectionSchema):
     id: int
-    url_cover: str
+    url_cover: str | None
     rom_ids: set[int]
     rom_count: int
     user_id: int

backend/endpoints/rom.py

@@ -15,6 +15,7 @@
     DISABLE_DOWNLOAD_ENDPOINT_AUTH,
     LIBRARY_BASE_PATH,
     RESOURCES_BASE_PATH,
+    str_to_bool,
 )
 from decorators.auth import protected_route
 from endpoints.responses import MessageResponse
@@ -237,18 +238,6 @@ async def head_rom_content(
 
     # Serve the file directly in development mode for emulatorjs
     if DEV_MODE:
-        if not rom.multi:
-            rom_path = f"{LIBRARY_BASE_PATH}/{rom.full_path}"
-            return FileResponse(
-                path=rom_path,
-                filename=rom.fs_name,
-                headers={
-                    "Content-Disposition": f'attachment; filename="{quote(rom.fs_name)}"',
-                    "Content-Type": "application/octet-stream",
-                    "Content-Length": str(rom.fs_size_bytes),
-                },
-            )
-
         if len(files) == 1:
             file = files[0]
             rom_path = f"{LIBRARY_BASE_PATH}/{file.full_path}"
@@ -270,12 +259,6 @@ async def head_rom_content(
         )
 
     # Otherwise proxy through nginx
-    if not rom.multi:
-        return FileRedirectResponse(
-            download_path=Path(f"/library/{rom.full_path}"),
-            filename=rom.fs_name,
-        )
-
     if len(files) == 1:
         return FileRedirectResponse(
             download_path=Path(f"/library/{files[0].full_path}"),
@@ -320,7 +303,7 @@ async def get_rom_content(
         raise RomNotFoundInDatabaseException(id)
 
     # https://muos.dev/help/addcontent#what-about-multi-disc-content
-    hidden_folder = request.query_params.get("hidden_folder", "").lower() == "true"
+    hidden_folder = str_to_bool(request.query_params.get("hidden_folder", ""))
 
     file_ids = request.query_params.get("file_ids") or ""
     file_ids = [int(f) for f in file_ids.split(",") if f]
@@ -331,18 +314,6 @@ async def get_rom_content(
 
     # Serve the file directly in development mode for emulatorjs
     if DEV_MODE:
-        if not rom.multi:
-            rom_path = f"{LIBRARY_BASE_PATH}/{rom.full_path}"
-            return FileResponse(
-                path=rom_path,
-                filename=rom.fs_name,
-                headers={
-                    "Content-Disposition": f'attachment; filename="{quote(rom.fs_name)}"',
-                    "Content-Type": "application/octet-stream",
-                    "Content-Length": str(rom.fs_size_bytes),
-                },
-            )
-
         if len(files) == 1:
             file = files[0]
             rom_path = f"{LIBRARY_BASE_PATH}/{file.full_path}"
@@ -421,12 +392,6 @@ async def build_zip_in_memory() -> bytes:
         )
 
     # Otherwise proxy through nginx
-    if not rom.multi:
-        return FileRedirectResponse(
-            download_path=Path(f"/library/{rom.full_path}"),
-            filename=rom.fs_name,
-        )
-
     if len(files) == 1:
         return FileRedirectResponse(
             download_path=Path(f"/library/{files[0].full_path}"),

backend/endpoints/sockets/scan.py

@@ -7,7 +7,7 @@
 
 import emoji
 import socketio  # type: ignore
-from config import REDIS_URL, SCAN_TIMEOUT
+from config import DEV_MODE, REDIS_URL, SCAN_TIMEOUT
 from endpoints.responses.platform import PlatformSchema
 from endpoints.responses.rom import SimpleRomSchema
 from exceptions.fs_exceptions import (
@@ -545,13 +545,13 @@ async def scan_handler(_sid: str, options: dict):
     roms_ids = options.get("roms_ids", [])
     metadata_sources = options.get("apis", [])
 
-    # Uncomment this to run scan in the current process
-    # await scan_platforms(
-    #     platform_ids=platform_ids,
-    #     scan_type=scan_type,
-    #     roms_ids=roms_ids,
-    #     metadata_sources=metadata_sources,
-    # )
+    if DEV_MODE:
+        return await scan_platforms(
+            platform_ids=platform_ids,
+            scan_type=scan_type,
+            roms_ids=roms_ids,
+            metadata_sources=metadata_sources,
+        )
 
     return high_prio_queue.enqueue(
         scan_platforms,

backend/handler/metadata/igdb_handler.py

@@ -211,8 +211,9 @@ class IGDBHandler(MetadataHandler):
     def __init__(self) -> None:
         self.BASE_URL = "https://api.igdb.com/v4"
         self.platform_endpoint = f"{self.BASE_URL}/platforms"
-        self.platform_version_endpoint = f"{self.BASE_URL}/platform_versions"
         self.platforms_fields = PLATFORMS_FIELDS
+        self.platform_version_endpoint = f"{self.BASE_URL}/platform_versions"
+        self.platform_version_fields = PLATFORMS_VERSION_FIELDS
         self.games_endpoint = f"{self.BASE_URL}/games"
         self.games_fields = GAMES_FIELDS
         self.search_endpoint = f"{self.BASE_URL}/search"
@@ -415,7 +416,7 @@ async def get_platform(self, slug: str) -> IGDBPlatform:
         # Check if platform is a version if not found
         platform_versions = await self._request(
             self.platform_version_endpoint,
-            data=f'fields {",".join(self.platforms_fields)}; where slug="{slug.lower()}";',
+            data=f'fields {",".join(self.platform_version_fields)}; where slug="{slug.lower()}";',
         )
         version = pydash.get(platform_versions, "[0]", None)
         if version:
@@ -747,6 +748,13 @@ async def get_oauth_token(self) -> str:
     "platform_logo.url",
 )
 
+PLATFORMS_VERSION_FIELDS = (
+    "id",
+    "name",
+    "url",
+    "platform_logo.url",
+)
+
 GAMES_FIELDS = (
     "id",
     "name",

docker/Dockerfile

@@ -158,8 +158,13 @@ COPY ./docker/nginx/templates/ /etc/nginx/templates/
 COPY ./docker/nginx/default.conf /etc/nginx/nginx.conf
 
 # User permissions
+# - Create default user `romm` (1000) and group `romm` (1000).
+# - Create base directories and make default user/group the owner.
+# - Make nginx configuration files writable by everyone, for `envsubst` to work
+#     when a custom UID/GID is used.
 RUN addgroup -g 1000 -S romm && adduser -u 1000 -D -S -G romm romm && \
-    mkdir /romm /redis-data && chown romm:romm /romm /redis-data
+    mkdir /romm /redis-data && chown romm:romm /romm /redis-data && \
+    chmod -R a+w /etc/nginx/conf.d
 
 
 FROM scratch AS slim-image

docker/nginx/templates/default.conf.template

@@ -46,7 +46,7 @@ server {
     # Internally redirect download requests
     location /library/ {
         internal;
-        alias /romm/library/;
+        alias "${ROMM_BASE_PATH}/library/";
     }
 
     # This location, and the related server at port 8081, are used to serve files when
@@ -76,6 +76,6 @@ server {
     server_name localhost;
 
     location /library/ {
-        alias /romm/library/;
+        alias "${ROMM_BASE_PATH}/library/";
     }
 }