Database Support

Dockerfile

@@ -18,4 +18,4 @@ COPY . .
 # install the package using pep 517
 RUN pip install . --no-deps --use-feature=in-tree-build
 
-CMD ["python", "-m", "modmail"]
+CMD ["sh", "-c", "aerich upgrade && python -m modmail"]

aerich.ini

@@ -0,0 +1,4 @@
+[aerich]
+tortoise_orm = modmail.bot.TORTOISE_ORM
+location = ./migrations
+src_folder = ./.

docker-compose.yaml

@@ -2,18 +2,32 @@
 
 version: "3.7"
 
-x-restart-policy: &restart_policy
-  restart: unless-stopped
-
 services:
+  postgres:
+    image: postgres:13-alpine
+    ports:
+      - "127.0.0.1:7777:5432"
+    environment:
+      POSTGRES_DB: modmail
+      POSTGRES_PASSWORD: modmail
+      POSTGRES_USER: modmail
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready -U modmail" ]
+      interval: 2s
+      timeout: 1s
+      retries: 5
+
   modmail:
-    << : *restart_policy
-    container_name: modmail
     build:
       context: .
       dockerfile: Dockerfile
+    depends_on:
+      postgres:
+        condition: service_healthy
+    tty: true
     volumes:
       - .:/modmail
     env_file:
       - .env
-    tty: true
+    environment:
+      database_uri: postgres://modmail:modmail@postgres:5432/modmail

docs/contributing.md

@@ -150,6 +150,39 @@ $ poetry install
         This runs our register pre-commit hooks on every commit to automatically point out issues in code such as missing semicolons, trailing whitespace, and debug statements. By pointing these issues out before code review, this allows a code reviewer to focus on the architecture of a change while not wasting time with trivial style nitpicks.
 
 
+### PostgreSQL setup
+
+Install PostgreSQL according to its [documentation](https://www.postgresql.org/download/).
+
+Enter psql, a terminal-based front-end to PostgreSQL:
+
+<div class="termy">
+
+```console
+$ psql -qd postgres
+```
+
+Run the following queries to create the user and database:
+
+```psql
+CREATE USER modmail WITH SUPERUSER PASSWORD 'modmail';
+CREATE DATABASE modmail WITH OWNER modmail;
+```
+
+Finally, enter `/q` to exit psql.
+
+Once the Database is started, you need run migrations to init tables and columns which can be ran through:
+
+<div class="termy">
+
+```console
+$ poetry run alembic upgrade heads
+
+---> 100%
+```
+
+
+
 ### Set up modmail config
 
 1. Create a copy of `config-default.yml` named `config.yml` in the the `modmail/` directory.
@@ -182,7 +215,14 @@ $ poetry install
 !!!note
         The entire file name is literally `.env`
 
-5. Open the file with any text editor and write the bot token to the files in this format: `TOKEN="my_token"`.
+5. Open the file with any text editor and write the bot token and the database URL to the files in this format:
+        * `TOKEN="my_token"`.
+        * `DATABASE_URI=postgres://modmail:modmail@localhost:5432/modmail`
+
+!!! note
+        If you configured PostgreSQL in a different manner or you are not hosting it locally, then you will need to
+determine the correct host and port yourself. The user, password, and database name should all still be `modmail` unless
+you deviated from the setup instructions in the previous section.
 
 ### Run The Project
 
@@ -262,6 +302,37 @@ your chapter weird.
 
 <!-- TODO: ... -->
 
+
+## Applying database migrations
+Migrations are like a version control system for your database. Each migration defines a change to the database
+and how to undo it. By modifying your database through migrations, you create a consistent, testable, and shareable
+way to evolve your databases over time.
+
+You can easily create migrations through aerich's CLI. But before you do that, you need to generate the database
+schemas, this can simply be done by running the bot once (`task start`). Once that's done, you can use the CLI
+to generate the migration raw SQL file:
+
+<div class="termy">
+
+```console
+$ aerich migrate
+
+---> 100%
+
+Success migrate 1_202029051520102929_drop_column.sql
+```
+
+</div>
+
+Now, check your `migrations/models` folder. You should see your brand-new migration! Notice that it also
+contains a timestamp. This allows `aerich` to run your migrations in the correct order. Format of migrate filename
+is `{version_num}_{datetime}_{name|update}.sql`. If you are renaming a column, aerich would ask you for confirmation,
+you can then choose `True` to rename the column without a column drop and `False` to drop the column then create.
+Note that the latter may **lose** data.
+
+
+
+
 ## Changelog Requirement
 
 Modmail has CI that will check for an entry corresponding to your PR in `CHANGES.md`.

migrations/models/0_20211031115120_init.sql

@@ -0,0 +1,59 @@
+-- upgrade --
+CREATE TABLE IF NOT EXISTS "messages" (
+    "id" BIGSERIAL NOT NULL PRIMARY KEY,
+    "ticket_id" BIGINT NOT NULL,
+    "mirrored_id" BIGINT NOT NULL,
+    "author_id" BIGINT NOT NULL,
+    "content" VARCHAR(4000) NOT NULL
+);
+COMMENT ON TABLE "messages" IS 'Database model representing a message sent in a modmail ticket.';
+CREATE TABLE IF NOT EXISTS "attachments" (
+    "id" BIGSERIAL NOT NULL PRIMARY KEY,
+    "filename" VARCHAR(255) NOT NULL,
+    "file_url" TEXT NOT NULL,
+    "message_id_id" BIGINT NOT NULL REFERENCES "messages" ("id") ON DELETE CASCADE
+);
+COMMENT ON TABLE "attachments" IS 'Database model representing a message attachment sent in a modmail ticket.';
+CREATE TABLE IF NOT EXISTS "embeds" (
+    "id" BIGSERIAL NOT NULL PRIMARY KEY,
+    "content" JSONB NOT NULL,
+    "message_id_id" BIGINT NOT NULL REFERENCES "messages" ("id") ON DELETE CASCADE
+);
+COMMENT ON TABLE "embeds" IS 'Database model representing a discord embed.';
+CREATE TABLE IF NOT EXISTS "emojis" (
+    "id" BIGSERIAL NOT NULL PRIMARY KEY,
+    "name" VARCHAR(32) NOT NULL,
+    "url" TEXT NOT NULL,
+    "animated" BOOL NOT NULL  DEFAULT False,
+    "message_id_id" BIGINT NOT NULL REFERENCES "messages" ("id") ON DELETE CASCADE
+);
+COMMENT ON TABLE "emojis" IS 'Database model representing a custom discord emoji.';
+CREATE TABLE IF NOT EXISTS "servers" (
+    "id" BIGSERIAL NOT NULL PRIMARY KEY,
+    "name" VARCHAR(200) NOT NULL,
+    "icon_url" TEXT NOT NULL
+);
+COMMENT ON TABLE "servers" IS 'Database model representing a discord server.';
+CREATE TABLE IF NOT EXISTS "configurations" (
+    "id" SERIAL NOT NULL PRIMARY KEY,
+    "target_bot_id" BIGINT,
+    "config_key" TEXT NOT NULL,
+    "config_value" TEXT NOT NULL,
+    "target_server_id_id" BIGINT REFERENCES "servers" ("id") ON DELETE CASCADE
+);
+COMMENT ON TABLE "configurations" IS 'Database model representing a discord modmail bot configurations.';
+CREATE TABLE IF NOT EXISTS "tickets" (
+    "id" BIGSERIAL NOT NULL PRIMARY KEY,
+    "thread_id" BIGINT NOT NULL UNIQUE,
+    "creater_id" BIGINT NOT NULL,
+    "creating_message_id" BIGINT NOT NULL,
+    "creating_channel_id" BIGINT NOT NULL,
+    "server_id_id" BIGINT NOT NULL REFERENCES "servers" ("id") ON DELETE CASCADE
+);
+COMMENT ON TABLE "tickets" IS 'An discord modmail ticket for a Discord user with id `creator_id`.';
+CREATE TABLE IF NOT EXISTS "aerich" (
+    "id" SERIAL NOT NULL PRIMARY KEY,
+    "version" VARCHAR(255) NOT NULL,
+    "app" VARCHAR(20) NOT NULL,
+    "content" JSONB NOT NULL
+);

migrations/models/1_20211101070158_update.sql

@@ -0,0 +1,10 @@
+-- upgrade --
+CREATE TABLE IF NOT EXISTS "stickers" (
+    "id" BIGSERIAL NOT NULL PRIMARY KEY,
+    "name" VARCHAR(32) NOT NULL,
+    "url" TEXT NOT NULL,
+    "message_id_id" BIGINT NOT NULL REFERENCES "messages" ("id") ON DELETE CASCADE
+);
+COMMENT ON TABLE "stickers" IS 'Database model representing a custom discord sticker.';
+-- downgrade --
+DROP TABLE IF EXISTS "stickers";

modmail/bot.py

@@ -1,15 +1,16 @@
 import asyncio
 import logging
 import signal
+import sys
 import typing as t
-from typing import Any
 
 import arrow
 import discord
 from aiohttp import ClientSession
 from discord import Activity, AllowedMentions, Intents
 from discord.client import _cleanup_loop
 from discord.ext import commands
+from tortoise import BaseDBAsyncClient, Tortoise
 
 from modmail.config import CONFIG
 from modmail.dispatcher import Dispatcher
@@ -27,6 +28,16 @@
     emojis_and_stickers=True,
 )
 
+TORTOISE_ORM = {
+    "connections": {"default": CONFIG.bot.database_uri},
+    "apps": {
+        "models": {
+            "models": ["modmail.database.models", "aerich.models"],
+            "default_connection": "default",
+        },
+    },
+}
+
 
 class ModmailBot(commands.Bot):
     """
@@ -65,6 +76,23 @@ def __init__(self, **kwargs):
             **kwargs,
         )
 
+    @property
+    def db(self, name: t.Optional[str] = "default") -> BaseDBAsyncClient:
+        """Get the default tortoise-orm connection."""
+        return Tortoise.get_connection(name)
+
+    async def init_db(self) -> None:
+        """Initiate the bot DB connection and check if the DB is alive."""
+        try:
+            self.logger.info("Initializing Tortoise...")
+            await Tortoise.init(TORTOISE_ORM)
+
+            self.logger.info("Generating database schema via Tortoise...")
+            await Tortoise.generate_schemas()
+        except Exception as e:
+            self.logger.error(f"DB connection at {CONFIG.bot.database_uri} not successful, raised:\n{e}")
+            sys.exit(e)
+
     async def start(self, token: str, reconnect: bool = True) -> None:
         """
         Start the bot.
@@ -73,6 +101,7 @@ async def start(self, token: str, reconnect: bool = True) -> None:
         asyncrhonous event loop running, before connecting the bot to discord.
         """
         try:
+            await self.init_db()
             # create the aiohttp session
             self.http_session = ClientSession(loop=self.loop)
             self.logger.trace("Created ClientSession.")
@@ -122,7 +151,7 @@ def run(self, *args, **kwargs) -> None:
         except NotImplementedError:
             pass
 
-        def stop_loop_on_completion(f: Any) -> None:
+        def stop_loop_on_completion(f: t.Any) -> None:
             loop.stop()
 
         future = asyncio.ensure_future(self.start(*args, **kwargs), loop=loop)
@@ -167,6 +196,7 @@ async def close(self) -> None:
         if self.http_session:
             await self.http_session.close()
 
+        await Tortoise.close_connections()
         await super().close()
 
     def load_extensions(self) -> None:

modmail/config.py

@@ -97,6 +97,7 @@ def customise_sources(
 class BotConfig(BaseSettings):
     prefix: str = "?"
     token: str = None
+    database_uri: Optional[str] = None
 
     class Config:
         # env_prefix = "bot."

modmail/database/models/__init__.py

@@ -0,0 +1,20 @@
+from .attachments import Attachments
+from .configuration import Configurations
+from .embeds import Embeds
+from .emojis import Emojis
+from .messages import Messages
+from .servers import Servers
+from .stickers import Stickers
+from .tickets import Tickets
+
+
+__all__ = (
+    "Attachments",
+    "Configurations",
+    "Embeds",
+    "Emojis",
+    "Messages",
+    "Servers",
+    "Stickers",
+    "Tickets",
+)

modmail/database/models/attachments.py

@@ -0,0 +1,15 @@
+from tortoise import fields
+from tortoise.models import Model
+
+from .messages import Messages
+
+
+class Attachments(Model):
+    """Database model representing a message attachment sent in a modmail ticket."""
+
+    id = fields.BigIntField(pk=True)
+    message_id: fields.ForeignKeyRelation[Messages] = fields.ForeignKeyField(
+        "models.Messages", related_name="attachments", to_field="id"
+    )
+    filename = fields.CharField(max_length=255)
+    file_url = fields.TextField()

modmail/database/models/configuration.py

@@ -0,0 +1,21 @@
+from tortoise import fields
+from tortoise.exceptions import ValidationError
+from tortoise.models import Model
+
+from .servers import Servers
+
+
+class Configurations(Model):
+    """Database model representing a discord modmail bot configurations."""
+
+    def __init__(self, **kwargs) -> None:
+        if kwargs.get("target_bot_id") and not kwargs.get("target_server_id"):
+            raise ValidationError("`target_bot_id` is mutually exclusive with `target_server_id`.")
+        super().__init__(**kwargs)
+
+    target_bot_id = fields.BigIntField(null=True)
+    target_server_id: fields.ForeignKeyRelation[Servers] = fields.ForeignKeyField(
+        "models.Servers", related_name="configurations", to_field="id", null=True
+    )
+    config_key = fields.TextField()
+    config_value = fields.TextField()

modmail/database/models/embeds.py

@@ -0,0 +1,14 @@
+from tortoise import fields
+from tortoise.models import Model
+
+from .messages import Messages
+
+
+class Embeds(Model):
+    """Database model representing a discord embed."""
+
+    id = fields.BigIntField(pk=True)
+    message_id: fields.ForeignKeyRelation[Messages] = fields.ForeignKeyField(
+        "models.Messages", related_name="embeds", to_field="id"
+    )
+    content = fields.JSONField()

modmail/database/models/emojis.py

@@ -0,0 +1,16 @@
+from tortoise import fields
+from tortoise.models import Model
+
+from .messages import Messages
+
+
+class Emojis(Model):
+    """Database model representing a custom discord emoji."""
+
+    id = fields.BigIntField(pk=True)
+    name = fields.CharField(max_length=32)
+    url = fields.TextField()
+    animated = fields.BooleanField(default=False)
+    message_id: fields.ForeignKeyRelation[Messages] = fields.ForeignKeyField(
+        "models.Messages", related_name="emojis", to_field="id"
+    )