README and docs (#8)

.github/dependabot.yml

@@ -8,6 +8,10 @@ updates:
     directory: "/app/"
     schedule:
       interval: "weekly"
+  - package-ecosystem: "pip"
+    directory: "/docs/"
+    schedule:
+      interval: "weekly"
   - package-ecosystem: "pip"
     directory: "/mail_receiver/"
     schedule:

.gitignore

@@ -1,2 +1,3 @@
 __pycache__
 logs/
+.env

.mypy.ini

@@ -4,6 +4,9 @@ ignore_missing_imports = True
 [mypy-dkim.*]
 ignore_missing_imports = True
 
+[mypy-sphinx_rtd_theme.*]
+ignore_missing_imports = True
+
 [mypy-jinja2_simple_tags.*]
 ignore_missing_imports = True
 

README.md

@@ -1,60 +1,61 @@
 # mailgoose
 
-## How to run locally
-To run the service locally, use:
+Mailgoose is a web application that allows the users to check whether their SPF, DMARC and
+DKIM configuration is set up correctly. CERT PL uses it to run
+<a href="https://bezpiecznapoczta.cert.pl/">bezpiecznapoczta.cert.pl</a>, an online service
+that helps Polish institutions to configure their domains to decrease the probability of successful
+e-mail spoofing.
 
-```
-cp env.example .env
-# Please review the file and set the configuration variables as needed,
-# especially APP_DOMAIN - the domain you will serve the application on.
-#
-# You may provide additional settings in this file, e.g. SSL certificate
-# settings: SSL_PRIVATE_KEY_PATH and SSL_CERTIFICATE_PATH.
-
-docker compose up --build
-```
-
-The application will listen on http://127.0.0.1:8000
+## [Quick Start 🔨](https://mailgoose.readthedocs.io/en/latest/quick-start.html) | [Docs 📚](https://mailgoose.readthedocs.io/en/latest/)
 
-## How to deploy to production
-Before deploying the system using the configuration in `docker-compose.yml` remember to:
+## Features
+For an up-to-date list of features, please refer to [the documentation](https://mailgoose.readthedocs.io/en/latest/features.html).
 
-- change the database password to a more secure one,
-- use Redis password,
-- consider whether you want to launch a database/Redis instance inside a container (instead
-  of e.g. attaching to your own PostgreSQL cluster),
-- check whether you want to use Google nameservers.
+## Screenshots
+![Check results](.github/screenshots/check_results.png)
 
-Instead of copying `docker-compose.yml`, you may override the configuration using the
-`docker compose -f docker-compose.yml -f docker-compose.override.yml` syntax.
+## Development
 
-## How to change the layout
-If you want to change the main layout template (e.g. to provide additional scripts or your own
-custom navbar with logo), mount a different file using Docker to `/app/templates/custom_layout.html`.
-Refer to `app/templates/base.html` to learn what block you can fill.
+### Tests
+To run the tests, use:
 
-You can also customize the root page (/) of the system by providing your own file that will
-replace `/app/templates/custom_root_layout.html`.
+```
+./scripts/test
+```
 
-By replacing `/app/templates/custom_failed_check_result_hints.html` you may provide your own
-text that will be displayed if the e-mail sender verification mechanisms checks fail.
+### Code formatting
+Mailgoose uses `pre-commit` to run linters and format the code.
+`pre-commit` is executed on CI to verify that the code is formatted properly.
 
-At CERT PL we use a separate `docker-compose.yml` file with additional configuration
-specific to our instance (https://bezpiecznapoczta.cert.pl/). Instead of copying
-`docker-compose.yml`, we override the configuration using the
-`docker compose -f docker-compose.yml -f docker-compose.override.yml` syntax.
+To run it locally, use:
 
-## How to use the HTTP API
+```
+pre-commit run --all-files
+```
 
-To check a domain using a HTTP API, use:
+To setup `pre-commit` so that it runs before each commit, use:
 
 ```
-curl -X POST http://127.0.0.1:8000/api/v1/check-domain?domain=example.com
+pre-commit install
 ```
 
-## How to run the tests
-To run the tests, use:
+### Building the docs
+
+To build the documentation, use:
 
 ```
-./script/test
+cd docs
+python3 -m venv venv
+. venv/bin/activate
+pip install -r requirements.txt
+make html
 ```
+
+## Contributing
+Contributions are welcome! We will appreciate both ideas for improvements (added as
+[GitHub issues](https://github.com/CERT-Polska/mailgoose)) as well as pull requests
+with new features or code improvements.
+
+However obvious it may seem we kindly remind you that by contributing to mailgoose
+you agree that the BSD 3-Clause License shall apply to your input automatically,
+without the need for any additional declarations to be made.

app/docker/Dockerfile

@@ -12,7 +12,7 @@ COPY app/translations/requirements.txt /translations/requirements.txt
 RUN pip install -r /requirements.txt
 
 COPY common/wait-for-it.sh /wait-for-it.sh
-COPY common/mail_receiver_utils.py /app/mail_receiver_utils.py
+COPY common/ /app/common/
 
 COPY app/ /app/
 RUN openssl rand -hex 10 > /app/build_id

app/src/app.py

@@ -7,14 +7,16 @@
 from email.utils import parseaddr
 from typing import Any, Callable, Optional
 
-import decouple
 from fastapi import FastAPI, Form, HTTPException, Request
 from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
-from mail_receiver_utils import get_key_from_username
 from redis import Redis
 from starlette.responses import Response
 
+from common.config import Config
+from common.language import Language
+from common.mail_receiver_utils import get_key_from_username
+
 from .app_utils import (
     get_from_and_dkim_domain,
     recipient_username_to_address,
@@ -26,23 +28,17 @@
 from .resolver import setup_resolver
 from .scan import DomainValidationException, ScanningException, ScanResult
 from .templates import setup_templates
-from .translate import Language, translate
+from .translate import translate
 
 app = FastAPI()
 LOGGER = build_logger(__name__)
+REDIS = Redis.from_url(Config.Data.REDIS_URL)
 
 app.mount("/static", StaticFiles(directory="static"), name="static")
 
 setup_resolver()
 
-APP_DOMAIN = decouple.config("APP_DOMAIN")
-LANGUAGE = decouple.config("LANGUAGE")
-REDIS_MESSAGE_DATA_EXPIRY_SECONDS = decouple.config("REDIS_MESSAGE_DATA_EXPIRY_SECONDS")
-NAMESERVERS = decouple.config("NAMESERVERS", default="8.8.8.8", cast=decouple.Csv(str))
-REDIS = Redis.from_url(decouple.config("REDIS_CONNECTION_STRING"))
-SITE_CONTACT_EMAIL = decouple.config("SITE_CONTACT_EMAIL", default=None)
-
-templates = setup_templates(LANGUAGE)
+templates = setup_templates(Config.UI.LANGUAGE)
 
 
 @dataclasses.dataclass
@@ -88,7 +84,7 @@ async def root(request: Request) -> Response:
 async def check_email_form(request: Request) -> Response:
     recipient_username = f"{binascii.hexlify(os.urandom(16)).decode('ascii')}"
     key = get_key_from_username(recipient_username)
-    REDIS.setex(b"requested-" + key, REDIS_MESSAGE_DATA_EXPIRY_SECONDS, 1)
+    REDIS.setex(b"requested-" + key, Config.Data.REDIS_MESSAGE_DATA_EXPIRY_SECONDS, 1)
 
     return RedirectResponse("/check-email/" + recipient_username)
 
@@ -118,7 +114,7 @@ async def check_email_results(request: Request, recipient_username: str) -> Resp
                 "request": request,
                 "recipient_username": recipient_username,
                 "recipient_address": recipient_address,
-                "site_contact_email": SITE_CONTACT_EMAIL,
+                "site_contact_email": Config.UI.SITE_CONTACT_EMAIL,
             },
         )
 
@@ -130,7 +126,7 @@ async def check_email_results(request: Request, recipient_username: str) -> Resp
     from_domain, dkim_domain = get_from_and_dkim_domain(message_data)
     if not from_domain:
         result = None
-        error = translate("Invalid or no e-mail domain in the message From header", Language.pl_PL)
+        error = translate("Invalid or no e-mail domain in the message From header", Language(Config.UI.LANGUAGE))
     else:
         try:
             result = scan_and_log(
@@ -141,13 +137,13 @@ async def check_email_results(request: Request, recipient_username: str) -> Resp
                 dkim_domain=dkim_domain,
                 message=message_data,
                 message_timestamp=message_timestamp,
-                nameservers=NAMESERVERS,
-                language=Language(LANGUAGE),
+                nameservers=Config.Network.NAMESERVERS,
+                language=Language(Config.UI.LANGUAGE),
             )
             error = None
         except (DomainValidationException, ScanningException) as e:
             result = None
-            error = translate(e.message, Language.pl_PL)
+            error = translate(e.message, Language(Config.UI.LANGUAGE))
 
     token = save_check_results(
         envelope_domain=envelope_domain,
@@ -182,13 +178,13 @@ async def check_domain_scan_post(request: Request, domain: str = Form()) -> Resp
             dkim_domain=None,
             message=None,
             message_timestamp=None,
-            nameservers=NAMESERVERS,
-            language=Language(LANGUAGE),
+            nameservers=Config.Network.NAMESERVERS,
+            language=Language(Config.UI.LANGUAGE),
         )
         error = None
     except (DomainValidationException, ScanningException) as e:
         result = None
-        error = translate(e.message, Language.pl_PL)
+        error = translate(e.message, Language(Config.UI.LANGUAGE))
 
     token = save_check_results(
         envelope_domain=domain,
@@ -240,8 +236,8 @@ async def check_domain_api(request: Request, domain: str) -> ScanAPICallResult:
             dkim_domain=None,
             message=None,
             message_timestamp=None,
-            nameservers=NAMESERVERS,
-            language=Language(LANGUAGE),
+            nameservers=Config.Network.NAMESERVERS,
+            language=Language(Config.UI.LANGUAGE),
         )
         return ScanAPICallResult(result=result)
     except (DomainValidationException, ScanningException):

app/src/app_utils.py

@@ -7,11 +7,13 @@
 from email.utils import parseaddr
 from typing import List, Optional, Tuple
 
-import decouple
 import dkim.util
 from email_validator import EmailNotValidError, validate_email
 from fastapi import Request
 
+from common.config import Config
+from common.language import Language
+
 from .db import (
     DKIMImplementationMismatchLogEntry,
     NonexistentTranslationLogEntry,
@@ -21,9 +23,8 @@
 )
 from .logging import build_logger
 from .scan import ScanResult, scan
-from .translate import Language, translate_scan_result
+from .translate import translate_scan_result
 
-APP_DOMAIN = decouple.config("APP_DOMAIN")
 LOGGER = build_logger(__name__)
 
 
@@ -129,7 +130,7 @@ def scan_and_log(
 def recipient_username_to_address(username: str) -> str:
     # We do not use the request hostname as due to proxy configuration we sometimes got the
     # Host header wrong, thus breaking the check-by-email feature.
-    return f"{username}@{APP_DOMAIN}"
+    return f"{username}@{Config.Network.APP_DOMAIN}"
 
 
 def _nonexistent_translation_handler(message: str) -> str:

app/src/check_results.py

@@ -6,14 +6,14 @@
 from typing import Any, Dict, Optional
 
 import dacite
-import decouple
 from redis import Redis
 
+from common.config import Config
+
 from .logging import build_logger
 from .scan import ScanResult
 
-OLD_CHECK_RESULTS_AGE_MINUTES = decouple.config("OLD_CHECK_RESULTS_AGE_MINUTES", default=60, cast=int)
-REDIS = Redis.from_url(decouple.config("REDIS_CONNECTION_STRING"))
+REDIS = Redis.from_url(Config.Data.REDIS_URL)
 
 LOGGER = build_logger(__name__)
 
@@ -90,8 +90,8 @@ def load_check_results(token: str) -> Optional[Dict[str, Any]]:
             config=dacite.Config(check_types=False),
         )
 
-    result["age_threshold_minutes"] = OLD_CHECK_RESULTS_AGE_MINUTES
+    result["age_threshold_minutes"] = Config.UI.OLD_CHECK_RESULTS_AGE_MINUTES
     result["is_old"] = (
         datetime.datetime.now() - result["created_at"]
-    ).total_seconds() > 60 * OLD_CHECK_RESULTS_AGE_MINUTES
+    ).total_seconds() > 60 * Config.UI.OLD_CHECK_RESULTS_AGE_MINUTES
     return result

app/src/db.py

@@ -1,12 +1,13 @@
 import datetime
-import os
 from enum import Enum
 
 from sqlalchemy import JSON, Boolean, Column, DateTime, Integer, String, create_engine
 from sqlalchemy.orm import declarative_base, sessionmaker  # type: ignore
 
+from common.config import Config
+
 Base = declarative_base()
-engine = create_engine(os.environ["DB_URL"])
+engine = create_engine(Config.Data.DB_URL)
 Session = sessionmaker(bind=engine)
 
 

app/src/tags.py

@@ -1,8 +1,15 @@
 from functools import cache
 
-import decouple
 from jinja2_simple_tags import StandaloneTag
 
+try:
+    from common.config import Config
+
+    LANGUAGE = Config.UI.LANGUAGE
+except ImportError:
+    # This may happen e.g. when pybabel is processing the templates to find messages to be translated
+    LANGUAGE = ""
+
 
 class BuildIDTag(StandaloneTag):  # type: ignore
     tags = {"build_id"}
@@ -15,8 +22,8 @@ def render(self) -> str:
 
 class LanguageTag(StandaloneTag):  # type: ignore
     tags = {"language"}
-    language = decouple.config("LANGUAGE", default="en_US").replace("_", "-")
+    language = LANGUAGE.replace("_", "-")
 
     @cache
     def render(self) -> str:
-        return self.language  # type: ignore
+        return self.language

app/src/translate.py

@@ -1,15 +1,10 @@
 import copy
 import re
-from enum import Enum
 from typing import Callable, List, Optional, Tuple
 
-from .scan import DKIMScanResult, DomainScanResult, ScanResult
-
-
-class Language(Enum):
-    en_US = "en_US"
-    pl_PL = "pl_PL"
+from common.language import Language
 
+from .scan import DKIMScanResult, DomainScanResult, ScanResult
 
 PLACEHOLDER = "__PLACEHOLDER__"
 SKIP_PLACEHOLDER = "__SKIP_PLACEHOLDER__"