diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 737a878..d15fc8e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,6 +8,10 @@ updates: directory: "/app/" schedule: interval: "weekly" + - package-ecosystem: "pip" + directory: "/docs/" + schedule: + interval: "weekly" - package-ecosystem: "pip" directory: "/mail_receiver/" schedule: diff --git a/.github/screenshots/check.png b/.github/screenshots/check.png new file mode 100644 index 0000000..f5504bf Binary files /dev/null and b/.github/screenshots/check.png differ diff --git a/.gitignore b/.gitignore index ced2950..c704599 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ __pycache__ logs/ +.env diff --git a/README.md b/README.md index ca736c5..97ef4a6 100644 --- a/README.md +++ b/README.md @@ -1,60 +1,61 @@ # mailgoose -## How to run locally -To run the service locally, use: +Mailgoose is a web application that allows the users to check whether their SPF, DMARC and +DKIM configuration is set up correctly. CERT PL uses it to run +bezpiecznapoczta.cert.pl, an online service +that helps Polish institutions to configure their domains to decrease the probability of successful +e-mail spoofing. -``` -cp env.example .env -# Please review the file and set the configuration variables as needed, -# especially APP_DOMAIN - the domain you will serve the application on. -# -# You may provide additional settings in this file, e.g. SSL certificate -# settings: SSL_PRIVATE_KEY_PATH and SSL_CERTIFICATE_PATH. - -docker compose up --build -``` - -The application will listen on http://127.0.0.1:8000 +## [Quick Start 🔨](https://mailgoose.readthedocs.io/en/latest/quick-start.html) | [Docs 📚](https://mailgoose.readthedocs.io/en/latest/) -## How to deploy to production -Before deploying the system using the configuration in `docker-compose.yml` remember to: +## Features +For an up-to-date list of features, please refer to [the documentation](https://mailgoose.readthedocs.io/en/latest/features.html). -- change the database password to a more secure one, -- use Redis password, -- consider whether you want to launch a database/Redis instance inside a container (instead - of e.g. attaching to your own PostgreSQL cluster), -- check whether you want to use Google nameservers. +## Screenshots +![Check results](.github/screenshots/check_results.png) -Instead of copying `docker-compose.yml`, you may override the configuration using the -`docker compose -f docker-compose.yml -f docker-compose.override.yml` syntax. +## Development -## How to change the layout -If you want to change the main layout template (e.g. to provide additional scripts or your own -custom navbar with logo), mount a different file using Docker to `/app/templates/custom_layout.html`. -Refer to `app/templates/base.html` to learn what block you can fill. +### Tests +To run the tests, use: -You can also customize the root page (/) of the system by providing your own file that will -replace `/app/templates/custom_root_layout.html`. +``` +./scripts/test +``` -By replacing `/app/templates/custom_failed_check_result_hints.html` you may provide your own -text that will be displayed if the e-mail sender verification mechanisms checks fail. +### Code formatting +Mailgoose uses `pre-commit` to run linters and format the code. +`pre-commit` is executed on CI to verify that the code is formatted properly. -At CERT PL we use a separate `docker-compose.yml` file with additional configuration -specific to our instance (https://bezpiecznapoczta.cert.pl/). Instead of copying -`docker-compose.yml`, we override the configuration using the -`docker compose -f docker-compose.yml -f docker-compose.override.yml` syntax. +To run it locally, use: -## How to use the HTTP API +``` +pre-commit run --all-files +``` -To check a domain using a HTTP API, use: +To setup `pre-commit` so that it runs before each commit, use: ``` -curl -X POST http://127.0.0.1:8000/api/v1/check-domain?domain=example.com +pre-commit install ``` -## How to run the tests -To run the tests, use: +### Building the docs + +To build the documentation, use: ``` -./script/test +cd docs +python3 -m venv venv +. venv/bin/activate +pip install -r requirements.txt +make html ``` + +## Contributing +Contributions are welcome! We will appreciate both ideas for improvements (added as +[GitHub issues](https://github.com/CERT-Polska/mailgoose)) as well as pull requests +with new features or code improvements. + +However obvious it may seem we kindly remind you that by contributing to mailgoose +you agree that the BSD 3-Clause License shall apply to your input automatically, +without the need for any additional declarations to be made. diff --git a/app/docker/Dockerfile b/app/docker/Dockerfile index 3d21bec..4524639 100644 --- a/app/docker/Dockerfile +++ b/app/docker/Dockerfile @@ -12,7 +12,7 @@ COPY app/translations/requirements.txt /translations/requirements.txt RUN pip install -r /requirements.txt COPY common/wait-for-it.sh /wait-for-it.sh -COPY common/mail_receiver_utils.py /app/mail_receiver_utils.py +COPY common/ /app/common/ COPY app/ /app/ RUN openssl rand -hex 10 > /app/build_id diff --git a/app/src/app.py b/app/src/app.py index 7891448..92a6663 100644 --- a/app/src/app.py +++ b/app/src/app.py @@ -7,14 +7,16 @@ from email.utils import parseaddr from typing import Any, Callable, Optional -import decouple from fastapi import FastAPI, Form, HTTPException, Request from fastapi.responses import HTMLResponse, RedirectResponse from fastapi.staticfiles import StaticFiles -from mail_receiver_utils import get_key_from_username from redis import Redis from starlette.responses import Response +from common.config import Config +from common.language import Language +from common.mail_receiver_utils import get_key_from_username + from .app_utils import ( get_from_and_dkim_domain, recipient_username_to_address, @@ -26,23 +28,17 @@ from .resolver import setup_resolver from .scan import DomainValidationException, ScanningException, ScanResult from .templates import setup_templates -from .translate import Language, translate +from .translate import translate app = FastAPI() LOGGER = build_logger(__name__) +REDIS = Redis.from_url(Config.Data.REDIS_URL) app.mount("/static", StaticFiles(directory="static"), name="static") setup_resolver() -APP_DOMAIN = decouple.config("APP_DOMAIN") -LANGUAGE = decouple.config("LANGUAGE") -REDIS_MESSAGE_DATA_EXPIRY_SECONDS = decouple.config("REDIS_MESSAGE_DATA_EXPIRY_SECONDS") -NAMESERVERS = decouple.config("NAMESERVERS", default="8.8.8.8", cast=decouple.Csv(str)) -REDIS = Redis.from_url(decouple.config("REDIS_CONNECTION_STRING")) -SITE_CONTACT_EMAIL = decouple.config("SITE_CONTACT_EMAIL", default=None) - -templates = setup_templates(LANGUAGE) +templates = setup_templates(Config.UI.LANGUAGE) @dataclasses.dataclass @@ -88,7 +84,7 @@ async def root(request: Request) -> Response: async def check_email_form(request: Request) -> Response: recipient_username = f"{binascii.hexlify(os.urandom(16)).decode('ascii')}" key = get_key_from_username(recipient_username) - REDIS.setex(b"requested-" + key, REDIS_MESSAGE_DATA_EXPIRY_SECONDS, 1) + REDIS.setex(b"requested-" + key, Config.Data.REDIS_MESSAGE_DATA_EXPIRY_SECONDS, 1) return RedirectResponse("/check-email/" + recipient_username) @@ -118,7 +114,7 @@ async def check_email_results(request: Request, recipient_username: str) -> Resp "request": request, "recipient_username": recipient_username, "recipient_address": recipient_address, - "site_contact_email": SITE_CONTACT_EMAIL, + "site_contact_email": Config.UI.SITE_CONTACT_EMAIL, }, ) @@ -130,7 +126,7 @@ async def check_email_results(request: Request, recipient_username: str) -> Resp from_domain, dkim_domain = get_from_and_dkim_domain(message_data) if not from_domain: result = None - error = translate("Invalid or no e-mail domain in the message From header", Language.pl_PL) + error = translate("Invalid or no e-mail domain in the message From header", Language(Config.UI.LANGUAGE)) else: try: result = scan_and_log( @@ -141,13 +137,13 @@ async def check_email_results(request: Request, recipient_username: str) -> Resp dkim_domain=dkim_domain, message=message_data, message_timestamp=message_timestamp, - nameservers=NAMESERVERS, - language=Language(LANGUAGE), + nameservers=Config.Network.NAMESERVERS, + language=Language(Config.UI.LANGUAGE), ) error = None except (DomainValidationException, ScanningException) as e: result = None - error = translate(e.message, Language.pl_PL) + error = translate(e.message, Language(Config.UI.LANGUAGE)) token = save_check_results( envelope_domain=envelope_domain, @@ -182,13 +178,13 @@ async def check_domain_scan_post(request: Request, domain: str = Form()) -> Resp dkim_domain=None, message=None, message_timestamp=None, - nameservers=NAMESERVERS, - language=Language(LANGUAGE), + nameservers=Config.Network.NAMESERVERS, + language=Language(Config.UI.LANGUAGE), ) error = None except (DomainValidationException, ScanningException) as e: result = None - error = translate(e.message, Language.pl_PL) + error = translate(e.message, Language(Config.UI.LANGUAGE)) token = save_check_results( envelope_domain=domain, @@ -240,8 +236,8 @@ async def check_domain_api(request: Request, domain: str) -> ScanAPICallResult: dkim_domain=None, message=None, message_timestamp=None, - nameservers=NAMESERVERS, - language=Language(LANGUAGE), + nameservers=Config.Network.NAMESERVERS, + language=Language(Config.UI.LANGUAGE), ) return ScanAPICallResult(result=result) except (DomainValidationException, ScanningException): diff --git a/app/src/app_utils.py b/app/src/app_utils.py index 52a6a34..b15eae8 100644 --- a/app/src/app_utils.py +++ b/app/src/app_utils.py @@ -7,11 +7,13 @@ from email.utils import parseaddr from typing import List, Optional, Tuple -import decouple import dkim.util from email_validator import EmailNotValidError, validate_email from fastapi import Request +from common.config import Config +from common.language import Language + from .db import ( DKIMImplementationMismatchLogEntry, NonexistentTranslationLogEntry, @@ -21,9 +23,8 @@ ) from .logging import build_logger from .scan import ScanResult, scan -from .translate import Language, translate_scan_result +from .translate import translate_scan_result -APP_DOMAIN = decouple.config("APP_DOMAIN") LOGGER = build_logger(__name__) @@ -129,7 +130,7 @@ def scan_and_log( def recipient_username_to_address(username: str) -> str: # We do not use the request hostname as due to proxy configuration we sometimes got the # Host header wrong, thus breaking the check-by-email feature. - return f"{username}@{APP_DOMAIN}" + return f"{username}@{Config.Network.APP_DOMAIN}" def _nonexistent_translation_handler(message: str) -> str: diff --git a/app/src/check_results.py b/app/src/check_results.py index 2eb3df7..3cfceea 100644 --- a/app/src/check_results.py +++ b/app/src/check_results.py @@ -6,14 +6,14 @@ from typing import Any, Dict, Optional import dacite -import decouple from redis import Redis +from common.config import Config + from .logging import build_logger from .scan import ScanResult -OLD_CHECK_RESULTS_AGE_MINUTES = decouple.config("OLD_CHECK_RESULTS_AGE_MINUTES", default=60, cast=int) -REDIS = Redis.from_url(decouple.config("REDIS_CONNECTION_STRING")) +REDIS = Redis.from_url(Config.Data.REDIS_URL) LOGGER = build_logger(__name__) @@ -90,8 +90,8 @@ def load_check_results(token: str) -> Optional[Dict[str, Any]]: config=dacite.Config(check_types=False), ) - result["age_threshold_minutes"] = OLD_CHECK_RESULTS_AGE_MINUTES + result["age_threshold_minutes"] = Config.UI.OLD_CHECK_RESULTS_AGE_MINUTES result["is_old"] = ( datetime.datetime.now() - result["created_at"] - ).total_seconds() > 60 * OLD_CHECK_RESULTS_AGE_MINUTES + ).total_seconds() > 60 * Config.UI.OLD_CHECK_RESULTS_AGE_MINUTES return result diff --git a/app/src/db.py b/app/src/db.py index dced133..086dfd7 100644 --- a/app/src/db.py +++ b/app/src/db.py @@ -1,12 +1,13 @@ import datetime -import os from enum import Enum from sqlalchemy import JSON, Boolean, Column, DateTime, Integer, String, create_engine from sqlalchemy.orm import declarative_base, sessionmaker # type: ignore +from common.config import Config + Base = declarative_base() -engine = create_engine(os.environ["DB_URL"]) +engine = create_engine(Config.Data.DB_URL) Session = sessionmaker(bind=engine) diff --git a/app/src/tags.py b/app/src/tags.py index 2b4d203..059cfd8 100644 --- a/app/src/tags.py +++ b/app/src/tags.py @@ -1,8 +1,9 @@ from functools import cache -import decouple from jinja2_simple_tags import StandaloneTag +from common.config import Config + class BuildIDTag(StandaloneTag): # type: ignore tags = {"build_id"} @@ -15,8 +16,8 @@ def render(self) -> str: class LanguageTag(StandaloneTag): # type: ignore tags = {"language"} - language = decouple.config("LANGUAGE", default="en_US").replace("_", "-") + language = Config.UI.LANGUAGE.replace("_", "-") @cache def render(self) -> str: - return self.language # type: ignore + return self.language diff --git a/app/src/translate.py b/app/src/translate.py index c4cacaf..f707f98 100644 --- a/app/src/translate.py +++ b/app/src/translate.py @@ -1,15 +1,10 @@ import copy import re -from enum import Enum from typing import Callable, List, Optional, Tuple -from .scan import DKIMScanResult, DomainScanResult, ScanResult - - -class Language(Enum): - en_US = "en_US" - pl_PL = "pl_PL" +from common.language import Language +from .scan import DKIMScanResult, DomainScanResult, ScanResult PLACEHOLDER = "__PLACEHOLDER__" SKIP_PLACEHOLDER = "__SKIP_PLACEHOLDER__" diff --git a/common/__init__.py b/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/common/config.py b/common/config.py new file mode 100644 index 0000000..4da409a --- /dev/null +++ b/common/config.py @@ -0,0 +1,92 @@ +from typing import Annotated, Any, List, get_type_hints + +from common.language import Language + +import decouple + +DEFAULTS = {} + + +def get_config(name: str, **kwargs) -> Any: # type: ignore + if "default" in kwargs: + DEFAULTS[name] = kwargs["default"] + return decouple.config(name, **kwargs) + + +class Config: + class Data: + DB_URL: Annotated[ + str, + "The URL used to connect to the database (as documented on " + "https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls). The CERT PL " + "production instance uses PostgreSQL - this is the database the system has been most " + "thoroughly tested on." + ] = get_config("DB_URL", default=None) + REDIS_MESSAGE_DATA_EXPIRY_SECONDS: Annotated[ + int, + "The data in Redis expires in order to make sure Redis never takes too much " + "memory - the full scan logs are stored in a Postgres database." + ] = get_config("REDIS_MESSAGE_DATA_EXPIRY_SECONDS", cast=int, default=10 * 24 * 60 * 60) + REDIS_URL: Annotated[ + str, + "The URL used to connect to Redis (as documented on " + "https://redis-py.readthedocs.io/en/stable/connections.html#redis.Redis.from_url), eg. " + "redis://redis:6379/0.", + ] = get_config("REDIS_URL") + + class Network: + APP_DOMAIN: Annotated[str, "The domain the site is running on."] = get_config("APP_DOMAIN") + NAMESERVERS: Annotated[ + List[str], + "A comma-separated list of nameservers that will be used to resolve domains. If you want " + "to provide custom ones, remember to modify the ones used by Docker. At CERT PL we use a " + "separate `docker-compose.yml` file with additional configuration specific to our instance." + ] = get_config("NAMESERVERS", default="8.8.8.8", cast=decouple.Csv(str)) + SSL_PRIVATE_KEY_PATH: Annotated[ + str, + "SSL private key path. Remember:\n" + " 1. to mount it into your Docker container,\n" + " 2. to restart the containers if a new one is generated,\n" + " 2. that autogenerated certificate files may be symbolic links - the destination directory must also be mounted." + ] = decouple.config("SSL_PRIVATE_KEY_PATH", default=None) + SSL_CERTIFICATE_PATH: Annotated[str, "SSL certificate path."] = decouple.config("SSL_PRIVATE_KEY_PATH", default=None) + + class UI: + LANGUAGE: Annotated[ + str, + "The language the site will use (in the form of language_COUNTRY, e.g. en_US). " + f"Supported options are: {', '.join(sorted(language.value for language in Language))}." + ] = get_config("LANGUAGE", default="en_US") + OLD_CHECK_RESULTS_AGE_MINUTES: Annotated[ + int, + "If the user is viewing old check results, they will see a message that the check result " + "may not describe the current configuration. This is the threshold (in minutes) how old " + "the check results need to be for that message to be displayed." + ] = get_config("OLD_CHECK_RESULTS_AGE_MINUTES", default=60, cast=int) + SITE_CONTACT_EMAIL: Annotated[ + str, + "The contact e-mail that will be displayed in the UI (currently in the message that " + "describes what to do if e-mails to the system aren't received)." + ] = get_config("SITE_CONTACT_EMAIL", default=None) + + @staticmethod + def verify_each_variable_is_annotated() -> None: + def verify_class(cls: type) -> None: + hints = get_type_hints(cls) + + for variable_name in dir(cls): + if variable_name.startswith("__"): + continue + member = getattr(cls, variable_name) + + if isinstance(member, type): + verify_class(member) + elif member == Config.verify_each_variable_is_annotated: + pass + else: + assert variable_name in hints, f"{variable_name} in {cls} has no type hint" + + verify_class(Config) + + +Config.verify_each_variable_is_annotated() diff --git a/common/language.py b/common/language.py new file mode 100644 index 0000000..5c9c282 --- /dev/null +++ b/common/language.py @@ -0,0 +1,6 @@ +from enum import Enum + + +class Language(Enum): + en_US = "en_US" + pl_PL = "pl_PL" diff --git a/docker-compose.yml b/docker-compose.yml index a35ab1e..6f3c60a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: environment: DB_URL: postgresql+psycopg2://postgres:postgres@db:5432/mailgoose FORWARDED_ALLOW_IPS: "*" - REDIS_CONNECTION_STRING: redis://redis:6379/0 + REDIS_URL: redis://redis:6379/0 env_file: - .env command: bash -c "/wait-for-it.sh db:5432 -- uvicorn src.app:app --host 0.0.0.0 --port 8000 --proxy-headers --workers 8" @@ -28,7 +28,7 @@ services: dockerfile: mail_receiver/Dockerfile command: python3 /opt/server.py environment: - REDIS_CONNECTION_STRING: redis://redis:6379/0 + REDIS_URL: redis://redis:6379/0 env_file: - .env volumes: diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..f1df6d9 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,2 @@ +_build +venv/ diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..7d6bf9f --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,77 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +import datetime +import os +import sys + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) +import sphinx_rtd_theme # noqa + +sys.path.insert(0, os.path.abspath("../common")) +sys.path.insert(0, os.path.abspath("..")) +sys.path.insert(0, os.path.abspath(".")) + + +# -- Project information ----------------------------------------------------- + +project = "mailgoose" +copyright = f"{datetime.datetime.now().year}, CERT Polska" +author = "CERT Polska" + +# The full version, including alpha/beta/rc tags +release = "0.1.0" + +latex_engine = "xelatex" + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx_rtd_theme", + "sphinx.ext.graphviz", + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "generate_config_docs", +] + +graphviz_output_format = "svg" + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "venv"] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = [] + + +html_css_files = [ + "style.css", +] diff --git a/docs/features.rst b/docs/features.rst new file mode 100644 index 0000000..cd96d7d --- /dev/null +++ b/docs/features.rst @@ -0,0 +1,20 @@ +Features +======== + +- checking **SPF** and **DMARC** configuration by providing a domain, +- checking **SPF**, **DMARC** and **DKIM** by sending a test e-mail, +- SMTP SSL support for incoming e-mails (please refer to ``SSL_CERTIFICATE_PATH`` and + ``SSL_PRIVATE_KEY_PATH`` settings in :doc:`user-guide/configuration` to learn how to set it up), +- easy translation to a different language, +- easy layout customization, +- REST API. + +REST API +-------- +REST API documentation is auto-generated by the FastAPI framework in the form of +Swagger and is available at your mailgoose instance under ``/docs`` URL. To check +whether a domain has SPF and DMARC set up correctly, use: + +.. code-block:: console + + curl -X POST http://127.0.0.1:8000/api/v1/check-domain?domain=example.com diff --git a/docs/generate_config_docs.py b/docs/generate_config_docs.py new file mode 100644 index 0000000..68ca39f --- /dev/null +++ b/docs/generate_config_docs.py @@ -0,0 +1,60 @@ +import os +import textwrap +from pathlib import Path +from typing import IO, Any, get_type_hints + +# By default, these variables are required by config. As we are importing the config +# only to get the docs, let's mock them. +os.environ["APP_DOMAIN"] = "" +os.environ["DB_URL"] = "" +os.environ["REDIS_URL"] = "" +from config import DEFAULTS, Config # type: ignore # noqa +from sphinx.application import Sphinx # type: ignore # noqa + + +def setup(app: Sphinx) -> None: + app.connect("config-inited", on_config_inited) + + +def on_config_inited(_1: Any, _2: Any) -> None: + output = Path(__file__).parents[0] / "user-guide" / "config-docs.inc" + + with open(output, "w") as f: + print_docs_for_class(Config, output_file=f) + + +def print_docs_for_class(cls: type, output_file: IO[str], depth: int = 0) -> None: + if depth > 0: + output_file.write(cls.__name__ + "\n") + + header_characters = '-^"' + output_file.write(header_characters[depth - 1] * len(cls.__name__) + "\n\n") + + hints = get_type_hints(cls, include_extras=True) + for variable_name in dir(cls): + if variable_name.startswith("__"): + continue + + member = getattr(cls, variable_name) + if isinstance(member, type): + print_docs_for_class(member, output_file, depth + 1) + continue + elif member == Config.verify_each_variable_is_annotated: + continue + + (hint,) = hints[variable_name].__metadata__ + indent = 4 * " " + doc = "\n".join(textwrap.wrap(hint.strip(), width=100, initial_indent=indent, subsequent_indent=indent)) + if variable_name in DEFAULTS: + default_str = f"{indent}Default: {DEFAULTS[variable_name]}\n\n" + else: + default_str = "" + + output_file.write( + textwrap.dedent( + f""" + {variable_name}\n{default_str}{doc} + """.strip() + ) + + "\n\n" + ) diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..2f7ad01 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,14 @@ +Welcome to mailgoose documentation! +==================================== + +Mailgoose is a web application that allows the users to check whether their SPF, DMARC and +DKIM configuration is set up correctly. + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + quick-start + features + user-guide/configuration + user-guide/translation diff --git a/docs/quick-start.rst b/docs/quick-start.rst new file mode 100644 index 0000000..09e2810 --- /dev/null +++ b/docs/quick-start.rst @@ -0,0 +1,56 @@ +Quick Start +=========== + +Running mailgoose locally +------------------------- +To run the service locally, use: + +.. code-block:: console + + cp env.example .env + + # Customize the settings in .env + + docker compose up --build + +The application will listen on http://127.0.0.1:8000 . + +To start the application, you need to configure only the variables present in +``env.example`` - all others are optional. To learn what settings are available, +please refer to :doc:`user-guide/configuration`. + +Production deployment +--------------------- +Before deploying the system using the configuration in ``docker-compose.yml`` remember: + +- that the ``mail_receiver`` container is responsible for saving incoming mails to + Redis - make sure ports 25 and 587 are exposed publicly so that mailgoose will be able + to receive a test e-mail. Make sure the domain configured in the ``APP_DOMAIN`` setting has ``MX`` DNS + records pointing to the server ``mail_receiver`` is running on, +- that SMTP SSL is supported - please refer to ``SSL_CERTIFICATE_PATH`` and ``SSL_PRIVATE_KEY_PATH`` + settings description in :doc:`user-guide/configuration` to learn how to set it up, +- to change the database password to a more secure one and to use Redis password (or make sure + the database and Redis are isolated on the network, +- to decide whether you want to launch a database/Redis instance inside a container or + e.g. attaching to your own PostgreSQL/Redis cluster, +- to check whether you want to use Google nameservers or other ones. + +Instead of copying ``docker-compose.yml``, you may override the configuration using the +``docker compose -f docker-compose.yml -f docker-compose.override.yml`` syntax. + +Changing the layout +------------------- +If you want to change the main layout template (e.g. to provide additional scripts or your own +custom navbar with logo), mount a different file using Docker to ``/app/templates/custom_layout.html``. +Refer to ``app/templates/base.html`` to learn what block you can fill. + +You can also customize the root page (/) of the system by providing your own file that will +replace ``/app/templates/custom_root_layout.html``. + +By replacing ``/app/templates/custom_failed_check_result_hints.html`` you may provide your own +text that will be displayed if the e-mail sender verification mechanisms checks fail. + +At CERT PL we use a separate ``docker-compose.yml`` file with additional configuration +specific to our instance (https://bezpiecznapoczta.cert.pl/). Instead of copying +``docker-compose.yml``, we override the configuration using the +``docker compose -f docker-compose.yml -f docker-compose.override.yml`` syntax. diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..f5b5e81 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,3 @@ +python-decouple==3.8 +Sphinx==7.2.6 +sphinx-rtd-theme==2.0.0 diff --git a/docs/user-guide/.gitignore b/docs/user-guide/.gitignore new file mode 100644 index 0000000..cca5d79 --- /dev/null +++ b/docs/user-guide/.gitignore @@ -0,0 +1 @@ +config-docs.inc diff --git a/docs/user-guide/configuration.rst b/docs/user-guide/configuration.rst new file mode 100644 index 0000000..d7fb00c --- /dev/null +++ b/docs/user-guide/configuration.rst @@ -0,0 +1,7 @@ +Configuration options +===================== + +Mailgoose can be configured by setting the following variables in the ``.env`` file (in the form of ``VARIABLE_NAME=VARIABLE_VALUE`` +directives, e.g. ``APP_DOMAIN=example.com``): + +.. include:: config-docs.inc diff --git a/docs/user-guide/translation.rst b/docs/user-guide/translation.rst new file mode 100644 index 0000000..a111f0f --- /dev/null +++ b/docs/user-guide/translation.rst @@ -0,0 +1,32 @@ +Translation +=========== +The UI translations reside in ``./app/translations``. If the original messages changed, e.g. because +you changed the UI messages, update the ``.po`` files by running: + +``./scripts/update_translation_files`` + +and then put the translations in the respective ``.po`` files. The compilation will happen +automatically when starting the system. + +The error message translations for DMARC/SPF/DKIM problems reside in the ``TRANSLATIONS`` dict in ``app/src/translate.py``. +The following syntax: + +.. code-block:: console + + ( + f"{PLACEHOLDER} is not a valid DMARC report URI", + f"{PLACEHOLDER} ... translation for your language...", + ), + + +means, that the ``{PLACEHOLDER}`` part will be copied verbatim into the translation - this is to +support situations, where the error message contains e.g. a domain, a URI or other part dependent on the configuration. + +Adding a new language +--------------------- +If you want to support a new language: + +- add it in ``./scripts/update_translation_files`` in the language list, +- add it in ``common/language.py`` in the language enum, +- run ``./scripts/update_translation_files`` and fill ``.po`` files for your language in ``./app/translations``, +- add the error message translations for your language in ``app/src/translate.py``. diff --git a/env.example b/env.example index ccce083..e2aad12 100644 --- a/env.example +++ b/env.example @@ -1,7 +1 @@ -LANGUAGE=en_US - APP_DOMAIN=change-this.example.com - -# 10 days to facillitate debugging. The data in Redis expires in order to make sure Redis never takes too much -# memory - the full scan logs are stored in a Postgres database. -REDIS_MESSAGE_DATA_EXPIRY_SECONDS=864000 diff --git a/mail_receiver/Dockerfile b/mail_receiver/Dockerfile index 4e9e008..74ef8b8 100644 --- a/mail_receiver/Dockerfile +++ b/mail_receiver/Dockerfile @@ -8,5 +8,5 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone COPY mail_receiver/requirements.txt /requirements.txt RUN pip install -r /requirements.txt -COPY common/mail_receiver_utils.py /opt/mail_receiver_utils.py +COPY common /opt/common/ COPY mail_receiver/server.py /opt/server.py diff --git a/mail_receiver/server.py b/mail_receiver/server.py index 73cc2a8..57426ae 100644 --- a/mail_receiver/server.py +++ b/mail_receiver/server.py @@ -7,13 +7,14 @@ from email.message import Message as EmailMessage from typing import Any, Dict, Optional, Sequence, Union -import decouple from aiosmtpd.controller import Controller from aiosmtpd.handlers import Message as BaseMessageHandler from aiosmtpd.smtp import SMTP, Envelope, Session -from mail_receiver_utils import get_key_from_username from redis import Redis +from common.config import Config +from common.mail_receiver_utils import get_key_from_username + logging.basicConfig( format="%(asctime)s %(levelname)-8s %(message)s", level=logging.INFO, @@ -21,19 +22,15 @@ ) LOGGER = logging.getLogger(__name__) -REDIS = Redis.from_url(decouple.config("REDIS_CONNECTION_STRING")) -REDIS_MESSAGE_DATA_EXPIRY_SECONDS = decouple.config("REDIS_MESSAGE_DATA_EXPIRY_SECONDS") - -SSL_PRIVATE_KEY_PATH = decouple.config("SSL_PRIVATE_KEY_PATH", default=None) -SSL_CERTIFICATE_PATH = decouple.config("SSL_CERTIFICATE_PATH", default=None) +REDIS = Redis.from_url(Config.Data.REDIS_URL) -if SSL_PRIVATE_KEY_PATH and SSL_CERTIFICATE_PATH: - assert os.path.exists(SSL_PRIVATE_KEY_PATH) - assert os.path.exists(SSL_CERTIFICATE_PATH) +if Config.Network.SSL_PRIVATE_KEY_PATH and Config.Network.SSL_CERTIFICATE_PATH: + assert os.path.exists(Config.Network.SSL_PRIVATE_KEY_PATH) + assert os.path.exists(Config.Network.SSL_CERTIFICATE_PATH) LOGGER.info("SSL key and certificate exist, creating context") SSL_CONTEXT: Optional[ssl.SSLContext] = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) assert SSL_CONTEXT - SSL_CONTEXT.load_cert_chain(SSL_CERTIFICATE_PATH, SSL_PRIVATE_KEY_PATH) + SSL_CONTEXT.load_cert_chain(Config.Network.SSL_CERTIFICATE_PATH, Config.Network.SSL_PRIVATE_KEY_PATH) else: LOGGER.info("SSL key and certificate don't exist, not creating context") SSL_CONTEXT = None @@ -74,15 +71,15 @@ def handle_DATA(self, server: SMTP, session: Session, envelope: Envelope) -> Any len(content), key, ) - REDIS.setex(key, REDIS_MESSAGE_DATA_EXPIRY_SECONDS, content) + REDIS.setex(key, Config.Data.REDIS_MESSAGE_DATA_EXPIRY_SECONDS, content) REDIS.setex( key + b"-timestamp", - REDIS_MESSAGE_DATA_EXPIRY_SECONDS, + Config.Data.REDIS_MESSAGE_DATA_EXPIRY_SECONDS, datetime.datetime.now().isoformat(), ) REDIS.setex( key + b"-sender", - REDIS_MESSAGE_DATA_EXPIRY_SECONDS, + Config.Data.REDIS_MESSAGE_DATA_EXPIRY_SECONDS, mail_from, ) LOGGER.info("Saved")