diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 31c8be7..2f6e3a6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,6 +25,7 @@ repos: - email-validator==2.1.0.post1 - fastapi==0.104.1 - Jinja2==3.1.2 + - rq==1.15.1 - sqlalchemy-stubs==0.4 - types-redis==4.6.0.11 - types-requests==2.31.0.10 diff --git a/app/requirements.txt b/app/requirements.txt index d2d2a65..c6b16d1 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -7,6 +7,7 @@ jinja2-simple-tags==0.5.0 psycopg2-binary==2.9.9 python-decouple==3.8 redis==5.0.1 +rq==1.15.1 SQLAlchemy==2.0.25 uvicorn==0.26.0 -r translations/requirements.txt diff --git a/app/src/app.py b/app/src/app.py index 7e0bdd6..46f5627 100644 --- a/app/src/app.py +++ b/app/src/app.py @@ -1,6 +1,5 @@ import binascii import dataclasses -import datetime import os import time import traceback @@ -12,27 +11,25 @@ from fastapi.staticfiles import StaticFiles from libmailgoose.language import Language from libmailgoose.scan import DomainValidationException, ScanningException, ScanResult -from libmailgoose.translate import translate from redis import Redis +from rq import Queue from starlette.responses import Response from common.config import Config from common.mail_receiver_utils import get_key_from_username -from .app_utils import ( - get_from_and_dkim_domain, - recipient_username_to_address, - scan_and_log, -) -from .check_results import load_check_results, save_check_results +from .app_utils import recipient_username_to_address, scan_and_log +from .check_results import load_check_results from .db import ScanLogEntrySource, ServerErrorLogEntry, Session from .logging import build_logger from .resolver import setup_resolver from .templates import setup_templates +from .worker import scan_domain_job, scan_message_and_domain_job app = FastAPI() LOGGER = build_logger(__name__) REDIS = Redis.from_url(Config.Data.REDIS_URL) +job_queue = Queue(connection=REDIS) app.mount("/static", StaticFiles(directory="static"), name="static") @@ -118,42 +115,24 @@ async def check_email_results(request: Request, recipient_username: str) -> Resp }, ) - message_timestamp = datetime.datetime.fromisoformat(message_timestamp_raw.decode("ascii")) - _, mail_from = parseaddr(mail_from_raw.decode("ascii")) _, envelope_domain = tuple(mail_from.split("@", 1)) - from_domain, dkim_domain = get_from_and_dkim_domain(message_data) - if not from_domain: - result = None - error = translate("Invalid or no e-mail domain in the message From header", Language(Config.UI.LANGUAGE)) - else: - try: - result = scan_and_log( - request=request, - source=ScanLogEntrySource.GUI, - envelope_domain=envelope_domain, - from_domain=from_domain, - dkim_domain=dkim_domain, - message=message_data, - message_timestamp=message_timestamp, - nameservers=Config.Network.NAMESERVERS, - language=Language(Config.UI.LANGUAGE), - ) - error = None - except (DomainValidationException, ScanningException) as e: - result = None - error = translate(e.message, Language(Config.UI.LANGUAGE)) - - token = save_check_results( - envelope_domain=envelope_domain, - from_domain=from_domain or envelope_domain, - dkim_domain=dkim_domain, - result=result, - error=error, - rescan_url="/check-email/", - message_recipient_username=recipient_username, + client_ip = request.client.host if request.client else None + client_user_agent = request.headers.get("user-agent", None) + + token = binascii.hexlify(os.urandom(32)).decode("ascii") + job_queue.enqueue( + scan_message_and_domain_job, + client_ip, + client_user_agent, + envelope_domain, + token, + key, + recipient_username, + job_id=token, ) + return RedirectResponse(f"/check-results/{token}", status_code=302) @@ -169,37 +148,24 @@ async def check_domain_scan_get(request: Request) -> Response: @app.post("/check-domain/scan", response_class=HTMLResponse, include_in_schema=False) async def check_domain_scan_post(request: Request, domain: str = Form()) -> Response: - try: - result = scan_and_log( - request=request, - source=ScanLogEntrySource.GUI, - envelope_domain=domain, - from_domain=domain, - dkim_domain=None, - message=None, - message_timestamp=None, - nameservers=Config.Network.NAMESERVERS, - language=Language(Config.UI.LANGUAGE), - ) - error = None - except (DomainValidationException, ScanningException) as e: - result = None - error = translate(e.message, Language(Config.UI.LANGUAGE)) - - token = save_check_results( - envelope_domain=domain, - from_domain=domain, - dkim_domain=None, - result=result, - error=error, - rescan_url="/check-domain/", - message_recipient_username=None, - ) + client_ip = request.client.host if request.client else None + client_user_agent = request.headers.get("user-agent", None) + + token = binascii.hexlify(os.urandom(32)).decode("ascii") + job_queue.enqueue(scan_domain_job, client_ip, client_user_agent, domain, token, job_id=token) + return RedirectResponse(f"/check-results/{token}", status_code=302) @app.get("/check-results/{token}", response_class=HTMLResponse, include_in_schema=False) async def check_results(request: Request, token: str) -> Response: + if job := job_queue.fetch_job(token): + if job.get_status(refresh=False) not in ["finished", "canceled", "failed"]: + return templates.TemplateResponse( + "check_running.html", + {"request": request}, + ) + check_results = load_check_results(token) if not check_results: @@ -228,8 +194,10 @@ async def check_domain_api(request: Request, domain: str) -> ScanAPICallResult: object will be empty, as DKIM can't be checked when given only a domain. """ try: + client_ip = request.client.host if request.client else None + client_user_agent = request.headers.get("user-agent", None) + result = scan_and_log( - request=request, source=ScanLogEntrySource.API, envelope_domain=domain, from_domain=domain, @@ -238,6 +206,8 @@ async def check_domain_api(request: Request, domain: str) -> ScanAPICallResult: message_timestamp=None, nameservers=Config.Network.NAMESERVERS, language=Language(Config.UI.LANGUAGE), + client_ip=client_ip, + client_user_agent=client_user_agent, ) return ScanAPICallResult(result=result) except (DomainValidationException, ScanningException): diff --git a/app/src/app_utils.py b/app/src/app_utils.py index 9325ed7..dc13083 100644 --- a/app/src/app_utils.py +++ b/app/src/app_utils.py @@ -9,7 +9,6 @@ import dkim.util from email_validator import EmailNotValidError, validate_email -from fastapi import Request from libmailgoose.language import Language from libmailgoose.scan import ScanResult, scan from libmailgoose.translate import translate_scan_result @@ -70,7 +69,6 @@ def dkim_implementation_mismatch_callback(message: bytes, dkimpy_valid: bool, op def scan_and_log( - request: Request, source: ScanLogEntrySource, envelope_domain: str, from_domain: str, @@ -79,6 +77,8 @@ def scan_and_log( message_timestamp: Optional[datetime.datetime], nameservers: List[str], language: Language, + client_ip: Optional[str], + client_user_agent: Optional[str], ) -> ScanResult: scan_log_entry = ScanLogEntry( envelope_domain=envelope_domain, @@ -86,8 +86,8 @@ def scan_and_log( dkim_domain=dkim_domain, message=message, source=source.value, - client_ip=request.client.host if request.client else None, - client_user_agent=request.headers.get("user-agent", None), + client_ip=client_ip, + client_user_agent=client_user_agent, check_options={ "nameservers": nameservers, }, diff --git a/app/src/check_results.py b/app/src/check_results.py index 73747e2..b25936b 100644 --- a/app/src/check_results.py +++ b/app/src/check_results.py @@ -1,8 +1,6 @@ -import binascii import dataclasses import datetime import json -import os from typing import Any, Dict, Optional import dacite @@ -35,8 +33,8 @@ def save_check_results( error: Optional[str], rescan_url: str, message_recipient_username: Optional[str], -) -> str: - token = binascii.hexlify(os.urandom(32)).decode("ascii") + token: str, +) -> None: # We don't use HSET or HMSET, as result is a recursive dict, and values that can be stored # using HSET/HMSET are bytes, string, int or float, so we still wouldn't avoid serialization. REDIS.set( @@ -56,7 +54,6 @@ def save_check_results( cls=JSONEncoderAdditionalTypes, ), ) - return token def load_check_results(token: str) -> Optional[Dict[str, Any]]: diff --git a/app/src/worker.py b/app/src/worker.py new file mode 100644 index 0000000..866a09e --- /dev/null +++ b/app/src/worker.py @@ -0,0 +1,107 @@ +import datetime +from typing import Optional + +from libmailgoose.language import Language +from libmailgoose.scan import DomainValidationException, ScanningException +from libmailgoose.translate import translate +from redis import Redis + +from common.config import Config + +from .app_utils import get_from_and_dkim_domain, scan_and_log +from .check_results import save_check_results +from .db import ScanLogEntrySource +from .logging import build_logger +from .resolver import setup_resolver + +LOGGER = build_logger(__name__) +REDIS = Redis.from_url(Config.Data.REDIS_URL) + +setup_resolver() + + +def scan_domain_job( + client_ip: Optional[str], + client_user_agent: Optional[str], + domain: str, + token: str, +) -> None: + try: + result = scan_and_log( + source=ScanLogEntrySource.GUI, + envelope_domain=domain, + from_domain=domain, + dkim_domain=None, + message=None, + message_timestamp=None, + nameservers=Config.Network.NAMESERVERS, + language=Language(Config.UI.LANGUAGE), + client_ip=client_ip, + client_user_agent=client_user_agent, + ) + error = None + except (DomainValidationException, ScanningException) as e: + result = None + error = translate(e.message, Language(Config.UI.LANGUAGE)) + + save_check_results( + envelope_domain=domain, + from_domain=domain, + dkim_domain=None, + result=result, + error=error, + rescan_url="/check-domain/", + message_recipient_username=None, + token=token, + ) + + +def scan_message_and_domain_job( + client_ip: Optional[str], + client_user_agent: Optional[str], + envelope_domain: str, + token: str, + message_key: bytes, + recipient_username: str, +) -> None: + message_data = REDIS.get(message_key) + message_timestamp_raw = REDIS.get(message_key + b"-timestamp") + + if not message_data or not message_timestamp_raw: + raise RuntimeError("Worker coudn't access message data") + + message_timestamp = datetime.datetime.fromisoformat(message_timestamp_raw.decode("ascii")) + + from_domain, dkim_domain = get_from_and_dkim_domain(message_data) + if not from_domain: + result = None + error = translate("Invalid or no e-mail domain in the message From header", Language(Config.UI.LANGUAGE)) + else: + try: + result = scan_and_log( + source=ScanLogEntrySource.GUI, + envelope_domain=envelope_domain, + from_domain=from_domain, + dkim_domain=dkim_domain, + message=message_data, + message_timestamp=message_timestamp, + nameservers=Config.Network.NAMESERVERS, + language=Language(Config.UI.LANGUAGE), + client_ip=client_ip, + client_user_agent=client_user_agent, + ) + error = None + except (DomainValidationException, ScanningException) as e: + result = None + error = translate(e.message, Language(Config.UI.LANGUAGE)) + + save_check_results( + envelope_domain=envelope_domain, + from_domain=from_domain or envelope_domain, + dkim_domain=dkim_domain, + result=result, + error=error, + rescan_url="/check-email/", + message_recipient_username=recipient_username, + token=token, + ) diff --git a/app/templates/check_running.html b/app/templates/check_running.html new file mode 100644 index 0000000..852c7c5 --- /dev/null +++ b/app/templates/check_running.html @@ -0,0 +1,20 @@ +{% extends "custom_layout.html" %} + +{% block header_additional %} + +{% endblock %} + +{% block body %} +
+
+

{% trans %}Configuration analysis is running{% endtrans %}

+
+ +
+
{% trans %}Waiting for the analysis to finish{% endtrans %}
+
{% trans %}This page will refresh automatically.{% endtrans %}
+
+
+
+
+{% endblock %} diff --git a/app/translations/en_US/LC_MESSAGES/messages.po b/app/translations/en_US/LC_MESSAGES/messages.po index 10cb03d..33ac716 100644 --- a/app/translations/en_US/LC_MESSAGES/messages.po +++ b/app/translations/en_US/LC_MESSAGES/messages.po @@ -240,6 +240,18 @@ msgid "" "detected only if earlier checks complete successfully." msgstr "" +#: app/templates/check_running.html:10 +msgid "Configuration analysis is running" +msgstr "" + +#: app/templates/check_running.html:14 +msgid "Waiting for the analysis to finish" +msgstr "" + +#: app/templates/check_running.html:15 +msgid "This page will refresh automatically." +msgstr "" + #: app/templates/root.html:9 msgid "" "Verify your DKIM, DMARC, and SPF settings by sending" diff --git a/app/translations/messages.pot b/app/translations/messages.pot index 10cb03d..33ac716 100644 --- a/app/translations/messages.pot +++ b/app/translations/messages.pot @@ -240,6 +240,18 @@ msgid "" "detected only if earlier checks complete successfully." msgstr "" +#: app/templates/check_running.html:10 +msgid "Configuration analysis is running" +msgstr "" + +#: app/templates/check_running.html:14 +msgid "Waiting for the analysis to finish" +msgstr "" + +#: app/templates/check_running.html:15 +msgid "This page will refresh automatically." +msgstr "" + #: app/templates/root.html:9 msgid "" "Verify your DKIM, DMARC, and SPF settings by sending" diff --git a/app/translations/pl_PL/LC_MESSAGES/messages.po b/app/translations/pl_PL/LC_MESSAGES/messages.po index dce71a2..b176f49 100644 --- a/app/translations/pl_PL/LC_MESSAGES/messages.po +++ b/app/translations/pl_PL/LC_MESSAGES/messages.po @@ -267,6 +267,18 @@ msgstr "" "Po poprawie błędów prosimy ponowić skanowanie - niektóre błędy mogą " "zostać znalezione dopiero po udanym wykonaniu wcześniejszych testów." +#: app/templates/check_running.html:10 +msgid "Configuration analysis is running" +msgstr "Trwa analiza konfiguracji" + +#: app/templates/check_running.html:14 +msgid "Waiting for the analysis to finish" +msgstr "Oczekiwanie na zakończenie analizy" + +#: app/templates/check_running.html:15 +msgid "This page will refresh automatically." +msgstr "Strona odświeży się automatycznie." + #: app/templates/root.html:9 msgid "" "Verify your DKIM, DMARC, and SPF settings by sending" diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 8a9ce06..46fbd9e 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -13,6 +13,17 @@ services: REDIS_MESSAGE_DATA_EXPIRY_SECONDS: 864000 REDIS_URL: redis://redis-test:6379/0 command: bash -c "/wait-for-it.sh db-test:5432 -- uvicorn src.app:app --host 0.0.0.0 --port 8000 --proxy-headers" + worker: + build: + context: . + dockerfile: app/docker/Dockerfile + environment: + APP_DOMAIN: "app" + DB_URL: postgresql+psycopg2://postgres:postgres@db-test:5432/mailgoose + LANGUAGE: en_US + REDIS_MESSAGE_DATA_EXPIRY_SECONDS: 864000 + REDIS_URL: redis://redis-test:6379/0 + command: bash -c "/wait-for-it.sh db:5432 -- rq worker" db-test: image: postgres:15.2-alpine environment: diff --git a/docker-compose.yml b/docker-compose.yml index e66349b..8d85134 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,6 +22,17 @@ services: ports: - 8000:8000 <<: *common-configuration + worker: + build: + context: . + dockerfile: app/docker/Dockerfile + environment: + DB_URL: postgresql+psycopg2://postgres:postgres@db:5432/mailgoose + REDIS_URL: redis://redis:6379/0 + env_file: + - .env + command: bash -c "/wait-for-it.sh db:5432 -- rq worker" + <<: *common-configuration mail_receiver: build: context: . diff --git a/test/base.py b/test/base.py index 31b9c3d..6189820 100644 --- a/test/base.py +++ b/test/base.py @@ -1,3 +1,4 @@ +from time import sleep from typing import Any, Dict from unittest import TestCase @@ -7,8 +8,24 @@ class BaseTestCase(TestCase): def check_domain(self, domain: str) -> str: - response = requests.post(APP_URL + "/check-domain/scan", {"domain": domain}) - return response.text.replace("\n", " ") + submission_response = requests.post(APP_URL + "/check-domain/scan", {"domain": domain}, allow_redirects=False) + submission_response.raise_for_status() + + if not submission_response.next or not submission_response.next.url: + raise RuntimeError("Did not receive a redirect after submitting domain for scanning") + + results_url = submission_response.next.url + + results_response = requests.get(results_url) + results_response.raise_for_status() + + while "Configuration analysis is running" in results_response.text: + results_response = requests.get(results_url) + results_response.raise_for_status() + + sleep(5) + + return results_response.text.replace("\n", " ") def check_domain_api_v1(self, domain: str) -> Dict[str, Any]: response = requests.post(APP_URL + "/api/v1/check-domain?domain=" + domain)