Fixing broken checkdmarc release, better message and logging on worker fail

.pre-commit-config.yaml

@@ -5,16 +5,16 @@ repos:
   - id: end-of-file-fixer
   - id: trailing-whitespace
 - repo: https://github.com/psf/black
-  rev: 23.11.0
+  rev: 24.1.1
   hooks:
   - id: black
 - repo: https://github.com/pycqa/isort
-  rev: 5.12.0
+  rev: 5.13.2
   hooks:
   - id: isort
     name: isort (python)
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.7.1
+  rev: v1.8.0
   hooks:
   - id: mypy
     args: [--strict]
@@ -30,7 +30,7 @@ repos:
     - types-redis==4.6.0.11
     - types-requests==2.31.0.10
 - repo: https://github.com/PyCQA/flake8
-  rev: 6.1.0
+  rev: 7.0.0
   hooks:
   - id: flake8
     args: [.]

app/src/worker.py

@@ -1,4 +1,5 @@
 import datetime
+import traceback
 from typing import Optional
 
 from libmailgoose.language import Language
@@ -10,7 +11,7 @@
 
 from .app_utils import get_from_and_dkim_domain, scan_and_log
 from .check_results import save_check_results
-from .db import ScanLogEntrySource
+from .db import ScanLogEntrySource, ServerErrorLogEntry, Session
 from .logging import build_logger
 from .resolver import setup_resolver
 
@@ -43,6 +44,14 @@ def scan_domain_job(
     except (DomainValidationException, ScanningException) as e:
         result = None
         error = translate(e.message, Language(Config.UI.LANGUAGE))
+    except Exception:
+        session = Session()
+        server_error_log_entry = ServerErrorLogEntry(url="worker", error=traceback.format_exc())
+        session.add(server_error_log_entry)
+        session.commit()
+
+        result = None
+        error = translate("An unknown error has occured during configuration validation.", Language(Config.UI.LANGUAGE))
 
     save_check_results(
         envelope_domain=domain,

mail_receiver/Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.12-alpine3.18
 
-RUN apk add tzdata
+RUN apk add tzdata git
 
 ENV TZ=Europe/Warsaw
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone

scan/libmailgoose/scan.py

@@ -592,13 +592,15 @@ def scan(
             dkim_domain=dkim_domain,
             nameservers=nameservers,
         ),
-        dkim=scan_dkim(
-            message=message,
-            message_parsed=message_parsed,
-            dkim_implementation_mismatch_callback=dkim_implementation_mismatch_callback,
-        )
-        if message and message_parsed
-        else None,
+        dkim=(
+            scan_dkim(
+                message=message,
+                message_parsed=message_parsed,
+                dkim_implementation_mismatch_callback=dkim_implementation_mismatch_callback,
+            )
+            if message and message_parsed
+            else None
+        ),
         timestamp=datetime.datetime.now(),
         message_timestamp=message_timestamp,
     )

scan/libmailgoose/translate.py

@@ -381,6 +381,10 @@
             f"Failed to retrieve MX records for the domain of {PLACEHOLDER} email address {PLACEHOLDER} - The domain {PLACEHOLDER} does not exist",
             f"Nie udało się pobrać rekordów MX domeny adresu e-mail podanego w tagu '{PLACEHOLDER}': {PLACEHOLDER} - domena {PLACEHOLDER} nie istnieje.",
         ),
+        (
+            "An unknown error has occured during configuration validation.",
+            "Wystąpił nieznany błąd podczas sprawdzania konfiguracji.",
+        ),
         # dkimpy messages
         (
             f"{PLACEHOLDER} value is not valid base64 {PLACEHOLDER}",
@@ -605,12 +609,16 @@ def translate_scan_result(
     nonexistent_translation_handler: Optional[Callable[[str], str]] = None,
 ) -> ScanResult:
     return ScanResult(
-        domain=_translate_domain_result(scan_result.domain, language, nonexistent_translation_handler)
-        if scan_result.domain
-        else None,
-        dkim=_translate_dkim_result(scan_result.dkim, language, nonexistent_translation_handler)
-        if scan_result.dkim
-        else None,
+        domain=(
+            _translate_domain_result(scan_result.domain, language, nonexistent_translation_handler)
+            if scan_result.domain
+            else None
+        ),
+        dkim=(
+            _translate_dkim_result(scan_result.dkim, language, nonexistent_translation_handler)
+            if scan_result.dkim
+            else None
+        ),
         timestamp=scan_result.timestamp,
         message_timestamp=scan_result.message_timestamp,
     )

scan/requirements.txt

@@ -1,3 +1,5 @@
+# This commit hash refers to 5.3.1 release - but the PyPI release is incorrect
+checkdmarc @ git+https://github.com/domainaware/checkdmarc@120ef5a9709fe49c8c452d0b44383b7f7712f066
 checkdmarc==5.3.1
 dkimpy==1.1.5
 python-multipart==0.0.7