mailgoose scanning is a python package (#37)

.github/dependabot.yml

@@ -16,6 +16,10 @@ updates:
     directory: "/mail_receiver/"
     schedule:
       interval: "weekly"
+  - package-ecosystem: "pip"
+    directory: "/scan/"
+    schedule:
+      interval: "weekly"
   - package-ecosystem: "pip"
     directory: "/test/"
     schedule:

.github/workflows/liccheck.yml

@@ -14,12 +14,14 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: "3.11"
+      - name: Remove checkdmarc installed from CERT PL fork from requirements as it's not supported by liccheck, remove /scan as it's a local package
+        run: cp app/requirements.txt app/requirements.txt.orig; cat app/requirements.txt.orig | grep -v ^git+.*checkdmarc | grep -v ^/scan > app/requirements.txt
+      - name: Remove /scan as it's a local package
+        run: cp mail_receiver/requirements.txt mail_receiver/requirements.txt.orig; cat mail_receiver/requirements.txt.orig | grep -v ^/scan > mail_receiver/requirements.txt
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install -r app/requirements.txt -r mail_receiver/requirements.txt -r test/requirements.txt liccheck==0.9.2
-      - name: Remove checkdmarc installed from CERT PL fork from requirements as it's not supported by liccheck
-        run: cp app/requirements.txt app/requirements.txt.orig; cat app/requirements.txt.orig | grep -v ^git+.*checkdmarc > app/requirements.txt
       - name: Run liccheck on app/requirements.txt
         run: liccheck -r app/requirements.txt
       - name: Run liccheck on mail_receiver/requirements.txt

.mypy.ini

@@ -7,6 +7,9 @@ ignore_missing_imports = True
 [mypy-dkim.*]
 ignore_missing_imports = True
 
+[mypy-libmailgoose.*]
+ignore_missing_imports = True
+
 [mypy-sphinx_rtd_theme.*]
 ignore_missing_imports = True
 

app/docker/Dockerfile

@@ -6,6 +6,7 @@ ENV TZ=Europe/Warsaw
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
 WORKDIR /app/
+COPY scan /scan
 COPY app/requirements.txt /requirements.txt
 COPY app/translations/requirements.txt /translations/requirements.txt
 

app/requirements.txt

@@ -1,15 +1,12 @@
-checkdmarc==5.3.1
+/scan/
 dacite==1.8.1
-dkimpy==1.1.5
 email-validator==2.1.0.post1
 fastapi==0.109.0
 Jinja2==3.1.3
 jinja2-simple-tags==0.5.0
 psycopg2-binary==2.9.9
 python-decouple==3.8
-python-multipart==0.0.6
 redis==5.0.1
 SQLAlchemy==2.0.25
 uvicorn==0.26.0
-validators==0.22.0
 -r translations/requirements.txt

app/src/app.py

@@ -10,11 +10,13 @@
 from fastapi import FastAPI, Form, HTTPException, Request
 from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
+from libmailgoose.language import Language
+from libmailgoose.scan import DomainValidationException, ScanningException, ScanResult
+from libmailgoose.translate import translate
 from redis import Redis
 from starlette.responses import Response
 
 from common.config import Config
-from common.language import Language
 from common.mail_receiver_utils import get_key_from_username
 
 from .app_utils import (
@@ -26,9 +28,7 @@
 from .db import ScanLogEntrySource, ServerErrorLogEntry, Session
 from .logging import build_logger
 from .resolver import setup_resolver
-from .scan import DomainValidationException, ScanningException, ScanResult
 from .templates import setup_templates
-from .translate import translate
 
 app = FastAPI()
 LOGGER = build_logger(__name__)

app/src/app_utils.py

@@ -10,9 +10,11 @@
 import dkim.util
 from email_validator import EmailNotValidError, validate_email
 from fastapi import Request
+from libmailgoose.language import Language
+from libmailgoose.scan import ScanResult, scan
+from libmailgoose.translate import translate_scan_result
 
 from common.config import Config
-from common.language import Language
 
 from .db import (
     DKIMImplementationMismatchLogEntry,
@@ -22,8 +24,6 @@
     Session,
 )
 from .logging import build_logger
-from .scan import ScanResult, scan
-from .translate import translate_scan_result
 
 LOGGER = build_logger(__name__)
 

app/src/check_results.py

@@ -6,12 +6,12 @@
 from typing import Any, Dict, Optional
 
 import dacite
+from libmailgoose.scan import ScanResult
 from redis import Redis
 
 from common.config import Config
 
 from .logging import build_logger
-from .scan import ScanResult
 
 REDIS = Redis.from_url(Config.Data.REDIS_URL)
 

common/config.py

@@ -1,8 +1,7 @@
 from typing import Annotated, Any, List, get_type_hints
 
 import decouple
-
-from common.language import Language
+from libmailgoose.language import Language
 
 DEFAULTS = {}
 

mail_receiver/Dockerfile

@@ -5,6 +5,7 @@ RUN apk add tzdata
 ENV TZ=Europe/Warsaw
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
+COPY scan /scan
 COPY mail_receiver/requirements.txt /requirements.txt
 RUN pip install -r /requirements.txt
 

mail_receiver/requirements.txt

@@ -1,3 +1,4 @@
 aiosmtpd==1.4.4.post2
 python-decouple==3.8
 redis==5.0.1
+/scan

scan/libmailgoose/logging.py

@@ -0,0 +1,10 @@
+import logging
+
+
+def build_logger(name: str) -> logging.Logger:
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.INFO)
+    handler = logging.StreamHandler()
+    handler.setLevel(logging.INFO)
+    logger.addHandler(handler)
+    return logger

app/src/translate.py → scan/libmailgoose/translate.py

@@ -2,8 +2,7 @@
 import re
 from typing import Callable, List, Optional, Tuple
 
-from common.language import Language
-
+from .language import Language
 from .scan import DKIMScanResult, DomainScanResult, ScanResult
 
 PLACEHOLDER = "__PLACEHOLDER__"

scan/requirements.txt

@@ -0,0 +1,4 @@
+checkdmarc==5.3.1
+dkimpy==1.1.5
+python-multipart==0.0.6
+validators==0.22.0

scan/setup.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+import os
+from distutils.core import setup
+
+with open(os.path.join(os.path.dirname(__file__), "requirements.txt")) as f:
+    requires = f.read().splitlines()
+
+
+setup(
+    name="libmailgoose",
+    version="1.0",
+    description="libmailgoose - check the settings needed to protect against e-mail spoofing",
+    author="CERT Polska",
+    author_email="[email protected]",
+    url="https://github.com/CERT-Polska/mailgoose",
+    packages=["libmailgoose"],
+    scripts=[],
+    install_requires=requires,
+)