From 864d84f49cddade9481c70541ddc35a01c5a1b21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jan 2024 10:20:07 +0100 Subject: [PATCH 1/3] Bump uvicorn from 0.25.0 to 0.26.0 in /app (#36) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [uvicorn](https://github.com/encode/uvicorn) from 0.25.0 to 0.26.0.
Release notes

Sourced from uvicorn's releases.

Version 0.26.0

Changed

Full Changelog: https://github.com/encode/uvicorn/compare/0.25.0...0.26.0

Changelog

Sourced from uvicorn's changelog.

0.26.0 - 2024-01-16

Changed

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=uvicorn&package-manager=pip&previous-version=0.25.0&new-version=0.26.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- app/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/requirements.txt b/app/requirements.txt index 7cbc5a0..9b5f07d 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -10,6 +10,6 @@ python-decouple==3.8 python-multipart==0.0.6 redis==5.0.1 SQLAlchemy==2.0.25 -uvicorn==0.25.0 +uvicorn==0.26.0 validators==0.22.0 -r translations/requirements.txt From 9f294eb2e186f1bc7190f159a2caff2988991e54 Mon Sep 17 00:00:00 2001 From: kazet Date: Wed, 24 Jan 2024 09:33:22 +0100 Subject: [PATCH 2/3] mailgoose scanning is a python package (#37) --- .github/dependabot.yml | 4 ++++ .github/workflows/liccheck.yml | 6 ++++-- .mypy.ini | 3 +++ app/docker/Dockerfile | 1 + app/requirements.txt | 5 +---- app/src/app.py | 6 +++--- app/src/app_utils.py | 6 +++--- app/src/check_results.py | 2 +- common/config.py | 3 +-- mail_receiver/Dockerfile | 1 + mail_receiver/requirements.txt | 1 + scan/libmailgoose/__init__.py | 0 {common => scan/libmailgoose}/language.py | 0 .../libmailgoose}/lax_record_query.py | 0 scan/libmailgoose/logging.py | 10 ++++++++++ {app/src => scan/libmailgoose}/scan.py | 0 {app/src => scan/libmailgoose}/translate.py | 3 +-- scan/requirements.txt | 4 ++++ scan/setup.py | 20 +++++++++++++++++++ 19 files changed, 58 insertions(+), 17 deletions(-) create mode 100644 scan/libmailgoose/__init__.py rename {common => scan/libmailgoose}/language.py (100%) rename {app/src => scan/libmailgoose}/lax_record_query.py (100%) create mode 100644 scan/libmailgoose/logging.py rename {app/src => scan/libmailgoose}/scan.py (100%) rename {app/src => scan/libmailgoose}/translate.py (99%) create mode 100644 scan/requirements.txt create mode 100644 scan/setup.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d15fc8e..852073c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -16,6 +16,10 @@ updates: directory: "/mail_receiver/" schedule: interval: "weekly" + - package-ecosystem: "pip" + directory: "/scan/" + schedule: + interval: "weekly" - package-ecosystem: "pip" directory: "/test/" schedule: diff --git a/.github/workflows/liccheck.yml b/.github/workflows/liccheck.yml index 3f9d0da..1dca802 100644 --- a/.github/workflows/liccheck.yml +++ b/.github/workflows/liccheck.yml @@ -14,12 +14,14 @@ jobs: uses: actions/setup-python@v2 with: python-version: "3.11" + - name: Remove checkdmarc installed from CERT PL fork from requirements as it's not supported by liccheck, remove /scan as it's a local package + run: cp app/requirements.txt app/requirements.txt.orig; cat app/requirements.txt.orig | grep -v ^git+.*checkdmarc | grep -v ^/scan > app/requirements.txt + - name: Remove /scan as it's a local package + run: cp mail_receiver/requirements.txt mail_receiver/requirements.txt.orig; cat mail_receiver/requirements.txt.orig | grep -v ^/scan > mail_receiver/requirements.txt - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r app/requirements.txt -r mail_receiver/requirements.txt -r test/requirements.txt liccheck==0.9.2 - - name: Remove checkdmarc installed from CERT PL fork from requirements as it's not supported by liccheck - run: cp app/requirements.txt app/requirements.txt.orig; cat app/requirements.txt.orig | grep -v ^git+.*checkdmarc > app/requirements.txt - name: Run liccheck on app/requirements.txt run: liccheck -r app/requirements.txt - name: Run liccheck on mail_receiver/requirements.txt diff --git a/.mypy.ini b/.mypy.ini index a312ee9..d28779a 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -7,6 +7,9 @@ ignore_missing_imports = True [mypy-dkim.*] ignore_missing_imports = True +[mypy-libmailgoose.*] +ignore_missing_imports = True + [mypy-sphinx_rtd_theme.*] ignore_missing_imports = True diff --git a/app/docker/Dockerfile b/app/docker/Dockerfile index 4524639..0808d95 100644 --- a/app/docker/Dockerfile +++ b/app/docker/Dockerfile @@ -6,6 +6,7 @@ ENV TZ=Europe/Warsaw RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone WORKDIR /app/ +COPY scan /scan COPY app/requirements.txt /requirements.txt COPY app/translations/requirements.txt /translations/requirements.txt diff --git a/app/requirements.txt b/app/requirements.txt index 9b5f07d..d2d2a65 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -1,15 +1,12 @@ -checkdmarc==5.3.1 +/scan/ dacite==1.8.1 -dkimpy==1.1.5 email-validator==2.1.0.post1 fastapi==0.109.0 Jinja2==3.1.3 jinja2-simple-tags==0.5.0 psycopg2-binary==2.9.9 python-decouple==3.8 -python-multipart==0.0.6 redis==5.0.1 SQLAlchemy==2.0.25 uvicorn==0.26.0 -validators==0.22.0 -r translations/requirements.txt diff --git a/app/src/app.py b/app/src/app.py index 92a6663..7e0bdd6 100644 --- a/app/src/app.py +++ b/app/src/app.py @@ -10,11 +10,13 @@ from fastapi import FastAPI, Form, HTTPException, Request from fastapi.responses import HTMLResponse, RedirectResponse from fastapi.staticfiles import StaticFiles +from libmailgoose.language import Language +from libmailgoose.scan import DomainValidationException, ScanningException, ScanResult +from libmailgoose.translate import translate from redis import Redis from starlette.responses import Response from common.config import Config -from common.language import Language from common.mail_receiver_utils import get_key_from_username from .app_utils import ( @@ -26,9 +28,7 @@ from .db import ScanLogEntrySource, ServerErrorLogEntry, Session from .logging import build_logger from .resolver import setup_resolver -from .scan import DomainValidationException, ScanningException, ScanResult from .templates import setup_templates -from .translate import translate app = FastAPI() LOGGER = build_logger(__name__) diff --git a/app/src/app_utils.py b/app/src/app_utils.py index b15eae8..9325ed7 100644 --- a/app/src/app_utils.py +++ b/app/src/app_utils.py @@ -10,9 +10,11 @@ import dkim.util from email_validator import EmailNotValidError, validate_email from fastapi import Request +from libmailgoose.language import Language +from libmailgoose.scan import ScanResult, scan +from libmailgoose.translate import translate_scan_result from common.config import Config -from common.language import Language from .db import ( DKIMImplementationMismatchLogEntry, @@ -22,8 +24,6 @@ Session, ) from .logging import build_logger -from .scan import ScanResult, scan -from .translate import translate_scan_result LOGGER = build_logger(__name__) diff --git a/app/src/check_results.py b/app/src/check_results.py index 3cfceea..73747e2 100644 --- a/app/src/check_results.py +++ b/app/src/check_results.py @@ -6,12 +6,12 @@ from typing import Any, Dict, Optional import dacite +from libmailgoose.scan import ScanResult from redis import Redis from common.config import Config from .logging import build_logger -from .scan import ScanResult REDIS = Redis.from_url(Config.Data.REDIS_URL) diff --git a/common/config.py b/common/config.py index 16e4144..249bbb6 100644 --- a/common/config.py +++ b/common/config.py @@ -1,8 +1,7 @@ from typing import Annotated, Any, List, get_type_hints import decouple - -from common.language import Language +from libmailgoose.language import Language DEFAULTS = {} diff --git a/mail_receiver/Dockerfile b/mail_receiver/Dockerfile index 74ef8b8..4e56d59 100644 --- a/mail_receiver/Dockerfile +++ b/mail_receiver/Dockerfile @@ -5,6 +5,7 @@ RUN apk add tzdata ENV TZ=Europe/Warsaw RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +COPY scan /scan COPY mail_receiver/requirements.txt /requirements.txt RUN pip install -r /requirements.txt diff --git a/mail_receiver/requirements.txt b/mail_receiver/requirements.txt index 132f0d6..bb40422 100644 --- a/mail_receiver/requirements.txt +++ b/mail_receiver/requirements.txt @@ -1,3 +1,4 @@ aiosmtpd==1.4.4.post2 python-decouple==3.8 redis==5.0.1 +/scan diff --git a/scan/libmailgoose/__init__.py b/scan/libmailgoose/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/common/language.py b/scan/libmailgoose/language.py similarity index 100% rename from common/language.py rename to scan/libmailgoose/language.py diff --git a/app/src/lax_record_query.py b/scan/libmailgoose/lax_record_query.py similarity index 100% rename from app/src/lax_record_query.py rename to scan/libmailgoose/lax_record_query.py diff --git a/scan/libmailgoose/logging.py b/scan/libmailgoose/logging.py new file mode 100644 index 0000000..a7c6394 --- /dev/null +++ b/scan/libmailgoose/logging.py @@ -0,0 +1,10 @@ +import logging + + +def build_logger(name: str) -> logging.Logger: + logger = logging.getLogger(name) + logger.setLevel(logging.INFO) + handler = logging.StreamHandler() + handler.setLevel(logging.INFO) + logger.addHandler(handler) + return logger diff --git a/app/src/scan.py b/scan/libmailgoose/scan.py similarity index 100% rename from app/src/scan.py rename to scan/libmailgoose/scan.py diff --git a/app/src/translate.py b/scan/libmailgoose/translate.py similarity index 99% rename from app/src/translate.py rename to scan/libmailgoose/translate.py index 2286c25..3d88704 100644 --- a/app/src/translate.py +++ b/scan/libmailgoose/translate.py @@ -2,8 +2,7 @@ import re from typing import Callable, List, Optional, Tuple -from common.language import Language - +from .language import Language from .scan import DKIMScanResult, DomainScanResult, ScanResult PLACEHOLDER = "__PLACEHOLDER__" diff --git a/scan/requirements.txt b/scan/requirements.txt new file mode 100644 index 0000000..0cb739c --- /dev/null +++ b/scan/requirements.txt @@ -0,0 +1,4 @@ +checkdmarc==5.3.1 +dkimpy==1.1.5 +python-multipart==0.0.6 +validators==0.22.0 diff --git a/scan/setup.py b/scan/setup.py new file mode 100644 index 0000000..1019dd1 --- /dev/null +++ b/scan/setup.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +import os +from distutils.core import setup + +with open(os.path.join(os.path.dirname(__file__), "requirements.txt")) as f: + requires = f.read().splitlines() + + +setup( + name="libmailgoose", + version="1.0", + description="libmailgoose - check the settings needed to protect against e-mail spoofing", + author="CERT Polska", + author_email="info@cert.pl", + url="https://github.com/CERT-Polska/mailgoose", + packages=["libmailgoose"], + scripts=[], + install_requires=requires, +) From b28216fd44a85b83d3c212c9cdb89394fc0fdd58 Mon Sep 17 00:00:00 2001 From: kazet Date: Wed, 24 Jan 2024 09:54:40 +0100 Subject: [PATCH 3/3] Package metadata (#38) --- scan/setup.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scan/setup.py b/scan/setup.py index 1019dd1..7ca832c 100644 --- a/scan/setup.py +++ b/scan/setup.py @@ -13,8 +13,13 @@ description="libmailgoose - check the settings needed to protect against e-mail spoofing", author="CERT Polska", author_email="info@cert.pl", + license="BSD", url="https://github.com/CERT-Polska/mailgoose", packages=["libmailgoose"], scripts=[], + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: BSD License", + ], install_requires=requires, )