[MC-514] Parse dkim, spf and dmarc indicators from Authentication-Results header

README.md

@@ -41,4 +41,5 @@ be on your way to contributing!
 
 ## Changelog
 
+* 1.1.0 - Parse dkim, spf and dmarc indicators from Authentication-Results header
 * 1.0.0 - Initial release

eml_parser/email_parser.py

@@ -59,7 +59,7 @@ def format_result(self, msg: message, mailbox_id: str) -> IconEmail:
         result.recipients = self.get_recipients(msg)
         result.body = self.get_body(msg)
         result.headers = self.get_headers(msg)
-        result.indicators = Indicators(result.body)
+        result.indicators = Indicators(result.body, result.headers)
 
         (
             result.attached_files,

eml_parser/icon_email.py

@@ -20,10 +20,10 @@ def __init__(self, **kwargs):
         self.sender = kwargs.get("sender", None)
         self.subject = kwargs.get("subject", None)
         self.body = kwargs.get("body", None)
-        self.indicators = Indicators(self.body) if self.body else None
+        self.headers = kwargs.get("headers", None)
+        self.indicators = Indicators(self.body, self.headers) if self.body else None
         self.categories = kwargs.get("categories", None)
         self.date_received = kwargs.get("date_received", None)
-        self.headers = kwargs.get("headers", None)
         self.attached_files = kwargs.get("attached_files", [])
         self.attached_emails = kwargs.get("attached_emails", [])
         self.has_attachments = kwargs.get("has_attachments", False)

eml_parser/icon_file.py

@@ -5,11 +5,11 @@
 
 
 class IconFile(object):
-    def __init__(self, file_name: str = "", content_type: str = "", content: str = ""):
+    def __init__(self, file_name: str = "", content_type: str = "", content: str = "", headers=None):
         self.name = file_name
         self.content = content
         self.content_type = content_type
-        self.indicators = Indicators(content)
+        self.indicators = Indicators(content, headers)
 
     # May not need this
     def make_serializable(self) -> dict:

eml_parser/indicators.py

@@ -5,12 +5,17 @@
 
 
 class Indicators(object):
-    def __init__(self, content: str):
+    def __init__(self, content: str, headers: list):
         encoded_content = content.encode("UTF-8")
         self.md5 = hashlib.md5(encoded_content).hexdigest()
         self.sha1 = hashlib.sha1(encoded_content).hexdigest()
         self.sha256 = hashlib.sha256(encoded_content).hexdigest()
 
+        auth_types = self.parse_auth_results_header(headers)
+        self.dkim = auth_types.get("dkim", None)
+        self.dmarc = auth_types.get("dmarc", None)
+        self.spf = auth_types.get("spf", None)
+
     # May not need this
     def make_serializable(self) -> dict:
         """Converts the File to a JSON-serializable, cleaned dict"""
@@ -40,3 +45,27 @@ def __hash__(self):
                 self.sha256,
             )
         )
+
+    @staticmethod
+    def parse_auth_results_header(headers: list) -> dict:
+        value = ""
+        auth_types = {}
+
+        if headers:
+            for header in headers:
+                name = header.get("name")
+                if name == "Authentication-Results":
+                    value = header.get("value")
+
+        if value:
+            split_header = value.split(";")
+            for auth_type in split_header:
+                stripped_auth_type = auth_type.strip().replace("\n", "")
+                if stripped_auth_type.startswith("dkim"):
+                    auth_types["dkim"] = stripped_auth_type
+                elif stripped_auth_type.startswith("dmarc"):
+                    auth_types["dmarc"] = stripped_auth_type
+                elif stripped_auth_type.startswith("spf"):
+                    auth_types["spf"] = stripped_auth_type
+
+        return auth_types

setup.py

@@ -7,7 +7,7 @@
 
 setup(
     name="python_eml_parser",
-    version="1.0.0",
+    version="1.1.0",
     description="Rapid7 InsightConnect email parser for email plugins",
     long_description=long_description,
     long_description_content_type="text/markdown",

unit_test/payloads/email_with_auth_results_header_without_dkim.txt

@@ -0,0 +1,146 @@
+Received: from example.com-example.com.example.com.com
+ (2603:10b6:408:c0::36) by example.com20.example.com.com with HTTPS
+ via example.com15.example.com.COM; Thu, 8 Aug 2019 21:19:42 +0000
+ARC-Seal: i=2; a=rsa-sha256; s=arcselector9901; d=example.com; cv=pass;
+ b=LZ9EvdIoPXYfqOAQchCntyc8OMq5Hq/t/voPAi0AmV8DqEott3fpkC8cO/3qLk9/8Ky9g7pGejcFmj9Ovf3/WfPZ0rFXbr2x6Pc5sIhgCZeLXIyXwyLdXnC/7w+b6huhiO+Le8cuWdkXEflfSR8N3gJoxdjudALaNEMW+rr3Ggwr60u9kRvDppJC9uvpbR10CJeXbc6O455neO88Qokxpamm7S7cu3jgKpgml+eLSkTlfMk9PhzY5Qk6yuDFZQYy4o/SBhtHzpHDvMhINdlOggNZkCvlgkPTwQgM0pGWOhPwZrBY+t9eq3v0bZL2/3Xb+r8GAaZstDEl6J+LZXGYtw==
+ARC-Message-Signature: i=2; a=rsa-sha256; c=relaxed/relaxed; d=example.com;
+ s=arcselector9901;
+ h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck;
+ bh=DIoO+sSplAukDl1wtTy/yLQIokUGR9rVByJNSjj5VjI=;
+ b=mF3gPrTCRDNs7Nab0MreYy5HKAs78TUoFMb6jX3C2NyIrXlE0I4orKLKT48HLebf765PjCWsx9nPMnjYlClv3Te3rznISejSqlyquVRCNfjMyU39akaMbWwNc8+NGX5Fooq+0Mbhh1a4939Q4E1i/yJ+2kIOIkLuQ4Z52zU8z/lCtVL0J/vh28IFiBuxVrKMNdPie7uOFApObXeW1frwgvyryY8lF6xnKRXhg8ABA5cIsm25zysfL1NahKBTeweshAn8B041T3Els6XJ8telHVRl9cwQvi+maGpBXa1kCnr9QwuguhAo8qYhQWhCX50kE+AEDL4V3wRxcEqpQf56uQ==
+ARC-Authentication-Results: i=2; mx.example.com 1; spf=pass (sender ip is
+ 198.162.1.1) example.com=example.com example.com=example.com;
+ dmarc=pass (p=none sp=none pct=100) action=none example.com=example.com;
+ dkim=pass (signature was verified) header.d=example.com; arc=pass (0 oda=0
+ ltdi=1)
+Received: from example.com-example.com.example.com.com
+ (198.162.1.1) by example.com-example.com.example.com.com
+ (198.162.1.1) with Microsoft SMTP Server (version=TLS1_2,
+ cipher=TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384) id 15.20.2157.15; Thu, 8 Aug
+ 2019 21:19:41 +0000
+Authentication-Results: spf=pass (sender IP is 198.162.1.1)
+ example.com=example.com; example.com; dmarc=pass action=none
+ example.com=example.com;
+Received-SPF: Pass (example.com.com: domain of example.com designates
+ 198.162.1.1 as permitted sender) receiver=example.com.com;
+ client-ip=198.162.1.1; helo=example.com.example.com.com;
+Received: from example.com.example.com.com (198.162.1.1) by
+ example.com.example.com.com (198.162.1.1) with Microsoft SMTP
+ Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384) id
+ 15.20.2157.15 via Frontend Transport; Thu, 8 Aug 2019 21:19:41 +0000
+X-IncomingTopHeaderMarker: OriginalChecksum:21336D8EF88D0BEBC867482C979C2130136B7EDA623D639C13E6A52955BFB745;UpperCasedChecksum:AD96A565A8F1E993C569A7CBDE50B562303AE3A8B32FB451D76A10FEC7200AC0;SizeAsReceived:5205;Count:38
+ARC-Seal: i=1; a=rsa-sha256; s=arcselector9901; d=example.com; cv=none;
+ b=IiZ0KlwH0+GBNLYms9zcFnLyKHd0rqsyzDb3481fyliZXRk6NN9FyjG94BpVzCAHn5f0gKZqrIYUy+a1EtunVPZb/eaIQUzrV9G7GjiOhlTJPeWTuABXkUPl4j7GrbCuPFbtF3b9GTCBx2y+FLoz4ty9/OMWpcZbzW1fbYvvBp1+PPh3vj/19XJa/7AcHpJ3x4ywqdNbWHOYo+0GpOIR+8F8Q1HvJNcndiUlne0BFWzYBab3QaHAOCShw/ZZrb+/igfSrzEniuy89wkFePdZXo50AgTU2zGCAylJdyhGQALYanbYAhnSjMe82whUInciT76AhMsPwHHF1CCcV5zOgA==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=example.com;
+ s=arcselector9901;
+ h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck;
+ bh=DIoO+sSplAukDl1wtTy/yLQIokUGR9rVByJNSjj5VjI=;
+ b=ipaT0Vc4kmpcD2Ls1xXW/7yfw45Ff0JmAssS0PQ1hBQwZzxJkwDiPt/UsFKxD5wekT//bJLYhALcjVYUImM7BqLhBj4HFZJllJ1kwYIk+qyUFchEIMHmgFym6H22TR6IyP25uZzUW4Tpqh5Cqbcm0rF04DeiOjFAx5Bt9owPOzvsazICmD7hDmzYea3rHe/HgzWG2YPhskpb+NOFo8lttnacXMVMZaa75urxxbc1IyLQWmv34ikStIlUL2DCPF0df2gBUlf3O0U7EqGDcrIk5+GrlJ6wFZ27GvH+2izzJb0t6NLTW+mPYi3rzFVAT1wJ6nrxeV7bOfHmETGPWSBwAg==
+ARC-Authentication-Results: i=1; mx.example.com 1; spf=none; dmarc=none;
+ dkim=none; arc=none
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=example.com;
+ s=selector1;
+ h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck;
+ bh=DIoO+sSplAukDl1wtTy/yLQIokUGR9rVByJNSjj5VjI=;
+ b=A7l6SdD2GSPHDCi1rKLgOh4GCzWgl/8TvirxPp+Y6M5b/mImlWEWd8nYwC3k7TgqMgumXYxqNUgiCku0S9h5RsZPVcJI6L4JlPUBylNPjSQVUhOA5DhhZ3+U7ZYeIoVxgOq9D/599uN2UMwE0m/8YzVx4tTZOrzGpUCWyH3mKrGPuBukSN8w93BP6QOi3LtL0MIw+DJJsu80us+pQiMnwyppIeDsdAmtqPVTpTV17nphkH2CpGj0yThzah+b/QrZnTpDaGN06U/kJzULMG/QlHvGUPrAom1TbDFsnIprFZP0UaLD965GJiwgCU15x4PxvvxHHbJWg6ExDjEhZPE0SQ==
+Received: from example.com-example.com.example.com.com
+ (198.162.1.1) by example.com-example.com.example.com.com
+ (198.162.1.1) with Microsoft SMTP Server (version=TLS1_2,
+ cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.2157.15; Thu, 8 Aug
+ 2019 21:19:38 +0000
+Received: from example.com20.example.com.com (198.162.1.1) by
+ example.com.example.com.com (198.162.1.1) with Microsoft SMTP
+ Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id
+ 15.20.2157.15 via Frontend Transport; Thu, 8 Aug 2019 21:19:38 +0000
+Received: from example.com20.example.com.com
+ ([fe80::4de1:ae2d:34c4:6147]) by example.com20.example.com.com
+ ([fe80::4de1:ae2d:34c4:6147%9]) with mapi id 15.20.2157.015; Thu, 8 Aug 2019
+ 21:19:38 +0000
+From: Example User <[email protected]>
+To: Example User <[email protected]>
+Subject: Basic Message Attachment
+Thread-Topic: Basic Message Attachment
+Thread-Index: AQHVTi77mfitcP9XvEqJVwjq0oydwg==
+Date: Thu, 8 Aug 2019 21:19:37 +0000
+Message-ID: <[email protected]>
+Accept-Language: en-US
+Content-Language: en-US
+X-MS-Has-Attach:
+X-MS-TNEF-Correlator:
+x-incomingtopheadermarker: OriginalChecksum:4B15105A1B0B93B456CAA4DC6FDA3A0B0CB644531DFEE2B0A3ECF0E055960EB8;UpperCasedChecksum:52E964A0D871DD0F13F3405C3605DEDCB1104AD19AA63653B4DE1888504A4951;SizeAsReceived:6492;Count:40
+x-tmn: [CWyKlqUIrDSIzJE4u0R6L5NXtS66R3jX]
+x-ms-publictraffictype: Email
+x-incomingheadercount: 40
+x-eopattributedmessage: 1
+x-ms-exchange-slblob-mailprops: fMOR/XCTRh0cmIUeXDoOIhgkHkQ3L6OaChqrFmHBpomEIfUI9SMY17Ov8FlN+qPS7r0sKbPb1SJlCvk6yyy6YzjHqO/JvpGAfdlzltSTuFrCFG9LwByqO7c/jSzqqKr8eiOcYprSwqD1yF7lDQgTdbKUlWTPWNk0vmjw77fTX5a5cD9PAG9BKUd5p5bohc1BkX39CcC+5QE2Lu+xi30JgLt2o/cq0ni9lLR5Ll4KGhEsmbB5q9lOZ/mhat5ebgDCUcfy6jHbmeqSYetCtUtOTfMPukg11FmRwv3Urps9B1u63OdBm4rng1D3ksTIdCYvDp0fwwh4s4iW5zsC1ROQh67/igiOcK5qMzyWEqmKbd3AHAdevUyj6WQRf1kh8BP6A2tBegNKog0eqC8iXdq1edovcQhpqPbN1BYG/bjMuEqPPJQJmXgUmjzbC1Zl8qhuVc+meyBZyOzFmU38Bw2N+Ku9k4300mFxUV3PvrIv4EeebQ0OUe5B/BdRkFdNWLJWz5xA774BFX0I02z4h+Yi0JeB/1TnfqWxsOGFMn2J5sKeQOiJatV4xESj4SfTDTEk8QEXW9TGfI3xo2uqbE88lQ==
+X-Microsoft-Antispam-Untrusted: BCL:0;PCL:0;RULEID:(2390118)(5050001)(7020095)(20181119110)(201702061078)(5061506573)(5061507331)(1603103135)(2017031320274)(201702181274)(2017031322404)(2017031323274)(2017031324274)(1601125500)(1603101475)(1701031045);SRVR:CO1NAM04HT238;
+X-MS-TrafficTypeDiagnostic: CO1NAM04HT238:|AM5EUR02HT029:
+X-Microsoft-Antispam-Message-Info-Original: MA4ctjdBg0zYzWFpY8ycDPYQ5diKN70j3Mad/x4nmGIDnZ69KWyBsNNAtIW7fnEaLqX6iU7qKlSWHgxXWy3M5bU7z/JGdpjFmKeKRoqvPoQxKL8B4gz0GZc3EDI/BW6XO1d39/MuYZExCQEAJv5g88OTqtOEutaFoNfkZ5VO/szi94/FbxGGAo1RvJtr8cig
+x-ms-exchange-transport-forked: True
+Content-Type: multipart/alternative;
+	boundary="_000_BN6PR20MB138076A7EF7F8CB1C16ADC9FF3D70BN6PR20MB1380namp_"
+X-MS-Exchange-Transport-CrossTenantHeadersStamped: CO1NAM04HT238
+X-IncomingHeaderCount: 38
+Return-Path: [email protected]
+X-MS-Exchange-Organization-ExpirationStartTime: 08 Aug 2019 21:19:41.2100
+ (UTC)
+X-MS-Exchange-Organization-ExpirationStartTimeReason: OriginalSubmit
+X-MS-Exchange-Organization-ExpirationInterval: 1:00:00:00.0000000
+X-MS-Exchange-Organization-ExpirationIntervalReason: OriginalSubmit
+X-MS-Exchange-Organization-Network-Message-Id: 82a86468-365b-428d-614b-08d71c462057
+X-EOPTenantAttributedMessage: 84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa:0
+X-MS-Exchange-Organization-MessageDirectionality: Incoming
+X-MS-Exchange-Transport-CrossTenantHeadersStripped: example.com-example.com.example.com.com
+X-MS-Exchange-Transport-CrossTenantHeadersPromoted: example.com-example.com.example.com.com
+X-Forefront-Antispam-Report: EFV:NLI;
+X-MS-Exchange-Organization-AuthSource: example.com-example.com.example.com.com
+X-MS-Exchange-Organization-AuthAs: Anonymous
+X-MS-UserLastLogonTime: 8/8/2019 9:19:12 PM
+X-MS-Office365-Filtering-Correlation-Id: 82a86468-365b-428d-614b-08d71c462057
+X-Microsoft-Antispam: BCL:0;PCL:0;RULEID:(2390118)(5000188)(711020)(4605104)(610169)(650170)(651021)(1124261)(8291501072);SRVR:AM5EUR02HT029;
+X-MS-Exchange-EOPDirect: true
+X-Sender-IP: 198.162.1.1
+X-SID-PRA: [email protected]
+X-SID-Result: PASS
+X-MS-Exchange-Organization-PCL: 2
+X-OriginatorOrg: example.com
+X-MS-Exchange-CrossTenant-OriginalArrivalTime: 08 Aug 2019 21:19:41.0468
+ (UTC)
+X-MS-Exchange-CrossTenant-Network-Message-Id: 82a86468-365b-428d-614b-08d71c462057
+X-MS-Exchange-CrossTenant-Id: 84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa
+X-MS-Exchange-CrossTenant-RMS-PersistedConsumerOrg: 00000000-0000-0000-0000-000000000000
+X-MS-Exchange-CrossTenant-rms-persistedconsumerorg: 00000000-0000-0000-0000-000000000000
+X-MS-Exchange-CrossTenant-FromEntityHeader: Internet
+X-MS-Exchange-Transport-CrossTenantHeadersStamped: AM5EUR02HT029
+X-MS-Exchange-Transport-EndToEndLatency: 00:00:01.0997185
+X-MS-Exchange-Processed-By-BccFoldering: 15.20.2157.000
+X-Microsoft-Antispam-Mailbox-Delivery: dkl:0;rwl:0;ucf:0;jmr:0;ex:0;auth:1;dest:I;OFR:SpamFilterPass;ENG:(5062000261)(5061607266)(5061608174)(1004385)(4900115)(4920090)(6220004)(4950130)(4990090)(9110004);
+X-Message-Info: 5vMbyqxGkdewuie/JpX7LhBhuoKqhm44L9FcAPY8h6MB18Wp1miPSPFT6U8u0rnjY7ujAFcOK/dQlhibF8seWAv+fR1ZOpqmbLj4cylGEWc25VmPCZaeCYOskA4mqm/oN8yH3b93zYkOMGkkOAdf8jsK16H7/x9H495G69YUXYlS+saYUEZmQMa4u9pLC9elHJ9T2JEzWwP3ReDEdJziCg==
+X-Message-Delivery: Vj0xLjE7dXM9MDtsPTA7YT0xO0Q9MTtHRD0xO1NDTD0tMQ==
+X-Microsoft-Antispam-Message-Info: Gc+cBUh7JvlgmM5gLd0EyrMvDXcPIeWAtytGnrSxu8K+ypqw/ScBFP0LZOGi0wuLlJ0tResqvnYT5zSRSj9Kkl6Vt8ZxduAeZmtI9v9GwILd9Q7aRWbTjLDsFha1m+M/GlJwjuY4q2wFDy9pDQUXewVB7pU4HZUgoZw2dSuWxOcg5nsbzB749p12bruGXTREj910/otO1k+yKOBP29Fsov7AnX+HLyC+bUaQlBOqInLoJsLdaQMfRjKZ+ibesxenhLC7Ojjs3+kPX8Rel83cOFb7ccygDs0+gpPMGDew/arqGBnl4g94tcq2PRVRzRG1dvhRH0WgEZeWH+K6UWHHiGs0EQ0mmKmLEJnWuzCYGAOzVedh3s88+oidCdfKeCE+aLMGdtStmrJJFs5PgCMWLiJCbMhdhN/PZLrCBA52Cyx6d0Yy5lDZ1/eAej75LgBSDfHKpjjwugeNzroVI4NbWNpvYP5byn/Qmx0USQm8bN+WahWFj6ABzSln2J+ouCigdVKSzTvCKR7YYQFl3I+ndlOzr7cHs3u5GUlsODesFwUOluZiGKzMHRbZKy9BjB++AnkrJ4I/EOpREr+FRm53pOh/iARIZ6Q3fTSS9OEtwWacTzxoId0L2xpUJ8u0dLPHBb7h9kh9jFj0nvjgLvYctx9Y63W9XJlrP6lm3YZGDE0n9lUOfm+d+pj2+SyGzd7OdUEjZjRiexKxzzN5SxMAuBspfGL0duZ1sBqij4kquZ+hOszlNE00Igk3glUALIzmqZbq7OgmY1hjwgzJdo3ZtTlaEySlbNRA5n3OV3yKE89+SxXf+JQE/CZGnSIYV4SAhhzEOfNAr0yZVUaEHOhyWQ1Aa8NwUb2JUnhQvJ/h97fDY3xI0d2t5GVDO2P5yDVM+wldxGYENMtJwTOLu4HmMw==
+MIME-Version: 1.0
+
+--_000_BN6PR20MB138076A7EF7F8CB1C16ADC9FF3D70BN6PR20MB1380namp_
+Content-Type: text/plain; charset="iso-8859-1"
+Content-Transfer-Encoding: quoted-printable
+
+Nothing here, just this text
+
+--_000_BN6PR20MB138076A7EF7F8CB1C16ADC9FF3D70BN6PR20MB1380namp_
+Content-Type: text/html; charset="iso-8859-1"
+Content-Transfer-Encoding: quoted-printable
+
+<html><head>
+<meta http-equiv=3D"Content-Type" content=3D"text/html; charset=3Diso-8859-=
+1">
+<style type=3D"text/css" style=3D"display:none;"> P {margin-top:0;margin-bo=
+ttom:0;} </style>
+</head>
+<body dir=3D"ltr">
+<div style=3D"font-family: Calibri, Helvetica, sans-serif; font-size: 12pt;=
+ color: rgb(0, 0, 0);">
+Nothing here, just this text</div>
+</body>
+</html>
+
+--_000_BN6PR20MB138076A7EF7F8CB1C16ADC9FF3D70BN6PR20MB1380namp_--