From 3838937f81b9bf486c6c7cbcfac5114fe0956e18 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 1 Aug 2023 10:54:49 -0700
Subject: [PATCH] Do not require validation between observable and message
 fields - these should be allowed to be independent.  Still require that all
 fields called out in the message or in the observable are present in the
 search. This commit adds in the static checks.

---
 .../detection_abstract.py                     | 25 ++++++++++--
 contentctl/objects/detection_tags.py          | 39 ++++++++++---------
 2 files changed, 41 insertions(+), 23 deletions(-)

diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py
index 5e5931f5..f17cc4ad 100644
--- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py
+++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py
@@ -3,7 +3,7 @@
 import requests
 import time
 import sys
-
+import re
 from pydantic import BaseModel, validator, root_validator, Extra
 from dataclasses import dataclass
 from typing import Union
@@ -92,16 +92,33 @@ def encode_error(cls, v, values, field):
 
     @validator("search")
     def search_obsersables_exist_validate(cls, v, values):
+        # All observable fields must appear in the search
         tags:DetectionTags = values.get("tags")
         if tags == None:
             raise ValueError("Unable to parse Detection Tags.  Please resolve Detection Tags errors")
         
-        observable_names = [ob.name for ob in tags.observable]
+        observable_fields = [ob.name.lower() for ob in tags.observable]
         
+        #All $field$ fields from the message must appear in the search
+        field_match_regex = r"\$([^\s.]*)\$"
         
-        missing_fields = set([name for name in observable_names if name not in v ])
+        message_fields = [match.replace("$", "").lower() for match in re.findall(field_match_regex, tags.message.lower())]
+        missing_fields = set([field for field in observable_fields if field not in v.lower()])
+
+        error_messages = []
         if len(missing_fields) > 0:
-            raise ValueError(f"The following fields are declared as observables, but do not exist in the search: {missing_fields}")
+            error_messages.append(f"The following fields are declared as observables, but do not exist in the search: {missing_fields}")
+
+        
+        missing_fields = set([field for field in message_fields if field not in v.lower()])
+        if len(missing_fields) > 0:
+            error_messages.append(f"The following fields are used as fields in the message, but do not exist in the search: {missing_fields}")
+        
+        if len(error_messages) > 0:
+            msg = "\n\t".join(error_messages)
+            raise(ValueError(msg))
+        
+        # Found everything
         return v
 
     @validator("tests")
diff --git a/contentctl/objects/detection_tags.py b/contentctl/objects/detection_tags.py
index 959cd606..7ba5c87e 100644
--- a/contentctl/objects/detection_tags.py
+++ b/contentctl/objects/detection_tags.py
@@ -130,30 +130,31 @@ def tags_calculate_risk_score(cls, v, values):
                              f"\n  Expected risk_score={calculated_risk_score}, found risk_score={int(v)}: {values['name']}")
         return v
     
-    @validator('message')
-    def validate_message(cls,v,values):
+    # The following validator is temporarily disabled pending further discussions
+    # @validator('message')
+    # def validate_message(cls,v,values):
         
-        observables:list[Observable] = values.get("observable",[])
-        observable_names = set([o.name for o in observables])
-        #find all of the observables used in the message by name
-        name_match_regex = r"\$([^\s.]*)\$"
+    #     observables:list[Observable] = values.get("observable",[])
+    #     observable_names = set([o.name for o in observables])
+    #     #find all of the observables used in the message by name
+    #     name_match_regex = r"\$([^\s.]*)\$"
         
-        message_observables = set()
+    #     message_observables = set()
 
-        #Make sure that all observable names in 
-        for match in re.findall(name_match_regex, v):
-            #Remove
-            match_without_dollars = match.replace("$", "")
-            message_observables.add(match_without_dollars)
+    #     #Make sure that all observable names in 
+    #     for match in re.findall(name_match_regex, v):
+    #         #Remove
+    #         match_without_dollars = match.replace("$", "")
+    #         message_observables.add(match_without_dollars)
         
 
-        missing_observables = message_observables - observable_names
-        unused_observables = observable_names - message_observables
-        if len(missing_observables) > 0:
-            raise ValueError(f"The following observables are referenced in the message, but were not declared as observables: {missing_observables}")
+    #     missing_observables = message_observables - observable_names
+    #     unused_observables = observable_names - message_observables
+    #     if len(missing_observables) > 0:
+    #         raise ValueError(f"The following observables are referenced in the message, but were not declared as observables: {missing_observables}")
         
-        if len(unused_observables) > 0:
-            raise ValueError(f"The following observables were declared, but are not referenced in the message: {unused_observables}")        
-        return v
+    #     if len(unused_observables) > 0:
+    #         raise ValueError(f"The following observables were declared, but are not referenced in the message: {unused_observables}")        
+    #     return v
 
     
\ No newline at end of file