Do not require validation between observable

and message fields - these should be allowed
to be independent.  Still require that all
fields called out in the message or in the
observable are present in the search. This
commit adds in the static checks.

contentctl/objects/abstract_security_content_objects/detection_abstract.py

@@ -3,7 +3,7 @@
 import requests
 import time
 import sys
-
+import re
 from pydantic import BaseModel, validator, root_validator, Extra
 from dataclasses import dataclass
 from typing import Union
@@ -92,16 +92,33 @@ def encode_error(cls, v, values, field):
 
     @validator("search")
     def search_obsersables_exist_validate(cls, v, values):
+        # All observable fields must appear in the search
         tags:DetectionTags = values.get("tags")
         if tags == None:
             raise ValueError("Unable to parse Detection Tags.  Please resolve Detection Tags errors")
 
-        observable_names = [ob.name for ob in tags.observable]
+        observable_fields = [ob.name.lower() for ob in tags.observable]
 
+        #All $field$ fields from the message must appear in the search
+        field_match_regex = r"\$([^\s.]*)\$"
 
-        missing_fields = set([name for name in observable_names if name not in v ])
+        message_fields = [match.replace("$", "").lower() for match in re.findall(field_match_regex, tags.message.lower())]
+        missing_fields = set([field for field in observable_fields if field not in v.lower()])
+
+        error_messages = []
         if len(missing_fields) > 0:
-            raise ValueError(f"The following fields are declared as observables, but do not exist in the search: {missing_fields}")
+            error_messages.append(f"The following fields are declared as observables, but do not exist in the search: {missing_fields}")
+
+
+        missing_fields = set([field for field in message_fields if field not in v.lower()])
+        if len(missing_fields) > 0:
+            error_messages.append(f"The following fields are used as fields in the message, but do not exist in the search: {missing_fields}")
+
+        if len(error_messages) > 0:
+            msg = "\n\t".join(error_messages)
+            raise(ValueError(msg))
+
+        # Found everything
         return v
 
     @validator("tests")

contentctl/objects/detection_tags.py

@@ -130,30 +130,31 @@ def tags_calculate_risk_score(cls, v, values):
                              f"\n  Expected risk_score={calculated_risk_score}, found risk_score={int(v)}: {values['name']}")
         return v
 
-    @validator('message')
-    def validate_message(cls,v,values):
+    # The following validator is temporarily disabled pending further discussions
+    # @validator('message')
+    # def validate_message(cls,v,values):
 
-        observables:list[Observable] = values.get("observable",[])
-        observable_names = set([o.name for o in observables])
-        #find all of the observables used in the message by name
-        name_match_regex = r"\$([^\s.]*)\$"
+    #     observables:list[Observable] = values.get("observable",[])
+    #     observable_names = set([o.name for o in observables])
+    #     #find all of the observables used in the message by name
+    #     name_match_regex = r"\$([^\s.]*)\$"
 
-        message_observables = set()
+    #     message_observables = set()
 
-        #Make sure that all observable names in 
-        for match in re.findall(name_match_regex, v):
-            #Remove
-            match_without_dollars = match.replace("$", "")
-            message_observables.add(match_without_dollars)
+    #     #Make sure that all observable names in 
+    #     for match in re.findall(name_match_regex, v):
+    #         #Remove
+    #         match_without_dollars = match.replace("$", "")
+    #         message_observables.add(match_without_dollars)
 
 
-        missing_observables = message_observables - observable_names
-        unused_observables = observable_names - message_observables
-        if len(missing_observables) > 0:
-            raise ValueError(f"The following observables are referenced in the message, but were not declared as observables: {missing_observables}")
+    #     missing_observables = message_observables - observable_names
+    #     unused_observables = observable_names - message_observables
+    #     if len(missing_observables) > 0:
+    #         raise ValueError(f"The following observables are referenced in the message, but were not declared as observables: {missing_observables}")
 
-        if len(unused_observables) > 0:
-            raise ValueError(f"The following observables were declared, but are not referenced in the message: {unused_observables}")        
-        return v
+    #     if len(unused_observables) > 0:
+    #         raise ValueError(f"The following observables were declared, but are not referenced in the message: {unused_observables}")        
+    #     return v