Merge branch 'main' into feature/validation_against_cms_main

.github/workflows/test_against_escu.yml

@@ -53,10 +53,11 @@ jobs:
           poetry install --no-interaction
 
 
-      - name: Clone the AtomicRedTeam Repo (for extended validation)
+      - name: Clone the AtomicRedTeam Repo and the Mitre/CTI repos for testing enrichments  
         run: |
           cd security_content
-          git clone --depth 1 https://github.com/redcanaryco/atomic-red-team
+          git clone --single-branch https://github.com/redcanaryco/atomic-red-team external_repos/atomic-red-team
+          git clone --single-branch https://github.com/mitre/cti external_repos/cti
 
       
       # We do not separately run validate and build 

README.md

@@ -134,7 +134,7 @@ This section is under active development.  It will allow you to a [MITRE Map](ht
 Choose TYPE {detection, story} to create new content for the Content Pack.  The tool will interactively ask a series of questions required for generating a basic piece of content and automatically add it to the Content Pack.
 
 ### contentctl inspect
-This section is under development.  It will enable the user to perform an appinspect of the content pack in preparation for deployment onto a Splunk Instance or via Splunk Cloud.
+This section is under development.  The inspect action performs a number of post-build validations. Primarily, it will enable the user to perform an appinspect of the content pack in preparation for deployment onto a Splunk Instance or via Splunk Cloud. It also compares detections in the new build against a prior build, confirming that any changed detections have had their versions incremented (this comparison happens at the savedsearch.conf level, which is why it must happen after the build). Please also note that new versions of contentctl may result in the generation of different savedsearches.conf files without any content changes in YML (new keys at the .conf level which will necessitate bumping of the version in the YML file).
 
 ### contentctl deploy
 The reason to build content is so that it can be deployed to your environment.  However, deploying content to multiple servers and different types of infrastructure can be tricky and time-consuming.  contentctl makes this easy by supporting a number of different deployment mechanisms. Deployment targets can be defined in [contentctl.yml](/contentctl/templates/contentctl_default.yml).

contentctl/actions/build.py

@@ -50,6 +50,7 @@ def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto:
             updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.investigations, SecurityContentType.investigations))
             updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.lookups, SecurityContentType.lookups))
             updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.macros, SecurityContentType.macros))
+            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.dashboards, SecurityContentType.dashboards))
             updated_conf_files.update(conf_output.writeAppConf())
 
             #Ensure that the conf file we just generated/update is syntactically valid

contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py

@@ -358,13 +358,17 @@ def configure_imported_roles(
         indexes.append(self.sync_obj.replay_index)
         indexes_encoded = ";".join(indexes)
 
-        # Include ES roles if installed
-        if self.es_installed:
-            imported_roles = imported_roles + enterprise_security_roles
+        # Set which roles should be configured. For Enterprise Security/Integration Testing,
+        # we must add some extra foles.
+        if self.global_config.enable_integration_testing:
+            roles = imported_roles + enterprise_security_roles
+        else:
+            roles = imported_roles
+
         try:
             self.get_conn().roles.post(
                 self.infrastructure.splunk_app_username,
-                imported_roles=imported_roles,
+                imported_roles=roles,
                 srchIndexesAllowed=indexes_encoded,
                 srchIndexesDefault=self.sync_obj.replay_index,
             )

contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py

@@ -49,13 +49,17 @@ def get_docker_client(self):
     def check_for_teardown(self):
 
         try:
-            self.get_docker_client().containers.get(self.get_name())
+            container: docker.models.containers.Container = self.get_docker_client().containers.get(self.get_name())
         except Exception as e:
             if self.sync_obj.terminate is not True:
                 self.pbar.write(
                     f"Error: could not get container [{self.get_name()}]: {str(e)}"
                 )
                 self.sync_obj.terminate = True
+        else:
+            if container.status != 'running':
+                self.sync_obj.terminate = True
+                self.container = None
 
         if self.sync_obj.terminate:
             self.finish()