Better support for reading and

validating the test section of
contentctl.yml. Ability to server
status web page on non
localhost connections. Avoid
re-validation of apps in test field.

contentctl/actions/detection_testing/views/DetectionTestingViewWeb.py

@@ -92,15 +92,15 @@ def log_exception(*args, **kwargs):
 
         self.options["handler_class"] = DontLog
         self.server = make_server(
-            "localhost", DEFAULT_WEB_UI_PORT, handler, **self.options
+            self.host, DEFAULT_WEB_UI_PORT, handler, **self.options
         )
 
         self.server.serve_forever()
 
 
 class DetectionTestingViewWeb(DetectionTestingView):
     bottleApp: Bottle = Bottle()
-    server: SimpleWebServer = SimpleWebServer()
+    server: SimpleWebServer = SimpleWebServer(host="0.0.0.0", port=DEFAULT_WEB_UI_PORT)
 
     class Config:
         arbitrary_types_allowed = True
@@ -117,7 +117,7 @@ def setup(self):
         t.start()
 
         try:
-            webbrowser.open(f"http://localhost:{DEFAULT_WEB_UI_PORT}")
+            webbrowser.open(f"http://{self.server.host}:{DEFAULT_WEB_UI_PORT}")
         except Exception as e:
             print(f"Could not open webbrowser for status page: {str(e)}")
 

contentctl/contentctl.py

@@ -39,6 +39,7 @@
 
 import tqdm
 import functools
+from typing import Union
 
 
 def configure_unattended(args: argparse.Namespace) -> argparse.Namespace:
@@ -88,17 +89,18 @@ def print_ascii_art():
     )
 
 
-def start(args) -> Config:
-    return ConfigHandler.read_config(pathlib.Path(args.path)/"contentctl.yml")
+def start(args, validate_test=False) -> Config:
+    return ConfigHandler.read_config(pathlib.Path(args.path)/"contentctl.yml", validate_test)
 
 
 def initialize(args) -> None:
 
     Initialize().execute(InitializeInputDto(path=pathlib.Path(args.path)))
 
 
-def build(args) -> DirectorOutputDto:
-    config = start(args)
+def build(args, config:Union[Config,None]=None) -> DirectorOutputDto:
+    if config == None:
+        config = start(args)
     product_type = SecurityContentProduct.SPLUNK_APP
     director_input_dto = DirectorInputDto(
         input_path=os.path.abspath(args.path), product=product_type, config=config
@@ -127,7 +129,8 @@ def deploy(args) -> None:
 
 def test(args: argparse.Namespace):
     args = configure_unattended(args)
-    config = start(args)
+    config = start(args, validate_test=True)
+
 
     # set some arguments that are not
     # yet exposed/written properly in
@@ -136,23 +139,14 @@ def test(args: argparse.Namespace):
     config.test.num_containers=1 
     config.test.post_test_behavior=PostTestBehavior(args.behavior)
     config.test.detections_list=args.detections_list
-    '''
-    test_config = TestConfig.parse_obj(
-        {
-            # "repo_path": args.path,
-            "mode": args.mode,
-            "num_containers": 1,
-            "post_test_behavior": args.behavior,
-            "detections_list": args.detections_list,
-        }
-    )
-    '''
+
+
 
     # We do this before generating the app to save some time if options are incorrect.
     # For example, if the detection(s) we are trying to test do not exist
     githubService = GithubService(config.test)
 
-    director_output_dto = build(args)
+    director_output_dto = build(args, config)
 
     # All this information will later come from the config, so we will
     # be able to do it in Test().execute. For now, we will do it here
@@ -255,7 +249,7 @@ def main():
     :param args: arguments passed by the user on command line while calling the script.
     :return: returns the output of the function called.
     """
-
+    
     # grab arguments
     parser = argparse.ArgumentParser(
         description="Use `contentctl action -h` to get help with any Splunk content action"
@@ -373,7 +367,7 @@ def main():
     )
 
     test_parser.add_argument("--unattended", action=argparse.BooleanOptionalAction)
-
+    
     test_parser.set_defaults(func=test)
 
     # parse them

contentctl/helper/config_handler.py

@@ -10,14 +10,16 @@
 class ConfigHandler:
 
     @classmethod
-    def read_config(cls, config_path: pathlib.Path) -> Config:
+    def read_config(cls, config_path: pathlib.Path, validate_test:bool=False) -> Config:
         try:
             yml_dict = YmlReader.load_file(config_path)
         except:
             print("ERROR: no contentctl.yml found in given path")
             sys.exit(1)
 
         try: 
+            if validate_test == False:
+                yml_dict.pop("test")
             config = Config.parse_obj(yml_dict)
         except Exception as e:
             print(e)

contentctl/objects/app.py

@@ -17,7 +17,7 @@
 import yaml
 
 SPLUNKBASE_URL = "https://splunkbase.splunk.com/app/{uid}/release/{release}/download"
-
+ENVIRONMENT_PATH_NOT_SET = "ENVIRONMENT_PATH_NOT_SET"
 
 class App(BaseModel, extra=Extra.forbid):
 
@@ -43,7 +43,7 @@ class App(BaseModel, extra=Extra.forbid):
     # Ultimate source of the app. Can be a local path or a Splunkbase Path.
     # This will be set via a function call and should not be provided in the YML
     # Note that this is the path relative to the container mount
-    environment_path: str = "ENVIRONMENT_PATH_NOT_SET"
+    environment_path: str = ENVIRONMENT_PATH_NOT_SET
 
     def configure_app_source_for_container(
         self,

contentctl/objects/config.py

@@ -151,7 +151,7 @@ class Config(BaseModel):
     enrichments: ConfigEnrichments = ConfigEnrichments()
     rest_api_deployment_targets: list[ConfigDeployRestAPI] = [ConfigDeployRestAPI()]
     acs_deployment_targets: list[ConfigDeployACS] = []
-    test: TestConfig = TestConfig()
+    test: TestConfig = TestConfig.construct() #Disable validation for default object
 
 
 

contentctl/objects/test_config.py

@@ -19,7 +19,7 @@
     DetectionTestingTargetInfrastructure,
 )
 
-from contentctl.objects.app import App
+from contentctl.objects.app import App, ENVIRONMENT_PATH_NOT_SET
 from contentctl.helper.utils import Utils
 
 
@@ -458,7 +458,7 @@ def validate_splunkbase_password(cls, v, values):
         else:
             return v
 
-    @validator("apps")
+    @validator("apps",)
     def validate_apps(cls, v, values):
         Utils.check_required_fields(
             "repo_url", values, ["splunkbase_username", "splunkbase_password"]
@@ -478,6 +478,10 @@ def validate_apps(cls, v, values):
             )
 
         for app in v:
+            if app.environment_path != ENVIRONMENT_PATH_NOT_SET:
+                #Avoid re-configuring the apps that have already been configured.
+                continue
+
             try:
                 app.configure_app_source_for_container(
                     username, password, app_directory, CONTAINER_APP_DIR

contentctl/templates/app_default.yml

@@ -4,54 +4,12 @@
   release: "1.0.4"
   http_path: https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/add-on-for-linux-sysmon_104.tgz
 
-- uid: 2757
-  appid: "Splunk_TA_paloalto"
-  title: "Palo Alto Networks Add-on for Splunk"
-  release: "7.1.0"
-  http_path: https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/palo-alto-networks-add-on-for-splunk_710.tgz
-
-- uid: 2882
-  appid: "Splunk_SA_Scientific_Python_linux_x86_64"
-  title: "Python for Scientific Computing (for Linux 64-bit)"
-  release: "4.0.0"
-  http_path: https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/python-for-scientific-computing-for-linux-64-bit_302.tgz
-
-- uid: 3719
-  appid: "Splunk_TA_aws-kinesis-firehose"
-  title: "Splunk Add-on for Amazon Kinesis Firehose"
-  release: "1.3.2"
-  http_path: https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-amazon-kinesis-firehose_132.tgz
-
-- uid: 4055
-  appid: "splunk_ta_o365"
-  title: "Splunk Add-on for Microsoft Office 365"
-  release: "4.1.0"
-  http_path: https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-microsoft-office-365_400.tgz
-
 - uid: 742
   appid: "Splunk_TA_windows"
   title: "Splunk Add-on for Microsoft Windows"
   release: "8.5.0"
   http_path: https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-microsoft-windows_850_PATCHED.tgz
 
-- uid: 3258
-  appid: "Splunk_TA_nginx"
-  title: "Splunk Add-on for NGINX"
-  release: "3.2.0"
-  http_path: https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-nginx_310.tgz
-
-- uid: 5238
-  appid: "Splunk_TA_stream"
-  title: "Splunk Add-on for Stream Forwarders"
-  release: "8.1.0"
-  http_path: https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-stream-forwarders_810.tgz
-
-- uid: 5234
-  appid: "Splunk_TA_stream_wire_data"
-  title: "Splunk Add-on for Stream Wire Data"
-  release: "8.1.0"
-  http_path: https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-stream-wire-data_810.tgz
-
 - uid: 5709
   appid: "Splunk_TA_microsoft_sysmon"
   title: "Splunk Add-on for Sysmon"
@@ -64,30 +22,6 @@
   release: "8.7.0"
   http_path: https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-unix-and-linux_860.tgz
 
-- uid: 1809
-  appid: "splunk_app_stream"
-  title: "Splunk App for Stream"
-  release: "8.1.0"
-  http_path: https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-app-for-stream_810.tgz
-
-- uid: 2890
-  appid: "Splunk_ML_Toolkit"
-  title: "Splunk Machine Learning Toolkit"
-  release: "5.3.3"
-  http_path: https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-machine-learning-toolkit_531.tgz
-
-- uid: 5466
-  appid: "Splunk_TA_zeek"
-  title: "TA for Zeek"
-  release: "1.0.5"
-  http_path: https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/ta-for-zeek_105.tgz
-
-- uid: 3110
-  appid: "Splunk_TA_microsoft-cloudservices"
-  title: "Splunk Add-on for Microsoft Cloud Services"
-  release: "4.5.0"
-  http_path: https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-microsoft-cloud-services_450.tgz
-
 - uid: 2734
   appid: "utbox"
   title: "URL Toolbox"