From a32fc9e682145ef05223a896feafd56b0a0dfe97 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Tue, 11 Jul 2023 19:51:13 -0700 Subject: [PATCH] Clean up messy contentctl.yml file by moving test parameter to contentctl_test.yml. Add support for this in test as well as init modes. Some more cleanup for appinspect output. --- contentctl/actions/initialize.py | 17 +++++++++++----- contentctl/contentctl.py | 18 +++++++++++++---- contentctl/helper/config_handler.py | 30 ++++++++++++++++++++++------- contentctl/input/yml_reader.py | 5 ++++- contentctl/objects/config.py | 10 ++++------ contentctl/output/conf_output.py | 4 +--- 6 files changed, 58 insertions(+), 26 deletions(-) diff --git a/contentctl/actions/initialize.py b/contentctl/actions/initialize.py index 2080eca2..596c7ad5 100644 --- a/contentctl/actions/initialize.py +++ b/contentctl/actions/initialize.py @@ -3,7 +3,7 @@ import os import pathlib from dataclasses import dataclass -from contentctl.objects.config import Config +from contentctl.objects.config import Config, TestConfig from contentctl.output.yml_writer import YmlWriter @dataclass(frozen=True) @@ -16,15 +16,22 @@ class Initialize: def execute(self, input_dto: InitializeInputDto) -> None: c = Config() + t = TestConfig.construct() #Disable validation for default object + + config_as_dict = c.dict() + config_as_dict.pop("test") + YmlWriter.writeYmlFile(os.path.join(input_dto.path, 'contentctl.yml'), config_as_dict) + + # This field serialization hack is required to get # enums declared in Pydantic Models serialized properly # without emitting tags that make them hard to read in yml import json - j = json.dumps(c.dict(),sort_keys=False) + j = json.dumps(t.dict(),sort_keys=False) obj=json.loads(j) - - YmlWriter.writeYmlFile(os.path.join(input_dto.path, 'contentctl.yml'), dict(obj)) - + YmlWriter.writeYmlFile(os.path.join(input_dto.path, 'contentctl_test.yml'), dict(obj)) + + folders = ['detections', 'stories', 'lookups', 'macros', 'baselines', 'dist', 'docs', 'reporting'] for folder in folders: os.makedirs(os.path.join(input_dto.path, folder)) diff --git a/contentctl/contentctl.py b/contentctl/contentctl.py index f06f31df..0e5fc7d7 100644 --- a/contentctl/contentctl.py +++ b/contentctl/contentctl.py @@ -89,8 +89,13 @@ def print_ascii_art(): ) -def start(args, validate_test=False) -> Config: - return ConfigHandler.read_config(pathlib.Path(args.path)/"contentctl.yml", validate_test) +def start(args, read_test_file:bool = False) -> Config: + base_config = ConfigHandler.read_config(pathlib.Path(args.path)/"contentctl.yml") + if read_test_file: + base_config.test = ConfigHandler.read_test_config(pathlib.Path(args.path)/"contentctl_test.yml") + return base_config + + def initialize(args) -> None: @@ -129,7 +134,7 @@ def deploy(args) -> None: def test(args: argparse.Namespace): args = configure_unattended(args) - config = start(args, validate_test=True) + config = start(args, read_test_file=True) # set some arguments that are not @@ -374,4 +379,9 @@ def main(): args = parser.parse_args() print_ascii_art() - args.func(args) + try: + args.func(args) + except Exception as e: + print(f"Error during contentctl:\n{str(e)}") + sys.exit(1) + diff --git a/contentctl/helper/config_handler.py b/contentctl/helper/config_handler.py index 55b45bc1..958efde0 100644 --- a/contentctl/helper/config_handler.py +++ b/contentctl/helper/config_handler.py @@ -4,28 +4,44 @@ import pathlib from contentctl.input.yml_reader import YmlReader -from contentctl.objects.config import Config +from contentctl.objects.config import Config, TestConfig class ConfigHandler: @classmethod - def read_config(cls, config_path: pathlib.Path, validate_test:bool=False) -> Config: + def read_config(cls, config_path: pathlib.Path) -> Config: try: - yml_dict = YmlReader.load_file(config_path) + yml_dict = YmlReader.load_file(config_path, add_fields=False) + except: print("ERROR: no contentctl.yml found in given path") sys.exit(1) try: - if validate_test == False: - yml_dict.pop("test") config = Config.parse_obj(yml_dict) except Exception as e: - print(e) - sys.exit(1) + raise Exception(f"Error reading config file: {str(e)}") + return config + + @classmethod + def read_test_config(cls, test_config_path: pathlib.Path) -> TestConfig: + try: + yml_dict = YmlReader.load_file(test_config_path, add_fields=False) + except: + print("ERROR: no contentctl_test.yml found in given path") + sys.exit(1) + + try: + test_config = TestConfig.parse_obj(yml_dict) + except Exception as e: + raise Exception(f"Error reading test config file: {str(e)}") + + + return test_config + diff --git a/contentctl/input/yml_reader.py b/contentctl/input/yml_reader.py index f9888a20..ee524ff0 100644 --- a/contentctl/input/yml_reader.py +++ b/contentctl/input/yml_reader.py @@ -7,7 +7,7 @@ class YmlReader(): @staticmethod - def load_file(file_path: pathlib.Path) -> Dict: + def load_file(file_path: pathlib.Path, add_fields=True) -> Dict: try: file_handler = open(file_path, 'r', encoding="utf-8") try: @@ -20,6 +20,9 @@ def load_file(file_path: pathlib.Path) -> Dict: print(exc) sys.exit(1) + if add_fields == False: + return yml_obj + yml_obj['file_path'] = str(file_path) if 'deprecated' in [parent.name for parent in file_path.parents]: diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index cad0a8ec..1e1b4d82 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, validator, ValidationError, Field +from pydantic import BaseModel, validator, ValidationError, Field, Extra import semantic_version from datetime import datetime from typing import Union @@ -143,15 +143,13 @@ class ConfigEnrichments(BaseModel): -class Config(BaseModel): +class Config(BaseModel, extra=Extra.forbid): #general: ConfigGlobal = ConfigGlobal() detection_configuration: ConfigDetectionConfiguration = ConfigDetectionConfiguration() deployments: Deployments = Deployments() build: ConfigBuild = ConfigBuild() enrichments: ConfigEnrichments = ConfigEnrichments() - rest_api_deployment_targets: list[ConfigDeployRestAPI] = [ConfigDeployRestAPI()] - acs_deployment_targets: list[ConfigDeployACS] = [] - test: TestConfig = TestConfig.construct() #Disable validation for default object - + test: Union[TestConfig,None] = None + diff --git a/contentctl/output/conf_output.py b/contentctl/output/conf_output.py index e7bf779f..b5f1b509 100644 --- a/contentctl/output/conf_output.py +++ b/contentctl/output/conf_output.py @@ -195,11 +195,9 @@ def inspectApp(self)-> None: for opt in excluded_tags: options_list += [EXCLUDED_TAGS_OPTION, opt] - cmdline = options_list + [arg[1] for arg in arguments_list] - print(cmdline) + cmdline = options_list + [arg[1] for arg in arguments_list] validate(cmdline) - #validate([str(name_without_version)], PRECERT_MODE, included_tags_string, excluded_tags_string) except SystemExit as e: if e.code == 0: print(f"AppInspect passed! Please check [ {appinspect_output} , {appinspect_logging} ] for verbose information.")