From cd601769b8191b1d5d56888117b1af7618bb9d0e Mon Sep 17 00:00:00 2001 From: Andy Wick Date: Wed, 6 Mar 2024 13:53:04 -0500 Subject: [PATCH 1/6] add new --viewer-prefix-list create parameter --- cdk-lib/cloud-demo.ts | 1 + cdk-lib/core/context-types.ts | 1 + cdk-lib/viewer-stacks/viewer-nodes-stack.ts | 16 +++++++- manage_arkime.py | 10 ++++- manage_arkime/commands/cluster_create.py | 28 +++++++------- manage_arkime/core/capacity_planning.py | 30 +++++++-------- manage_arkime/core/user_config.py | 4 +- .../commands/test_cluster_create.py | 38 +++++++++---------- 8 files changed, 76 insertions(+), 52 deletions(-) diff --git a/cdk-lib/cloud-demo.ts b/cdk-lib/cloud-demo.ts index 4d927dd..1819c51 100644 --- a/cdk-lib/cloud-demo.ts +++ b/cdk-lib/cloud-demo.ts @@ -100,6 +100,7 @@ case 'ClusterMgmtParams': { ssmParamNameViewerConfig: params.nameViewerConfigSsmParam, ssmParamNameViewerDetails: params.nameViewerDetailsSsmParam, planCluster: params.planCluster, + userConfig: params.userConfig, }); viewerNodesStack.addDependency(captureBucketStack); viewerNodesStack.addDependency(vpcStackToUse); diff --git a/cdk-lib/core/context-types.ts b/cdk-lib/core/context-types.ts index 8bf6919..13321c2 100644 --- a/cdk-lib/core/context-types.ts +++ b/cdk-lib/core/context-types.ts @@ -97,6 +97,7 @@ export interface UserConfig { historyDays: number; replicas: number; pcapDays: number; + viewerPrefixList: string; } /** diff --git a/cdk-lib/viewer-stacks/viewer-nodes-stack.ts b/cdk-lib/viewer-stacks/viewer-nodes-stack.ts index 13da272..0b02c51 100644 --- a/cdk-lib/viewer-stacks/viewer-nodes-stack.ts +++ b/cdk-lib/viewer-stacks/viewer-nodes-stack.ts @@ -12,7 +12,7 @@ import * as ssm from 'aws-cdk-lib/aws-ssm'; import * as path from 'path'; import { Construct } from 'constructs'; import * as ssmwrangling from '../core/ssm-wrangling'; -import * as plan from '../core/context-types'; +import * as types from '../core/context-types'; export interface ViewerNodesStackProps extends cdk.StackProps { readonly arnViewerCert: string; @@ -24,7 +24,8 @@ export interface ViewerNodesStackProps extends cdk.StackProps { readonly osPassword: secretsmanager.Secret; readonly ssmParamNameViewerConfig: string; readonly ssmParamNameViewerDetails: string; - readonly planCluster: plan.ClusterPlan; + readonly planCluster: types.ClusterPlan; + readonly userConfig: types.UserConfig; } export class ViewerNodesStack extends cdk.Stack { @@ -113,6 +114,17 @@ export class ViewerNodesStack extends cdk.Stack { internetFacing: true, loadBalancerName: `${props.clusterName}-Viewer`.toLowerCase() // Receives a random suffix, which minimizes DNS collisions }); + + // If we have a prefix list, we need to create a SG for the LB that allows traffic from the prefix list + if (props.userConfig.viewerPrefixList) { + const sg = new ec2.SecurityGroup(this, 'ALBSG', { + vpc: props.viewerVpc, + description: 'Control access viewer ALB', + }); + sg.addIngressRule(ec2.Peer.prefixList(props.userConfig.viewerPrefixList), ec2.Port.tcp(443), 'Allow HTTPS traffic from my prefix list'); + lb.addSecurityGroup(sg); + } + const listener = lb.addListener('Listener', { protocol: elbv2.ApplicationProtocol.HTTP, port: 80, diff --git a/manage_arkime.py b/manage_arkime.py index cd21879..f07b215 100755 --- a/manage_arkime.py +++ b/manage_arkime.py @@ -124,13 +124,19 @@ def demo_traffic_destroy(ctx): default=None, type=click.STRING, required=False) +@click.option( + "--viewer-prefix-list", + help=("The Prefix List to use for the Viewer LB."), + default=None, + type=click.STRING, + required=False) @click.pass_context def cluster_create(ctx, name, expected_traffic, spi_days, history_days, replicas, pcap_days, preconfirm_usage, - just_print_cfn, capture_cidr, viewer_cidr): + just_print_cfn, capture_cidr, viewer_cidr, viewer_prefix_list): profile = ctx.obj.get("profile") region = ctx.obj.get("region") cmd_cluster_create(profile, region, name, expected_traffic, spi_days, history_days, replicas, pcap_days, - preconfirm_usage, just_print_cfn, capture_cidr, viewer_cidr) + preconfirm_usage, just_print_cfn, capture_cidr, viewer_cidr, viewer_prefix_list) cli.add_command(cluster_create) @click.command(help="Tears down the Arkime Cluster in your account; by default, leaves your data intact") diff --git a/manage_arkime/commands/cluster_create.py b/manage_arkime/commands/cluster_create.py index 9bd73a7..e3c8553 100644 --- a/manage_arkime/commands/cluster_create.py +++ b/manage_arkime/commands/cluster_create.py @@ -32,7 +32,7 @@ logger = logging.getLogger(__name__) def cmd_cluster_create(profile: str, region: str, name: str, expected_traffic: float, spi_days: int, history_days: int, replicas: int, - pcap_days: int, preconfirm_usage: bool, just_print_cfn: bool, capture_cidr: str, viewer_cidr: str): + pcap_days: int, preconfirm_usage: bool, just_print_cfn: bool, capture_cidr: str, viewer_cidr: str, viewer_prefix_list: str): logger.debug(f"Invoking cluster-create with profile '{profile}' and region '{region}'") aws_provider = AwsClientProvider(aws_profile=profile, aws_region=region) @@ -51,7 +51,7 @@ def cmd_cluster_create(profile: str, region: str, name: str, expected_traffic: f # Generate our capacity plan, then confirm it's what the user expected and it's safe to proceed with the operation previous_user_config = _get_previous_user_config(name, aws_provider) - next_user_config = _get_next_user_config(name, expected_traffic, spi_days, history_days, replicas, pcap_days, aws_provider) + next_user_config = _get_next_user_config(name, expected_traffic, spi_days, history_days, replicas, pcap_days, viewer_prefix_list, aws_provider) previous_capacity_plan = _get_previous_capacity_plan(name, aws_provider) next_capacity_plan = _get_next_capacity_plan(next_user_config, previous_capacity_plan, capture_cidr, viewer_cidr, aws_provider) @@ -111,19 +111,19 @@ def _is_initial_invocation(cluster_name: str, aws_provider: AwsClientProvider) - def _should_proceed_with_operation(initial_invocation: bool, previous_capacity_plan: ClusterPlan, next_capacity_plan: ClusterPlan, previous_user_config: UserConfig, next_user_config: UserConfig, preconfirm_usage: bool, capture_cidr_block: str, viewer_cidr_block: str) -> bool: - + if (not initial_invocation) and (capture_cidr_block or viewer_cidr_block): # We can't change the CIDR without tearing down the VPC, which effectively means tearing down the entire # Cluster and re-creating it. Instead of attempting to do that, we make the CIDR only set-able on creation. logger.error("You can only specify the VPC CIDR(s) when you initially create the Cluster, as changing it" " requires tearing down the entire Cluster. Aborting...") return False - + if next_capacity_plan.viewerVpc: # Ensure the Viewer VPC's CIDR, if it exists, doesn't overlap with the Capture VPC's CIDR viewer_network = ipaddress.ip_network(next_capacity_plan.viewerVpc.cidr.block, strict=False) - capture_network = ipaddress.ip_network(next_capacity_plan.captureVpc.cidr.block, strict=False) - + capture_network = ipaddress.ip_network(next_capacity_plan.captureVpc.cidr.block, strict=False) + if viewer_network.overlaps(capture_network): logger.error(f"Your specified Viewer VPC CIDR ({str(viewer_network)}) overlaps with your Capture VPC" f" CIDR ({str(capture_network)}). Please ensure these two CIDRs do not overlap.") @@ -139,7 +139,7 @@ def _should_proceed_with_operation(initial_invocation: bool, previous_capacity_p logger.error(f"Your specified Capture capacity plan does not fit in the VPC; there are {available_ips} usable IPs in your VPC" f" and your plan requires {required_ips} IPs. Aborting...") return False - + return True def _get_previous_user_config(cluster_name: str, aws_provider: AwsClientProvider) -> UserConfig: @@ -157,9 +157,9 @@ def _get_previous_user_config(cluster_name: str, aws_provider: AwsClientProvider return UserConfig(None, None, None, None, None) def _get_next_user_config(cluster_name: str, expected_traffic: float, spi_days: int, history_days: int, replicas: int, - pcap_days: int, aws_provider: AwsClientProvider) -> UserConfig: + pcap_days: int, viewer_prefix_list: str, aws_provider: AwsClientProvider) -> UserConfig: # At least one parameter isn't defined - if None in [expected_traffic, spi_days, replicas, pcap_days, history_days]: + if None in [expected_traffic, spi_days, replicas, pcap_days, history_days, viewer_prefix_list]: # Re-use the existing configuration if it exists try: stored_config_json = ssm_ops.get_ssm_param_json_value( @@ -179,12 +179,14 @@ def _get_next_user_config(cluster_name: str, expected_traffic: float, spi_days: user_config.replicas = replicas if pcap_days is not None: user_config.pcapDays = pcap_days + if viewer_prefix_list is not None: + user_config.viewerPrefixList = viewer_prefix_list return user_config # Existing configuration doesn't exist, use defaults except ssm_ops.ParamDoesNotExist: - return UserConfig(MINIMUM_TRAFFIC, DEFAULT_SPI_DAYS, DEFAULT_HISTORY_DAYS, DEFAULT_REPLICAS, DEFAULT_S3_STORAGE_DAYS) + return UserConfig(MINIMUM_TRAFFIC, DEFAULT_SPI_DAYS, DEFAULT_HISTORY_DAYS, DEFAULT_REPLICAS, DEFAULT_S3_STORAGE_DAYS, None) # All of the parameters defined else: return UserConfig(expected_traffic, spi_days, history_days, replicas, pcap_days) @@ -361,7 +363,7 @@ def _tag_domain(cluster_name: str, aws_provider: AwsClientProvider): "domainArn", aws_provider ) - + opensearch_client = aws_provider.get_opensearch() opensearch_client.add_tags( ARN=os_domain_Arn, @@ -391,7 +393,7 @@ def _get_stacks_to_deploy(cluster_name: str, next_user_config: UserConfig, next_ def _get_cdk_context(cluster_name: str, next_user_config: UserConfig, next_capacity_plan: ClusterPlan, cert_arn: str, aws_env: AwsEnvironment): - + # We might not deploy all these, but we need to tell the CDK that they exist as something we might deploy in order # for its auto-wiring to work. stack_names = context.ClusterStackNames( @@ -410,4 +412,4 @@ def _get_cdk_context(cluster_name: str, next_user_config: UserConfig, next_capac next_user_config, constants.get_config_bucket_name(aws_env.aws_account, aws_env.aws_region, cluster_name), stack_names - ) \ No newline at end of file + ) diff --git a/manage_arkime/core/capacity_planning.py b/manage_arkime/core/capacity_planning.py index 340c2b3..5e63d1d 100644 --- a/manage_arkime/core/capacity_planning.py +++ b/manage_arkime/core/capacity_planning.py @@ -110,14 +110,14 @@ def get_capture_node_capacity_plan(expected_traffic: float, azs: List[str]) -> C if expected_traffic > MAX_TRAFFIC: raise TooMuchTraffic(expected_traffic) - + chosen_instance = next(instance for instance in CAPTURE_INSTANCES if expected_traffic <= instance.maxTraffic) desired_instances = max( chosen_instance.minNodes, math.ceil(expected_traffic/chosen_instance.trafficPer) ) - + return CaptureNodesPlan( chosen_instance.instanceType, desired_instances, @@ -373,20 +373,20 @@ def _validate_cidr(self, cidr_str: str ): overall_form = re.compile("^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}/[0-9]{1,2}$") if not overall_form.match(cidr_str): raise InvalidCidr(cidr_str) - + prefix_portion = cidr_str.split("/")[0] prefix_portions_correct = list(map(lambda x: int(x) >=0 and int(x) <= 255, prefix_portion.split("."))) if not all(prefix_portions_correct): raise InvalidCidr(cidr_str) - + mask_portion = cidr_str.split("/")[1] mask_portion_correct = int(mask_portion) >= 0 and int(mask_portion) <= 32 if not mask_portion_correct: raise InvalidCidr(cidr_str) - + def __eq__(self, other) -> bool: return (self.block == other.block and self.prefix == other.prefix and self.mask == other.mask) - + def __str__(self) -> str: return self.block @@ -396,7 +396,7 @@ def to_dict(self) -> Dict[str, any]: "prefix": self.prefix, "mask": self.mask, } - + DEFAULT_VPC_CIDR = Cidr("10.0.0.0/16") # What AWS VPC gives by default DEFAULT_CAPTURE_PUBLIC_MASK = 28 # minimum subnet size; we don't need much in the Capture VPC public subnets DEFAULT_VIEWER_PUBLIC_MASK = 28 # minimum subnet size; we don't need much in the Viewer VPC public subnets either @@ -433,7 +433,7 @@ def from_dict(cls: Type[T_VpcPlan], input: Dict[str, any]) -> T_VpcPlan: publicSubnetMask = input["publicSubnetMask"] return cls(cidr, azs, publicSubnetMask) - + def get_usable_ips(self) -> int: total_ips = 2 ** (32 - int(self.cidr.mask)) public_ips = 2 ** (32 - int(self.publicSubnetMask)) * len(self.azs) @@ -441,7 +441,7 @@ def get_usable_ips(self) -> int: reserved_private_ips = reserved_ips_per_subnet * len(self.azs) return total_ips - public_ips - reserved_private_ips - + def get_capture_vpc_plan(previous_plan: VpcPlan, capture_cidr_block: str, azs: List[str]) -> VpcPlan: if previous_plan and all(value is not None for value in vars(previous_plan).values()): return previous_plan @@ -449,7 +449,7 @@ def get_capture_vpc_plan(previous_plan: VpcPlan, capture_cidr_block: str, azs: L return VpcPlan(DEFAULT_VPC_CIDR, azs, DEFAULT_CAPTURE_PUBLIC_MASK) else: return VpcPlan(Cidr(capture_cidr_block), azs, DEFAULT_CAPTURE_PUBLIC_MASK) - + def get_viewer_vpc_plan(previous_plan: VpcPlan, viewer_cidr_block: str, azs: List[str]) -> VpcPlan: if previous_plan and all(value is not None for value in vars(previous_plan).values()): return previous_plan @@ -488,7 +488,7 @@ class ClusterPlan: viewerVpc: VpcPlan def __eq__(self, other) -> bool: - return (self.captureNodes == other.captureNodes and self.captureVpc == other.captureVpc + return (self.captureNodes == other.captureNodes and self.captureVpc == other.captureVpc and self.ecsResources == other.ecsResources and self.osDomain == other.osDomain and self.s3 == other.s3 and self.viewerNodes == other.viewerNodes and self.viewerVpc == other.viewerVpc) @@ -522,15 +522,15 @@ def from_dict(cls: Type[T_ClusterPlan], input: Dict[str, any]) -> T_ClusterPlan: viewer_vpc = None return cls(capture_nodes, capture_vpc, ecs_resources, os_domain, s3, viewer_nodes, viewer_vpc) - + def get_required_capture_ips(self) -> int: required_capture_ips = self.captureNodes.maxCount + self.osDomain.dataNodes.count + self.osDomain.masterNodes.count required_viewer_ips = self.viewerNodes.maxCount if not self.viewerVpc else 0 return required_capture_ips + required_viewer_ips - + def will_capture_plan_fit(self) -> bool: usable_ips = self.captureVpc.get_usable_ips() - required_ips = self.get_required_capture_ips() + required_ips = self.get_required_capture_ips() - return usable_ips >= required_ips \ No newline at end of file + return usable_ips >= required_ips diff --git a/manage_arkime/core/user_config.py b/manage_arkime/core/user_config.py index c75aca2..ba7814a 100644 --- a/manage_arkime/core/user_config.py +++ b/manage_arkime/core/user_config.py @@ -11,6 +11,7 @@ class UserConfig: historyDays: int replicas: int pcapDays: int + viewerPrefixList: str = None """ Only process fields we still need, this allows us to ignore config no longer used """ @classmethod @@ -29,6 +30,7 @@ def to_dict(self) -> Dict[str, any]: 'spiDays': self.spiDays, 'replicas': self.replicas, 'pcapDays': self.pcapDays, - 'historyDays': self.historyDays + 'historyDays': self.historyDays, + 'viewerPrefixList': self.viewerPrefixList, } diff --git a/test_manage_arkime/commands/test_cluster_create.py b/test_manage_arkime/commands/test_cluster_create.py index 735e5af..fcbc60e 100644 --- a/test_manage_arkime/commands/test_cluster_create.py +++ b/test_manage_arkime/commands/test_cluster_create.py @@ -74,7 +74,7 @@ def test_WHEN_cmd_cluster_create_called_THEN_cdk_command_correct(mock_cdk_client mock_get_context.return_value = {"key": "value"} # Run our test - cmd_cluster_create("profile", "region", "my-cluster", None, None, None, None, None, True, False, None, None) + cmd_cluster_create("profile", "region", "my-cluster", None, None, None, None, None, True, False, None, None, None) # Check our results expected_calls = [ @@ -146,7 +146,7 @@ def test_WHEN_cmd_cluster_create_called_AND_ver_mismatch_THEN_as_expected(mock_c mock_confirm_ver.side_effect = CliClusterVersionMismatch(2, 1) # Run our test - cmd_cluster_create("profile", "region", "my-cluster", None, None, None, None, None, True, False, "1.2.3.4/24", "2.3.4.5/26") + cmd_cluster_create("profile", "region", "my-cluster", None, None, None, None, None, True, False, "1.2.3.4/24", "2.3.4.5/26", None) # Check our results expected_proceed_calls = [] @@ -200,7 +200,7 @@ def test_WHEN_cmd_cluster_create_called_AND_shouldnt_proceed_THEN_as_expected(mo mock_proceed.return_value = False # Run our test - cmd_cluster_create("profile", "region", "my-cluster", None, None, None, None, None, True, False, "1.2.3.4/24", "2.3.4.5/26") + cmd_cluster_create("profile", "region", "my-cluster", None, None, None, None, None, True, False, "1.2.3.4/24", "2.3.4.5/26", None) # Check our results expected_proceed_calls = [ @@ -283,7 +283,7 @@ def test_WHEN_cmd_cluster_create_called_AND_just_print_THEN_as_expected(mock_cdk mock_get_context.return_value = {"key": "value"} # Run our test - cmd_cluster_create("profile", "region", "my-cluster", None, None, None, None, None, True, True, None, None) + cmd_cluster_create("profile", "region", "my-cluster", None, None, None, None, None, True, True, None, None, None) # Check our results expected_calls = [ @@ -337,7 +337,7 @@ def test_WHEN_cmd_cluster_create_called_AND_just_print_THEN_as_expected(mock_cdk def test_WHEN_should_proceed_with_operation_AND_happy_path_THEN_as_expected(mock_confirm): # Set up our mock azs = ["az1", "az2"] - user_config = UserConfig(1, 30, 365, 2, 30) + user_config = UserConfig(1, 30, 365, 2, 30) cluster_plan = ClusterPlan( CaptureNodesPlan("m5.xlarge", 1, 2, 1), @@ -367,7 +367,7 @@ def test_WHEN_should_proceed_with_operation_AND_happy_path_THEN_as_expected(mock def test_WHEN_should_proceed_with_operation_AND_change_capture_cidr_THEN_as_expected(mock_confirm): # Set up our mock azs = ["az1", "az2"] - user_config = UserConfig(1, 30, 365, 2, 30) + user_config = UserConfig(1, 30, 365, 2, 30) cluster_plan = ClusterPlan( CaptureNodesPlan("m5.xlarge", 20, 25, 1), @@ -394,7 +394,7 @@ def test_WHEN_should_proceed_with_operation_AND_change_capture_cidr_THEN_as_expe def test_WHEN_should_proceed_with_operation_AND_change_viewer_cidr_THEN_as_expected(mock_confirm): # Set up our mock azs = ["az1", "az2"] - user_config = UserConfig(1, 30, 365, 2, 30) + user_config = UserConfig(1, 30, 365, 2, 30) cluster_plan = ClusterPlan( CaptureNodesPlan("m5.xlarge", 20, 25, 1), @@ -432,7 +432,7 @@ def test_WHEN_should_proceed_with_operation_AND_check_overlapping_cidrs_THEN_as_ S3Plan(DEFAULT_S3_STORAGE_CLASS, DEFAULT_S3_STORAGE_DAYS), ViewerNodesPlan(20, 5), VpcPlan(Cidr("2.3.4.5/24"), azs, DEFAULT_CAPTURE_PUBLIC_MASK), - ) + ) actual_value = _should_proceed_with_operation(False, cluster_plan, cluster_plan, user_config, user_config, True, None, None) assert True == actual_value @@ -458,7 +458,7 @@ def test_WHEN_should_proceed_with_operation_AND_check_overlapping_cidrs_THEN_as_ def test_WHEN_should_proceed_with_operation_AND_abort_usage_THEN_as_expected(mock_confirm): # Set up our mock azs = ["az1", "az2"] - user_config = UserConfig(1, 30, 365, 2, 30) + user_config = UserConfig(1, 30, 365, 2, 30) cluster_plan = ClusterPlan( CaptureNodesPlan("m5.xlarge", 1, 2, 1), @@ -487,7 +487,7 @@ def test_WHEN_should_proceed_with_operation_AND_abort_usage_THEN_as_expected(moc def test_WHEN_should_proceed_with_operation_AND_doesnt_fit_THEN_as_expected(mock_confirm): # Set up our mock azs = ["az1", "az2"] - user_config = UserConfig(1, 30, 365, 2, 30) + user_config = UserConfig(1, 30, 365, 2, 30) cluster_plan = ClusterPlan( CaptureNodesPlan("m5.xlarge", 100, 200, 100), @@ -574,7 +574,7 @@ def test_WHEN_get_next_user_config_called_AND_use_existing_THEN_as_expected(mock mock_provider = mock.Mock() # Run our test - actual_value = _get_next_user_config("my-cluster", None, None, None, None, None, mock_provider) + actual_value = _get_next_user_config("my-cluster", None, None, None, None, None, None, mock_provider) # Check our results assert UserConfig(1.2, 40, 120, 2, 35) == actual_value @@ -600,7 +600,7 @@ def test_WHEN_get_next_user_config_called_AND_partial_update_THEN_as_expected(mo mock_provider = mock.Mock() # Run our test - actual_value = _get_next_user_config("my-cluster", None, 30, None, None, None, mock_provider) + actual_value = _get_next_user_config("my-cluster", None, 30, None, None, None, None, mock_provider) # Check our results assert UserConfig(1.2, 30, 120, 2, 35) == actual_value @@ -619,7 +619,7 @@ def test_WHEN_get_next_user_config_called_AND_use_default_THEN_as_expected(mock_ mock_provider = mock.Mock() # Run our test - actual_value = _get_next_user_config("my-cluster", None, None, None, None, None, mock_provider) + actual_value = _get_next_user_config("my-cluster", None, None, None, None, None, None, mock_provider) # Check our results assert UserConfig(MINIMUM_TRAFFIC, DEFAULT_SPI_DAYS, DEFAULT_HISTORY_DAYS, DEFAULT_REPLICAS, DEFAULT_S3_STORAGE_DAYS) == actual_value @@ -637,10 +637,10 @@ def test_WHEN_get_next_user_config_called_AND_specify_all_THEN_as_expected(mock_ mock_provider = mock.Mock() # Run our test - actual_value = _get_next_user_config("my-cluster", 10, 40, 120, 2, 35, mock_provider) + actual_value = _get_next_user_config("my-cluster", 10, 40, 120, 2, 35, "viewer-prefix-list", mock_provider) # Check our results - assert UserConfig(10, 40, 120, 2, 35) == actual_value + assert UserConfig(10, 40, 120, 2, 35, "viewer-prefix-list") == actual_value expected_get_ssm_calls = [] assert expected_get_ssm_calls == mock_ssm_ops.get_ssm_param_json_value.call_args_list @@ -1176,7 +1176,7 @@ def test_WHEN_is_initial_invocation_called_THEN_as_expected(mock_get_ssm): actual_value = _is_initial_invocation("MyCluster", mock_provider) assert True == actual_value - + expected_get_ssm_calls = [ mock.call(constants.get_cluster_ssm_param_name("MyCluster"), mock_provider) ] @@ -1190,7 +1190,7 @@ def test_WHEN_is_initial_invocation_called_THEN_as_expected(mock_get_ssm): def test_WHEN_get_stacks_to_deploy_called_THEN_as_expected(): cluster_name = "MyCluster" - user_config = UserConfig(1, 30, 365, 2, 30) + user_config = UserConfig(1, 30, 365, 2, 30) # TEST: No Viewer VPC cluster_plan = ClusterPlan( @@ -1212,7 +1212,7 @@ def test_WHEN_get_stacks_to_deploy_called_THEN_as_expected(): constants.get_opensearch_domain_stack_name(cluster_name), constants.get_viewer_nodes_stack_name(cluster_name) ] - assert expected_value == actual_value + assert expected_value == actual_value # TEST: Has a Viewer VPC cluster_plan = ClusterPlan( @@ -1286,4 +1286,4 @@ def test_WHEN_get_cdk_context_called_THEN_as_expected(): } assert expected_value == actual_value - \ No newline at end of file + From 996f5bd51ca9e902dbc74cf2d803f1cffa651ce4 Mon Sep 17 00:00:00 2001 From: Andy Wick Date: Wed, 27 Mar 2024 10:18:23 -0400 Subject: [PATCH 2/6] do not create HTTP listener --- cdk-lib/viewer-stacks/viewer-nodes-stack.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cdk-lib/viewer-stacks/viewer-nodes-stack.ts b/cdk-lib/viewer-stacks/viewer-nodes-stack.ts index 0b02c51..cc6a429 100644 --- a/cdk-lib/viewer-stacks/viewer-nodes-stack.ts +++ b/cdk-lib/viewer-stacks/viewer-nodes-stack.ts @@ -125,12 +125,6 @@ export class ViewerNodesStack extends cdk.Stack { lb.addSecurityGroup(sg); } - const listener = lb.addListener('Listener', { - protocol: elbv2.ApplicationProtocol.HTTP, - port: 80, - open: true - }); - // Our Arkime Capture container const container = taskDefinition.addContainer('ViewerContainer', { image: ecs.ContainerImage.fromAsset(path.resolve(__dirname, '..', '..', 'docker-viewer-node')), @@ -153,6 +147,13 @@ export class ViewerNodesStack extends cdk.Stack { hostPort: viewerPort }); + /* + const listener = lb.addListener('Listener', { + protocol: elbv2.ApplicationProtocol.HTTP, + port: 80, + open: true + }); + listener.addTargets('TargetGroup', { protocol: elbv2.ApplicationProtocol.HTTP, port: viewerPort, @@ -168,6 +169,7 @@ export class ViewerNodesStack extends cdk.Stack { interval: cdk.Duration.seconds(30), }, }); + */ const certificate = acm.Certificate.fromCertificateArn(this, 'ViewerCert', props.arnViewerCert); const httpsListener = lb.addListener('HttpsListener', { From 7c25a8b0fa67cd6e3bf7efafc5b72f2542cb25a2 Mon Sep 17 00:00:00 2001 From: Andy Wick Date: Wed, 27 Mar 2024 11:41:19 -0400 Subject: [PATCH 3/6] update eq to check viewerPrefixList --- manage_arkime/core/user_config.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/manage_arkime/core/user_config.py b/manage_arkime/core/user_config.py index ba7814a..c4737ff 100644 --- a/manage_arkime/core/user_config.py +++ b/manage_arkime/core/user_config.py @@ -21,8 +21,12 @@ def from_dict(cls, d): return cls(**valid_kwargs) def __eq__(self, other): - return (self.expectedTraffic == other.expectedTraffic and self.spiDays == other.spiDays - and self.replicas == other.replicas and self.pcapDays == other.pcapDays and self.historyDays == other.historyDays) + return (self.expectedTraffic == other.expectedTraffic and + self.spiDays == other.spiDays and + self.historyDays == other.historyDays and + self.replicas == other.replicas and + self.pcapDays == other.pcapDays and + self.viewerPrefixList == other.viewerPrefixList) def to_dict(self) -> Dict[str, any]: return { From 34d8fe60d45ded32d4d2507c1505e9469b95e5af Mon Sep 17 00:00:00 2001 From: Andy Wick Date: Wed, 27 Mar 2024 12:19:03 -0400 Subject: [PATCH 4/6] fix pytest --- manage_arkime/commands/cluster_create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manage_arkime/commands/cluster_create.py b/manage_arkime/commands/cluster_create.py index e3c8553..a85ff4e 100644 --- a/manage_arkime/commands/cluster_create.py +++ b/manage_arkime/commands/cluster_create.py @@ -189,7 +189,7 @@ def _get_next_user_config(cluster_name: str, expected_traffic: float, spi_days: return UserConfig(MINIMUM_TRAFFIC, DEFAULT_SPI_DAYS, DEFAULT_HISTORY_DAYS, DEFAULT_REPLICAS, DEFAULT_S3_STORAGE_DAYS, None) # All of the parameters defined else: - return UserConfig(expected_traffic, spi_days, history_days, replicas, pcap_days) + return UserConfig(expected_traffic, spi_days, history_days, replicas, pcap_days, viewer_prefix_list) def _get_previous_capacity_plan(cluster_name: str, aws_provider: AwsClientProvider) -> ClusterPlan: # Pull the existing plan, if possible From aa4725f4c9b5df0f87f0eb2adb70ac6e8416e82f Mon Sep 17 00:00:00 2001 From: Andy Wick Date: Wed, 27 Mar 2024 12:36:37 -0400 Subject: [PATCH 5/6] move to v4 checkout to reduce warnings --- .github/workflows/auto_tag.yml | 4 ++-- .github/workflows/eslint.yml | 2 +- .github/workflows/manage_arkime.yml | 2 +- .github/workflows/tests.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/auto_tag.yml b/.github/workflows/auto_tag.yml index d1af22f..3190381 100644 --- a/.github/workflows/auto_tag.yml +++ b/.github/workflows/auto_tag.yml @@ -13,7 +13,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.merge_commit_sha }} fetch-depth: '0' @@ -23,4 +23,4 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DEFAULT_BUMP: minor - WITH_V: true \ No newline at end of file + WITH_V: true diff --git a/.github/workflows/eslint.yml b/.github/workflows/eslint.yml index e981ab7..08aace6 100644 --- a/.github/workflows/eslint.yml +++ b/.github/workflows/eslint.yml @@ -4,7 +4,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Install modules run: npm ci - name: Run ESLint diff --git a/.github/workflows/manage_arkime.yml b/.github/workflows/manage_arkime.yml index b0902a7..54229d0 100644 --- a/.github/workflows/manage_arkime.yml +++ b/.github/workflows/manage_arkime.yml @@ -16,7 +16,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_SECRET_ACCESS_KEY }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: 3.9 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 93dff2c..ccfd2b2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ jobs: python-version: ["3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: From 3d5e2bc46b0aa18bfbef7b2942d44cab4eddb3fc Mon Sep 17 00:00:00 2001 From: Andy Wick Date: Wed, 27 Mar 2024 12:42:44 -0400 Subject: [PATCH 6/6] move to v5 setup-python to reduce warnings --- .github/workflows/manage_arkime.yml | 2 +- .github/workflows/tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manage_arkime.yml b/.github/workflows/manage_arkime.yml index 54229d0..bb01612 100644 --- a/.github/workflows/manage_arkime.yml +++ b/.github/workflows/manage_arkime.yml @@ -17,7 +17,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: 3.9 - name: Install dependencies diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ccfd2b2..07fc213 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,7 +17,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies