diff --git a/CHANGELOG b/CHANGELOG index 00738e4ce..b45c44ff2 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [LOGOM] Add the ability to define the spooling directory for DA queues - [TENANTS] New additional_config field - [NETWORK] Automatically select the best interface for outgoing Backend/LogForwarders using system routes +- [API_PARSER] [CISCO_MERAKI] Add support for security logs ### Changed - [DEPENDENCIES] Upgrade djongo and code for pymongo>=4 - [FRONTEND] [GUI] Improve binding information for Filebeat, Redis and Kafka listeners @@ -24,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [LOGOM] Allow to deactivate limitation of the size of DA queues on disk - [LOGOM] [ELASTICSEARCH] Limit available certificates to non-CA ones - [API_PARSER] [CYBEREASON] Refacto code and avoid API errors by getting bulks of 1000 logs +- [API_PARSER] [CISCO_MERAKI] Improve logging and collector early termination ### Fixed - [NETWORK] Correctly assign a new/changed IPv6 IP to an existing interface - [FRONTEND] JSONField default value diff --git a/vulture_os/services/frontend/form.py b/vulture_os/services/frontend/form.py index d33b0ee02..30f61c5d7 100644 --- a/vulture_os/services/frontend/form.py +++ b/vulture_os/services/frontend/form.py @@ -225,7 +225,7 @@ def __init__(self, *args, **kwargs): "mdatp_api_tenant", "mdatp_api_appid", "mdatp_api_secret", "cortex_xdr_host", "cortex_xdr_apikey_id", "cortex_xdr_apikey", "cybereason_host", "cybereason_username", "cybereason_password", - "cisco_meraki_apikey", 'proofpoint_tap_host', 'proofpoint_tap_endpoint', 'proofpoint_tap_principal', + "cisco_meraki_apikey", "cisco_meraki_get_security_logs", 'proofpoint_tap_host', 'proofpoint_tap_endpoint', 'proofpoint_tap_principal', "carbon_black_host", 'carbon_black_orgkey', 'carbon_black_apikey', "netskope_host", 'netskope_apikey', 'rapid7_idr_host', 'rapid7_idr_apikey', @@ -332,7 +332,7 @@ class Meta: "mdatp_api_tenant", "mdatp_api_appid", "mdatp_api_secret", "cortex_xdr_host", "cortex_xdr_apikey_id", "cortex_xdr_apikey", "cybereason_host", "cybereason_username", "cybereason_password", - "cisco_meraki_apikey", 'proofpoint_tap_host', 'proofpoint_tap_endpoint', 'proofpoint_tap_principal', + "cisco_meraki_apikey", "cisco_meraki_get_security_logs", 'proofpoint_tap_host', 'proofpoint_tap_endpoint', 'proofpoint_tap_principal', "proofpoint_tap_secret", "sentinel_one_host", "sentinel_one_apikey", "sentinel_one_account_type", "netskope_host", "netskope_apikey", @@ -463,6 +463,7 @@ class Meta: 'cybereason_username': TextInput(attrs={'class': 'form-control'}), 'cybereason_password': TextInput(attrs={'type': "password", 'class': 'form-control'}), 'cisco_meraki_apikey': TextInput(attrs={'class': 'form-control'}), + 'cisco_meraki_get_security_logs': CheckboxInput(attrs={'class': 'js-switch'}), 'proofpoint_tap_host': TextInput(attrs={'class': 'form-control'}), 'proofpoint_tap_endpoint': Select(attrs={'class': 'form-control select2'}), 'proofpoint_tap_principal': TextInput(attrs={'class': 'form-control'}), diff --git a/vulture_os/services/frontend/models.py b/vulture_os/services/frontend/models.py index 866912948..fc20150d4 100644 --- a/vulture_os/services/frontend/models.py +++ b/vulture_os/services/frontend/models.py @@ -766,6 +766,11 @@ class Frontend(models.Model): cisco_meraki_timestamp = models.JSONField( default=dict ) + cisco_meraki_get_security_logs = models.BooleanField( + default=False, + verbose_name=_("Get security logs"), + help_text=_("Get security logs"), + ) # Proofpoint TAP attributes proofpoint_tap_host = models.TextField( help_text=_("ProofPoint TAP host"), diff --git a/vulture_os/services/templates/services/frontend_edit.html b/vulture_os/services/templates/services/frontend_edit.html index d11e6d3fd..c0bca99be 100644 --- a/vulture_os/services/templates/services/frontend_edit.html +++ b/vulture_os/services/templates/services/frontend_edit.html @@ -1184,6 +1184,13 @@

{% translate "Form errors {{ form.cisco_meraki_apikey.errors|safe }} +
+ +
+ {{ form.cisco_meraki_get_security_logs }} + {{ form.cisco_meraki_get_security_logs.errors|safe }} +
+
diff --git a/vulture_os/toolkit/api_parser/cisco_meraki/cisco_meraki.py b/vulture_os/toolkit/api_parser/cisco_meraki/cisco_meraki.py index 290db38b1..f65d4ea59 100644 --- a/vulture_os/toolkit/api_parser/cisco_meraki/cisco_meraki.py +++ b/vulture_os/toolkit/api_parser/cisco_meraki/cisco_meraki.py @@ -48,6 +48,7 @@ def __init__(self, data): super().__init__(data) self.cisco_meraki_apikey = data["cisco_meraki_apikey"] + self.cisco_meraki_get_security_logs = data["cisco_meraki_get_security_logs"] self.session = None @@ -80,6 +81,14 @@ def get_organization_networks(self, orga_id): except Exception as e: raise CiscoMerakiAPIError(e) + def get_organization_appliance_security_events(self, since, orga_id): + self._connect() + try: + return self.session.appliance.getOrganizationApplianceSecurityEvents(orga_id, t0=since, + perPage=1000, total_pages='all') + except Exception as e: + raise CiscoMerakiAPIError(e) + def test(self): try: orga = self.get_organizations()[0] @@ -87,6 +96,10 @@ def test(self): extra={'frontend': str(self.frontend)}) # retreive organisation & networks data = self.get_organization_networks(orga['id']) + + if self.cisco_meraki_get_security_logs: + since = (timezone.now()-timedelta(days=1)).isoformat() + data.extend(self.get_organization_appliance_security_events(since, orga['id'])) return { "status": True, "data": data @@ -113,23 +126,31 @@ def get_logs(self, network_id, product_type, since): def execute(self): for orga in self.get_organizations(): + if self.evt_stop.is_set(): + break logger.info(f"[{__parser__}]:execute: Getting organisation {orga['name']}", extra={'frontend': str(self.frontend)}) for network in self.get_organization_networks(orga['id']): + if self.evt_stop.is_set(): + break logger.info(f"[{__parser__}]:execute: Getting organisation network {network['name']}", extra={'frontend': str(self.frontend)}) for product_type in network['productTypes']: + if self.evt_stop.is_set(): + break + since = self.frontend.cisco_meraki_timestamp.get(f"{network['id']}_{product_type}", + (timezone.now()-timedelta(days=1)).isoformat()) logger.info(f"[{__parser__}]:execute: Getting organisation network {network['name']} " - f"product {product_type}", + f"product {product_type}, from {since}", extra={'frontend': str(self.frontend)}) nb_events = 1 - while nb_events > 0: + total_nb_events = 0 + while nb_events > 0 and not self.evt_stop.is_set(): status, tmp_logs = self.get_logs(network['id'], product_type, - self.frontend.cisco_meraki_timestamp.get(f"{network['id']}_{product_type}") or \ - (timezone.now()-timedelta(days=1)).isoformat()) + since) if not status: logger.error(f"[{__parser__}]:execute: " @@ -143,9 +164,11 @@ def execute(self): self.update_lock() nb_events = len(tmp_logs['events']) + total_nb_events += nb_events logs = tmp_logs['events'] - def format_log(log): + def format_network_log(log): + log['log_type'] = "network" log['organization_id'] = orga['id'] log['organization_name'] = orga['name'] log['network'] = network['name'] @@ -153,12 +176,44 @@ def format_log(log): log['timestamp'] = log['occurredAt'] return json.dumps(log) - self.write_to_file([format_log(l) for l in logs]) + self.write_to_file([format_network_log(log) for log in logs]) # Writting may take some while, so refresh token in Redis self.update_lock() if nb_events > 0: # No need to make_aware, date already contains timezone self.frontend.cisco_meraki_timestamp[f"{network['id']}_{product_type}"] = tmp_logs['pageEndAt'] + since = tmp_logs['pageEndAt'] + + if total_nb_events: + logger.info(f"[{__parser__}]:execute: Got {total_nb_events} logs for organisation {orga['name']}, " + f"network {network['name']}, " + f"product {product_type}", + extra={'frontend': str(self.frontend)}) + + + if self.cisco_meraki_get_security_logs and not self.evt_stop.is_set(): + logger.info(f"[{__parser__}]:execute: Getting organisation {orga['name']} security events", extra={'frontend': str(self.frontend)}) + + since = self.frontend.cisco_meraki_timestamp.get(f"org{orga['id']}_security_events", (timezone.now()-timedelta(days=1)).isoformat()) + security_events = self.get_organization_appliance_security_events(since, orga['id']) + # Parsing 1k lines may take some while, so refresh token in Redis before + self.update_lock() + + def format_security_log(log): + log['log_type'] = "security" + log['organization_id'] = orga['id'] + log['organization_name'] = orga['name'] + log['timestamp'] = log['ts'] + return json.dumps(log) + + self.write_to_file([format_security_log(log) for log in security_events]) + # Writting may take some while, so refresh token in Redis + self.update_lock() + + if len(security_events) > 0: + # No need to make_aware, date already contains timezone + self.frontend.cisco_meraki_timestamp[f"org{orga['id']}_security_events"] = security_events[-1]['ts'] + logger.info(f"[{__parser__}]:execute: Parser ending", extra={'frontend': str(self.frontend)})