[API_PARSER][CISCO_MERAKI] Add support of security logs (#431)

### Added - **[API_PARSER][CISCO_MERAKI]** Add support for security logs ### Changed - **[API_PARSER][CISCO_MERAKI]** Improve logging and collector early termination --------- Co-authored-by: Nicolas Lançon <[email protected]> Co-authored-by: Théo Bertin <[email protected]>
VultureProject · Oct 23, 2024 · 9d81bd7 · 9d81bd7
1 parent 2945785
commit 9d81bd7
Show file tree

Hide file tree

Showing 5 changed files with 78 additions and 8 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [LOGOM] Add the ability to define the spooling directory for DA queues
 - [TENANTS] New additional_config field
 - [NETWORK] Automatically select the best interface for outgoing Backend/LogForwarders using system routes
+- [API_PARSER] [CISCO_MERAKI] Add support for security logs
 ### Changed
 - [DEPENDENCIES] Upgrade djongo and code for pymongo>=4
 - [FRONTEND] [GUI] Improve binding information for Filebeat, Redis and Kafka listeners
@@ -24,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [LOGOM] Allow to deactivate limitation of the size of DA queues on disk
 - [LOGOM] [ELASTICSEARCH] Limit available certificates to non-CA ones
 - [API_PARSER] [CYBEREASON] Refacto code and avoid API errors by getting bulks of 1000 logs
+- [API_PARSER] [CISCO_MERAKI] Improve logging and collector early termination
 ### Fixed
 - [NETWORK] Correctly assign a new/changed IPv6 IP to an existing interface
 - [FRONTEND] JSONField default value

diff --git a/vulture_os/services/frontend/form.py b/vulture_os/services/frontend/form.py
@@ -225,7 +225,7 @@ def __init__(self, *args, **kwargs):
                            "mdatp_api_tenant", "mdatp_api_appid", "mdatp_api_secret",
                            "cortex_xdr_host", "cortex_xdr_apikey_id", "cortex_xdr_apikey",
                            "cybereason_host", "cybereason_username", "cybereason_password",
-                           "cisco_meraki_apikey", 'proofpoint_tap_host', 'proofpoint_tap_endpoint', 'proofpoint_tap_principal',
+                           "cisco_meraki_apikey", "cisco_meraki_get_security_logs", 'proofpoint_tap_host', 'proofpoint_tap_endpoint', 'proofpoint_tap_principal',
                            "carbon_black_host", 'carbon_black_orgkey', 'carbon_black_apikey',
                            "netskope_host", 'netskope_apikey',
                            'rapid7_idr_host', 'rapid7_idr_apikey',
@@ -332,7 +332,7 @@ class Meta:
                   "mdatp_api_tenant", "mdatp_api_appid", "mdatp_api_secret",
                   "cortex_xdr_host", "cortex_xdr_apikey_id", "cortex_xdr_apikey",
                   "cybereason_host", "cybereason_username", "cybereason_password",
-                  "cisco_meraki_apikey", 'proofpoint_tap_host', 'proofpoint_tap_endpoint', 'proofpoint_tap_principal',
+                  "cisco_meraki_apikey", "cisco_meraki_get_security_logs", 'proofpoint_tap_host', 'proofpoint_tap_endpoint', 'proofpoint_tap_principal',
                   "proofpoint_tap_secret",
                   "sentinel_one_host", "sentinel_one_apikey", "sentinel_one_account_type",
                   "netskope_host", "netskope_apikey",
@@ -463,6 +463,7 @@ class Meta:
             'cybereason_username': TextInput(attrs={'class': 'form-control'}),
             'cybereason_password': TextInput(attrs={'type': "password", 'class': 'form-control'}),
             'cisco_meraki_apikey': TextInput(attrs={'class': 'form-control'}),
+            'cisco_meraki_get_security_logs': CheckboxInput(attrs={'class': 'js-switch'}),
             'proofpoint_tap_host': TextInput(attrs={'class': 'form-control'}),
             'proofpoint_tap_endpoint': Select(attrs={'class': 'form-control select2'}),
             'proofpoint_tap_principal': TextInput(attrs={'class': 'form-control'}),

diff --git a/vulture_os/services/frontend/models.py b/vulture_os/services/frontend/models.py
@@ -766,6 +766,11 @@ class Frontend(models.Model):
     cisco_meraki_timestamp = models.JSONField(
         default=dict
     )
+    cisco_meraki_get_security_logs = models.BooleanField(
+        default=False,
+        verbose_name=_("Get security logs"),
+        help_text=_("Get security logs"),
+    )
     # Proofpoint TAP attributes
     proofpoint_tap_host = models.TextField(
         help_text=_("ProofPoint TAP host"),

diff --git a/vulture_os/services/templates/services/frontend_edit.html b/vulture_os/services/templates/services/frontend_edit.html
@@ -1184,6 +1184,13 @@ <h4 class="panel-title"><i class="icon fa fa-ban"></i> {% translate "Form errors
                             {{ form.cisco_meraki_apikey.errors|safe }}
                           </div>
                         </div>
+                        <div class="form-group">
+                          <label class="col-sm-3 control-label">{{ form.cisco_meraki_get_security_logs.label }}</label>
+                          <div class="col-sm-5">
+                            {{ form.cisco_meraki_get_security_logs }}
+                            {{ form.cisco_meraki_get_security_logs.errors|safe }}
+                          </div>
+                        </div>
                       </div>
                       <div class="col-md-12 api_clients_row" id="api_cybereason_row">
                         <div class="form-group">

diff --git a/vulture_os/toolkit/api_parser/cisco_meraki/cisco_meraki.py b/vulture_os/toolkit/api_parser/cisco_meraki/cisco_meraki.py
@@ -48,6 +48,7 @@ def __init__(self, data):
         super().__init__(data)
 
         self.cisco_meraki_apikey = data["cisco_meraki_apikey"]
+        self.cisco_meraki_get_security_logs = data["cisco_meraki_get_security_logs"]
 
         self.session = None
 
@@ -80,13 +81,25 @@ def get_organization_networks(self, orga_id):
         except Exception as e:
             raise CiscoMerakiAPIError(e)
 
+    def get_organization_appliance_security_events(self, since, orga_id):
+        self._connect()
+        try:
+            return self.session.appliance.getOrganizationApplianceSecurityEvents(orga_id, t0=since,
+                                                                                 perPage=1000, total_pages='all')
+        except Exception as e:
+            raise CiscoMerakiAPIError(e)
+
     def test(self):
         try:
             orga = self.get_organizations()[0]
             logger.info(f"[{__parser__}]:test: Getting organisation {orga['name']} networks",
                         extra={'frontend': str(self.frontend)})
             # retreive organisation & networks
             data = self.get_organization_networks(orga['id'])
+
+            if self.cisco_meraki_get_security_logs:
+                since = (timezone.now()-timedelta(days=1)).isoformat()
+                data.extend(self.get_organization_appliance_security_events(since, orga['id']))
             return {
                 "status": True,
                 "data": data
@@ -113,23 +126,31 @@ def get_logs(self, network_id, product_type, since):
     def execute(self):
 
         for orga in self.get_organizations():
+            if self.evt_stop.is_set():
+                break
 
             logger.info(f"[{__parser__}]:execute: Getting organisation {orga['name']}", extra={'frontend': str(self.frontend)})
             for network in self.get_organization_networks(orga['id']):
+                if self.evt_stop.is_set():
+                    break
 
                 logger.info(f"[{__parser__}]:execute: Getting organisation network {network['name']}",
                             extra={'frontend': str(self.frontend)})
                 for product_type in network['productTypes']:
+                    if self.evt_stop.is_set():
+                        break
 
+                    since = self.frontend.cisco_meraki_timestamp.get(f"{network['id']}_{product_type}",
+                                                                     (timezone.now()-timedelta(days=1)).isoformat())
                     logger.info(f"[{__parser__}]:execute: Getting organisation network {network['name']} "
-                                f"product {product_type}",
+                                f"product {product_type}, from {since}",
                                 extra={'frontend': str(self.frontend)})
                     nb_events = 1
-                    while nb_events > 0:
+                    total_nb_events = 0
+                    while nb_events > 0 and not self.evt_stop.is_set():
                         status, tmp_logs = self.get_logs(network['id'],
                                                          product_type,
-                                                         self.frontend.cisco_meraki_timestamp.get(f"{network['id']}_{product_type}") or \
-                                                         (timezone.now()-timedelta(days=1)).isoformat())
+                                                         since)
 
                         if not status:
                             logger.error(f"[{__parser__}]:execute: "
@@ -143,22 +164,56 @@ def execute(self):
                         self.update_lock()
 
                         nb_events = len(tmp_logs['events'])
+                        total_nb_events += nb_events
                         logs = tmp_logs['events']
 
-                        def format_log(log):
+                        def format_network_log(log):
+                            log['log_type'] = "network"
                             log['organization_id'] = orga['id']
                             log['organization_name'] = orga['name']
                             log['network'] = network['name']
                             log['product'] = product_type
                             log['timestamp'] = log['occurredAt']
                             return json.dumps(log)
 
-                        self.write_to_file([format_log(l) for l in logs])
+                        self.write_to_file([format_network_log(log) for log in logs])
                         # Writting may take some while, so refresh token in Redis
                         self.update_lock()
 
                         if nb_events > 0:
                             # No need to make_aware, date already contains timezone
                             self.frontend.cisco_meraki_timestamp[f"{network['id']}_{product_type}"] = tmp_logs['pageEndAt']
+                            since = tmp_logs['pageEndAt']
+
+                    if total_nb_events:
+                        logger.info(f"[{__parser__}]:execute: Got {total_nb_events} logs for organisation {orga['name']}, "
+                                    f"network {network['name']}, "
+                                    f"product {product_type}",
+                                    extra={'frontend': str(self.frontend)})
+
+
+            if self.cisco_meraki_get_security_logs and not self.evt_stop.is_set():
+                logger.info(f"[{__parser__}]:execute: Getting organisation {orga['name']} security events", extra={'frontend': str(self.frontend)})
+
+                since = self.frontend.cisco_meraki_timestamp.get(f"org{orga['id']}_security_events", (timezone.now()-timedelta(days=1)).isoformat())
+                security_events = self.get_organization_appliance_security_events(since, orga['id'])
+                # Parsing 1k lines may take some while, so refresh token in Redis before
+                self.update_lock()
+
+                def format_security_log(log):
+                    log['log_type'] = "security"
+                    log['organization_id'] = orga['id']
+                    log['organization_name'] = orga['name']
+                    log['timestamp'] = log['ts']
+                    return json.dumps(log)
+
+                self.write_to_file([format_security_log(log) for log in security_events])
+                # Writting may take some while, so refresh token in Redis
+                self.update_lock()
+
+                if len(security_events) > 0:
+                    # No need to make_aware, date already contains timezone
+                    self.frontend.cisco_meraki_timestamp[f"org{orga['id']}_security_events"] = security_events[-1]['ts']
+
 
         logger.info(f"[{__parser__}]:execute: Parser ending", extra={'frontend': str(self.frontend)})