From c3834164a57a3a6d8f049d1d0bc358dbeae6c413 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Lan=C3=A7on?= Date: Thu, 23 Jan 2025 16:44:49 +0100 Subject: [PATCH 01/18] NEW parser beyondtrust_reportings --- vulture_os/services/frontend/form.py | 6 + vulture_os/services/frontend/models.py | 25 ++ .../templates/services/frontend_edit.html | 30 ++ .../beyondtrust_reportings/__init__.py | 0 .../beyondtrust_reportings.py | 312 ++++++++++++++++++ vulture_os/toolkit/api_parser/utils.py | 3 +- 6 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 vulture_os/toolkit/api_parser/beyondtrust_reportings/__init__.py create mode 100644 vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py diff --git a/vulture_os/services/frontend/form.py b/vulture_os/services/frontend/form.py index cdfe0195d..f2b5da6be 100644 --- a/vulture_os/services/frontend/form.py +++ b/vulture_os/services/frontend/form.py @@ -269,6 +269,7 @@ def __init__(self, *args, **kwargs): "cisco_umbrella_managed_org_get_dns", "cisco_umbrella_managed_org_get_proxy", "catonetworks_account_id", "catonetworks_api_key", "infoblox_threat_defense_host", "infoblox_threat_defense_token", + "beyondtrust_reportings_client_id", "beyondtrust_reportings_secret", "beyondtrust_reportings_host", "beyondtrust_reportings_get_support_session_logs", ]: self.fields[field_name].required = False @@ -384,6 +385,7 @@ class Meta: 'cisco_umbrella_managed_org_get_dns', 'cisco_umbrella_managed_org_get_proxy', 'catonetworks_api_key', 'catonetworks_account_id', "infoblox_threat_defense_host", "infoblox_threat_defense_token", + "beyondtrust_reportings_client_id", "beyondtrust_reportings_secret", "beyondtrust_reportings_host", "beyondtrust_reportings_get_support_session_logs", ) widgets = { @@ -594,6 +596,10 @@ class Meta: 'catonetworks_account_id': TextInput(attrs={'class': 'form-control'}), 'infoblox_threat_defense_host': TextInput(attrs={'class': 'form-control'}), 'infoblox_threat_defense_token': TextInput(attrs={'type': 'password', 'class': 'form-control'}), + 'beyondtrust_reportings_client_id': TextInput(attrs={'class': 'form-control'}), + 'beyondtrust_reportings_secret': TextInput(attrs={'type': 'password', 'class': 'form-control'}), + 'beyondtrust_reportings_host': TextInput(attrs={'class': 'form-control'}), + 'beyondtrust_reportings_get_support_session_logs': CheckboxInput(attrs={'class': 'js-switch'}), } def clean_name(self): diff --git a/vulture_os/services/frontend/models.py b/vulture_os/services/frontend/models.py index d9461c3bf..a0f67bd20 100644 --- a/vulture_os/services/frontend/models.py +++ b/vulture_os/services/frontend/models.py @@ -1467,6 +1467,31 @@ class Frontend(models.Model): help_text=_("Infoblox Threat Defense Token"), default="", ) + # Beyondtrust Reportings attributes + beyondtrust_reportings_client_id = models.TextField( + verbose_name=_("Beyondtrust Reportings client id"), + help_text=_("Beyondtrust Reportings client id"), + default="", + ) + beyondtrust_reportings_secret = models.TextField( + verbose_name=_("Beyondtrust Reportings secret"), + help_text=_("Beyondtrust Reportings secret"), + default="", + ) + beyondtrust_reportings_host = models.TextField( + verbose_name=_("Beyondtrust Reportings host"), + help_text=_("Beyondtrust Reportings host"), + default="", + ) + beyondtrust_reportings_api_token = models.JSONField( + default=dict + ) + beyondtrust_reportings_get_support_session_logs = models.BooleanField( + verbose_name=_("Retrieve Beyondtrust Reportings SupportSession logs"), + help_text=_("Retrieve Beyondtrust Reportings SupportSession logs"), + default=False, + ) + @staticmethod def str_attrs(): diff --git a/vulture_os/services/templates/services/frontend_edit.html b/vulture_os/services/templates/services/frontend_edit.html index 9583e53a1..a06510b0d 100644 --- a/vulture_os/services/templates/services/frontend_edit.html +++ b/vulture_os/services/templates/services/frontend_edit.html @@ -2132,6 +2132,36 @@

{% translate "Form errors +
+
+ +
+ {{ form.beyondtrust_reportings_client_id }} + {{ form.beyondtrust_reportings_client_id.errors|safe }} +
+
+
+ +
+ {{ form.beyondtrust_reportings_secret }} + {{ form.beyondtrust_reportings_secret.errors|safe }} +
+
+
+ +
+ {{ form.beyondtrust_reportings_host }} + {{ form.beyondtrust_reportings_host.errors|safe }} +
+
+
+ +
+ {{ form.beyondtrust_reportings_get_support_session_logs }} + {{ form.beyondtrust_reportings_get_support_session_logs.errors|safe }} +
+
+
+
+ +
+ {{ form.beyondtrust_reportings_get_team_logs }} + {{ form.beyondtrust_reportings_get_team_logs.errors|safe }} +
+
+
+ +
+ {{ form.beyondtrust_reportings_get_access_session_logs }} + {{ form.beyondtrust_reportings_get_access_session_logs.errors|safe }} +
+
+
+ +
+ {{ form.beyondtrust_reportings_get_vault_account_activity_logs }} + {{ form.beyondtrust_reportings_get_vault_account_activity_logs.errors|safe }} +
+
diff --git a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py index 85094f456..7fbf05377 100644 --- a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py +++ b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py @@ -20,8 +20,8 @@ __version__ = "4.0.0" __maintainer__ = "Vulture OS" __email__ = "contact@vultureproject.org" -__doc__ = 'Beyondtrust PRA API' -__parser__ = 'Beyondtrust PRA' +__doc__ = 'Beyondtrust Reportings API' +__parser__ = 'Beyondtrust Reportings' import json import logging @@ -58,7 +58,13 @@ def __init__(self, data): self.beyondtrust_reportings_secret = data["beyondtrust_reportings_secret"] self.beyondtrust_reportings_host = data["beyondtrust_reportings_host"] self.beyondtrust_reportings_api_token = data.get("beyondtrust_reportings_api_token", {}) - self.beyondtrust_reportings_get_support_session_logs = data.get("beyondtrust_reportings_get_support_session_logs", False) + + self.allowed_reporting_types = { + "Team": data.get("beyondtrust_reportings_get_team_logs", False), + "AccessSession": data.get("beyondtrust_reportings_get_access_session_logs", False), + "VaultAccountActivity": data.get("beyondtrust_reportings_get_vault_account_activity_logs", False), + "SupportSession": data.get("beyondtrust_reportings_get_support_session_logs", False), + } if not self.beyondtrust_reportings_host.startswith('https://') and not self.beyondtrust_reportings_host.startswith('http://'): self.beyondtrust_reportings_host = f"https://{self.beyondtrust_reportings_host}" @@ -147,19 +153,24 @@ def _execute_query(self, url, query=None, timeout=20): def test(self): # Get the last 12 hours since = timezone.now() - timedelta(hours=12) - try: - logs, _ = self.get_logs("reporting", "SupportSession", since) + err = "" + for requested_type in self.allowed_reporting_types.keys(): + try: + logs, _ = self.get_logs("reporting", requested_type, since) + + return { + "status": True, + "data": logs + } + except Exception as e: + err = e + continue - return { - "status": True, - "data": logs - } - except Exception as e: - logger.exception(f"{[__parser__]}:test: {str(e)}", extra={'frontend': str(self.frontend)}) - return { - "status": False, - "error": str(e) - } + logger.exception(f"{[__parser__]}:test: {str(err)}", extra={'frontend': str(self.frontend)}) + return { + "status": False, + "error": str(err) + } @staticmethod def format_team_logs(logs): @@ -232,7 +243,7 @@ def get_logs(self, resource: str, requested_type: str, since: datetime): if error := logs.get('error'): # if "report information matching your chosen criteria is available." in error.get('#text'): - if any(x in error.get('#text') for x in ["Aucune information de rapport de session correspondant à vos critères n’est disponible.", "report information matching your chosen criteria is available."]): + if any(x in error.get('#text') for x in ["correspondant à vos critères n’est disponible.", "matching your chosen criteria is available."]): return [], None msg = f"[{__parser__}]:get_logs: Error while getting '{requested_type}' logs : {error.get('#text')}" raise BeyondtrustReportingsAPIError(msg) @@ -272,10 +283,11 @@ def format_log(self, log, resource, requested_type): def execute(self): # Get reporting logs - reporting_types = ["Team", "AccessSession", "VaultAccountActivity"] + reporting_types = [r_type for r_type, value in self.allowed_reporting_types.items() if value] - if self.beyondtrust_reportings_get_support_session_logs: - reporting_types.append("SupportSession") + if not reporting_types: + msg = f"[{__parser__}]:execute: Reporting types list cannot be empty." + raise BeyondtrustReportingsAPIError(msg) for report_type in reporting_types: since = self.last_collected_timestamps.get(f"beyondtrust_reportings_{report_type}") or (timezone.now() - timedelta(days=30)) From c082ed78d1baea19a7ab91ff04106f5a1b88bbb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Lan=C3=A7on?= Date: Mon, 27 Jan 2025 12:38:39 +0100 Subject: [PATCH 03/18] Beyondtrust_Reportings:: Refacto --- .../beyondtrust_reportings.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py index 7fbf05377..2dc262105 100644 --- a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py +++ b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py @@ -23,11 +23,11 @@ __doc__ = 'Beyondtrust Reportings API' __parser__ = 'Beyondtrust Reportings' -import json +from json import dumps as json_dumps import logging import requests -import xmltodict +from xmltodict import parse as xmltodict_parse from datetime import datetime, timedelta from django.utils import timezone @@ -99,6 +99,9 @@ def get_token(self): verify=self.api_parser_custom_certificate or self.api_parser_verify_ssl) res.raise_for_status() res_json = res.json() + if any(k not in res_json.keys() for k in ('access_token', 'expires_in')): + raise BeyondtrustReportingsAPIError(f"Error on get_token(): response does not contain access_token or expires_in keys") + return res_json['access_token'], res_json['expires_in'] def _connect(self): @@ -111,7 +114,7 @@ def _connect(self): except Exception as err: raise BeyondtrustReportingsAPIError(f"Error on _connect(): {err}") - def _execute_query(self, url, query=None, timeout=20): + def _execute_query(self, url, query={}, timeout=20): msg = f"URL : {url} Query : {str(query)}" logger.debug(f"[{__parser__}] Request API : {msg}", extra={'frontend': str(self.frontend)}) @@ -134,17 +137,15 @@ def _execute_query(self, url, query=None, timeout=20): retry += 1 continue # Handle token expiration - if response.status_code == 401: + elif response.status_code == 401: logger.warning(f"[{__parser__}]:execute: 401 received, will try to re-authenticate...", extra={'frontend': str(self.frontend)}) self.session = None self.beyondtrust_reportings_api_token = None self.evt_stop.wait(5.0) retry += 1 - elif response.status_code != 200: - raise BeyondtrustReportingsAPIError( - f"Error at Beyondtrust PRA API Call URL: {url} Code: {response.status_code} Content: {response.content}") - else: + continue + elif response.status_code == 200: return response raise BeyondtrustReportingsAPIError( @@ -239,7 +240,7 @@ def get_logs(self, resource: str, requested_type: str, since: datetime): res = self._execute_query(url, query=parameters) - logs = xmltodict.parse(res.text) + logs = xmltodict_parse(res.text) if error := logs.get('error'): # if "report information matching your chosen criteria is available." in error.get('#text'): @@ -278,7 +279,7 @@ def format_log(self, log, resource, requested_type): log["resource"] = resource log["requested_type"] = requested_type cleaned_log = self.remove_hash_from_keys(log) - return json.dumps(cleaned_log) + return json_dumps(cleaned_log) def execute(self): From 42ccf05c0720ccf3fedc9dbd049ca2659d39577a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Lan=C3=A7on?= Date: Tue, 28 Jan 2025 10:32:08 +0100 Subject: [PATCH 04/18] BEYONDTRUST_REPORTINGS:: Add a deprecated mention in BEYONDTRUST_PRA collector --- .../toolkit/api_parser/beyondtrust_pra/beyondtrust_pra.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vulture_os/toolkit/api_parser/beyondtrust_pra/beyondtrust_pra.py b/vulture_os/toolkit/api_parser/beyondtrust_pra/beyondtrust_pra.py index e4e703030..64c340ff7 100644 --- a/vulture_os/toolkit/api_parser/beyondtrust_pra/beyondtrust_pra.py +++ b/vulture_os/toolkit/api_parser/beyondtrust_pra/beyondtrust_pra.py @@ -23,6 +23,8 @@ __doc__ = 'Beyondtrust PRA API' __parser__ = 'Beyondtrust PRA' +# WARNING : THIS COLLECTOR IS DEPRECATED. PLEASE DO NOT UPDATE IT AND USE BEYONDTRUST_REPORTINGS INSTEAD + import json import logging from copy import deepcopy From c4e2a3088de670f9516427929d73243807655483 Mon Sep 17 00:00:00 2001 From: nlancon <53610122+nlancon@users.noreply.github.com> Date: Thu, 30 Jan 2025 12:08:06 +0100 Subject: [PATCH 05/18] Update vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théo Bertin --- .../beyondtrust_reportings/beyondtrust_reportings.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py index 2dc262105..4b38fe643 100644 --- a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py +++ b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py @@ -66,9 +66,7 @@ def __init__(self, data): "SupportSession": data.get("beyondtrust_reportings_get_support_session_logs", False), } - if not self.beyondtrust_reportings_host.startswith('https://') and not self.beyondtrust_reportings_host.startswith('http://'): - self.beyondtrust_reportings_host = f"https://{self.beyondtrust_reportings_host}" - self.beyondtrust_reportings_host = self.beyondtrust_reportings_host.rstrip("/") + self.beyondtrust_reportings_host = "https://" + self.beyondtrust_reportings_host.split("://")[-1].rstrip("/") self.session = None From af279948208d0aa6786e8eae6927669576f42658 Mon Sep 17 00:00:00 2001 From: nlancon <53610122+nlancon@users.noreply.github.com> Date: Thu, 30 Jan 2025 12:08:17 +0100 Subject: [PATCH 06/18] Update vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théo Bertin --- .../api_parser/beyondtrust_reportings/beyondtrust_reportings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py index 4b38fe643..de34cbbf3 100644 --- a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py +++ b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py @@ -74,7 +74,6 @@ def save_access_token(self): try: if self.frontend: self.frontend.beyondtrust_reportings_api_token = self.beyondtrust_reportings_api_token - self.frontend.save() except Exception as e: raise BeyondtrustReportingsAPIError(f"Unable to save access token: {e}") From 0db46bdd5fd6545c3b0240f38a69551462dbdec3 Mon Sep 17 00:00:00 2001 From: nlancon <53610122+nlancon@users.noreply.github.com> Date: Thu, 30 Jan 2025 12:08:26 +0100 Subject: [PATCH 07/18] Update vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théo Bertin --- .../api_parser/beyondtrust_reportings/beyondtrust_reportings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py index de34cbbf3..88f160f69 100644 --- a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py +++ b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py @@ -97,7 +97,7 @@ def get_token(self): res.raise_for_status() res_json = res.json() if any(k not in res_json.keys() for k in ('access_token', 'expires_in')): - raise BeyondtrustReportingsAPIError(f"Error on get_token(): response does not contain access_token or expires_in keys") + raise BeyondtrustReportingsAPIError("Error while getting a new access token: response does not contain access_token or expires_in keys") return res_json['access_token'], res_json['expires_in'] From a7a43f4a064cbc852374bd1f2165b2934bcf25b7 Mon Sep 17 00:00:00 2001 From: nlancon <53610122+nlancon@users.noreply.github.com> Date: Thu, 30 Jan 2025 12:09:41 +0100 Subject: [PATCH 08/18] Update vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théo Bertin --- .../beyondtrust_reportings/beyondtrust_reportings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py index 88f160f69..81dd2f361 100644 --- a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py +++ b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py @@ -309,14 +309,14 @@ def execute(self): # Update timestamp +24 if since < now, otherwise +1 delta = 24 if since < timezone.now() - timedelta(hours=24) else 1 msg = f"No logs, advancing timestamp by {delta} hour(s)" - logger.debug(f"[{__parser__}]:execute: {msg}", extra={'frontend': str(self.frontend)}) + logger.info(f"[{__parser__}]:execute: {msg}", extra={'frontend': str(self.frontend)}) self.last_collected_timestamps[f"beyondtrust_reportings_{report_type}"] = since + timedelta(hours=delta) finally: - msg = "Current timestamp for {report_type} is {timestamp}".format( + msg = "New timestamp for {report_type} is {timestamp}".format( report_type=report_type, timestamp=self.last_collected_timestamps[f"beyondtrust_reportings_{report_type}"] ) - logger.debug(f"[{__parser__}]:execute: {msg}", extra={'frontend': str(self.frontend)}) + logger.info(f"[{__parser__}]:execute: {msg}", extra={'frontend': str(self.frontend)}) since = self.last_collected_timestamps[f"beyondtrust_reportings_{report_type}"] logger.info(f"[{__parser__}]:execute: Parsing done.", extra={'frontend': str(self.frontend)}) From 63942964133879558d044e4db9184d34a2b0aa05 Mon Sep 17 00:00:00 2001 From: nlancon <53610122+nlancon@users.noreply.github.com> Date: Thu, 30 Jan 2025 14:12:09 +0100 Subject: [PATCH 09/18] Update vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théo Bertin --- .../beyondtrust_reportings.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py index 81dd2f361..8951e7a6b 100644 --- a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py +++ b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py @@ -112,11 +112,11 @@ def _connect(self): raise BeyondtrustReportingsAPIError(f"Error on _connect(): {err}") def _execute_query(self, url, query={}, timeout=20): + def _execute_query(self, url, query={}, timeout=20, tries=1): msg = f"URL : {url} Query : {str(query)}" logger.debug(f"[{__parser__}] Request API : {msg}", extra={'frontend': str(self.frontend)}) - retry = 0 - while retry < 2 and not self.evt_stop.is_set(): + while tries > 0 and not self.evt_stop.is_set(): if self.session is None: self._connect() response = self.session.get( @@ -130,8 +130,9 @@ def _execute_query(self, url, query={}, timeout=20): if response.status_code == 429: logger.warning(f"[{__parser__}]:execute: API Rate limit exceeded, waiting 10 seconds...", extra={'frontend': str(self.frontend)}) - self.evt_stop.wait(10.0) - retry += 1 + tries -= 1 + if tries > 0: + self.evt_stop.wait(10.0) continue # Handle token expiration elif response.status_code == 401: @@ -139,14 +140,14 @@ def _execute_query(self, url, query={}, timeout=20): extra={'frontend': str(self.frontend)}) self.session = None self.beyondtrust_reportings_api_token = None - self.evt_stop.wait(5.0) - retry += 1 + tries -= 1 + if tries > 0: + self.evt_stop.wait(5.0) continue elif response.status_code == 200: return response - raise BeyondtrustReportingsAPIError( - f"Error at Beyondtrust PRA API Call URL: {url} Code: {response.status_code} Content: {response.content}") + raise BeyondtrustReportingsAPIError("Maximum number of tries reached") def test(self): # Get the last 12 hours From 20501a481ee18b75ca413142460ea9f7512adac7 Mon Sep 17 00:00:00 2001 From: nlancon <53610122+nlancon@users.noreply.github.com> Date: Thu, 30 Jan 2025 14:12:18 +0100 Subject: [PATCH 10/18] Update vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théo Bertin --- .../api_parser/beyondtrust_reportings/beyondtrust_reportings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py index 8951e7a6b..1c9dc0722 100644 --- a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py +++ b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py @@ -224,7 +224,7 @@ def format_support_session_logs(logs): log["@timestamp"] = log["start_time"]["@timestamp"] return formated_logs - def get_logs(self, resource: str, requested_type: str, since: datetime): + def get_logs(self, resource: str, requested_type: str, since: datetime, tries: int = 1): url = f"{self.beyondtrust_reportings_host}/api/{resource}" parameters = { From a1d09bcdda95ab4f5455a141d3b8ec85262feba1 Mon Sep 17 00:00:00 2001 From: nlancon <53610122+nlancon@users.noreply.github.com> Date: Thu, 30 Jan 2025 14:12:26 +0100 Subject: [PATCH 11/18] Update vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théo Bertin --- .../api_parser/beyondtrust_reportings/beyondtrust_reportings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py index 1c9dc0722..157f7e6c5 100644 --- a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py +++ b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py @@ -297,7 +297,7 @@ def execute(self): while since < timezone.now() - timedelta(hours=1) and not self.evt_stop.is_set(): logs, last_datetime = self.get_logs("reporting", report_type, since) - + logs, last_datetime = self.get_logs("reporting", report_type, since, tries=2) if logs: self.write_to_file([self.format_log(log, "reporting", report_type) for log in logs]) # Downloading may take some while, so refresh token in Redis From 97045ee4688f43e0c23f8696c70597e977ab25ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Lan=C3=A7on?= Date: Thu, 30 Jan 2025 14:57:16 +0100 Subject: [PATCH 12/18] BEYONDTRUST_REPORTINGS:: Refacto and add more tests --- .../beyondtrust_reportings.py | 113 ++++++++++-------- 1 file changed, 66 insertions(+), 47 deletions(-) diff --git a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py index 157f7e6c5..dbf53c907 100644 --- a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py +++ b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py @@ -27,6 +27,7 @@ import logging import requests +from requests.exceptions import JSONDecodeError from xmltodict import parse as xmltodict_parse from datetime import datetime, timedelta @@ -95,11 +96,15 @@ def get_token(self): proxies=self.proxies, verify=self.api_parser_custom_certificate or self.api_parser_verify_ssl) res.raise_for_status() - res_json = res.json() - if any(k not in res_json.keys() for k in ('access_token', 'expires_in')): - raise BeyondtrustReportingsAPIError("Error while getting a new access token: response does not contain access_token or expires_in keys") + try: + res_json = res.json() + except JSONDecodeError as err: + raise BeyondtrustReportingsAPIError(f"Error on getting token: {err}") + else: + if any(k not in res_json.keys() for k in ('access_token', 'expires_in')): + raise BeyondtrustReportingsAPIError("Error while getting a new access token: response does not contain access_token or expires_in keys") - return res_json['access_token'], res_json['expires_in'] + return res_json['access_token'], res_json['expires_in'] def _connect(self): try: @@ -111,7 +116,6 @@ def _connect(self): except Exception as err: raise BeyondtrustReportingsAPIError(f"Error on _connect(): {err}") - def _execute_query(self, url, query={}, timeout=20): def _execute_query(self, url, query={}, timeout=20, tries=1): msg = f"URL : {url} Query : {str(query)}" logger.debug(f"[{__parser__}] Request API : {msg}", extra={'frontend': str(self.frontend)}) @@ -171,58 +175,73 @@ def test(self): "error": str(err) } - @staticmethod - def format_team_logs(logs): + def format_team_logs(self, logs): formated_logs = [] - logs = logs['team_activity_list'].get('team_activity', []) - if not isinstance(logs, list): - logs = [logs] - - for log in logs: - events = log['events']['event'] - if not isinstance(events, list): - events = [events] - - for event in events: - event['team'] = { - "name": log['@name'], - "id": log['@id'], - } - formated_logs.append(event) + try: + logs = logs['team_activity_list'].get('team_activity', []) + if not isinstance(logs, list): + logs = [logs] + + for log in logs: + events = log['events']['event'] + if not isinstance(events, list): + events = [events] + + for event in events: + event['team'] = { + "name": log['@name'], + "id": log['@id'], + } + formated_logs.append(event) + except KeyError as e: + logger.info(f"[{__parser__}] An error occurred while formating Team logs: {e}", + extra={'frontend': str(self.frontend)}) return formated_logs - @staticmethod - def format_access_session_logs(logs): + def format_access_session_logs(self, logs): formated_logs = [] - sessions_logs = logs['session_list'].get('session', []) - if not isinstance(sessions_logs, list): - sessions_logs = [sessions_logs] - - for log in sessions_logs: - del log['session_details'] # Avoid too long session log - log["@timestamp"] = log["start_time"]["@timestamp"] - formated_logs.append(log) - return formated_logs + try: + sessions_logs = logs['session_list'].get('session', []) + if not isinstance(sessions_logs, list): + sessions_logs = [sessions_logs] + + for log in sessions_logs: + del log['session_details'] # Avoid too long session log + log["@timestamp"] = log["start_time"]["@timestamp"] + formated_logs.append(log) + except KeyError as e: + logger.info(f"[{__parser__}] An error occurred while formating AccessSession logs: {e}", + extra={'frontend': str(self.frontend)}) - @staticmethod - def format_vault_account_activity_list_logs(logs): - formated_logs = logs['vault_account_activity_list'].get('vault_account_activity', []) - if not isinstance(formated_logs, list): - formated_logs = [formated_logs] return formated_logs - @staticmethod - def format_support_session_logs(logs): - formated_logs = logs.get('session_list', {}).get('session', []) + def format_vault_account_activity_list_logs(self, logs): + try: + formated_logs = logs['vault_account_activity_list'].get('vault_account_activity', []) + if not isinstance(formated_logs, list): + formated_logs = [formated_logs] + except KeyError as e: + logger.info(f"[{__parser__}] An error occurred while formating AccountActivityList logs: {e}", + extra={'frontend': str(self.frontend)}) + else: + return formated_logs + + def format_support_session_logs(self, logs): + try: + formated_logs = logs.get('session_list', {}).get('session', []) - if isinstance(formated_logs, dict): - formated_logs = [formated_logs] + if isinstance(formated_logs, dict): + formated_logs = [formated_logs] - for log in formated_logs: - del log['session_details'] # Avoid too long session log - log["@timestamp"] = log["start_time"]["@timestamp"] - return formated_logs + for log in formated_logs: + del log['session_details'] # Avoid too long session log + log["@timestamp"] = log["start_time"]["@timestamp"] + except KeyError as e: + logger.info(f"[{__parser__}] An error occurred while formating AccountActivityList logs: {e}", + extra={'frontend': str(self.frontend)}) + else: + return formated_logs def get_logs(self, resource: str, requested_type: str, since: datetime, tries: int = 1): url = f"{self.beyondtrust_reportings_host}/api/{resource}" From 0232a789dea19484efa35718fad449fc052e6218 Mon Sep 17 00:00:00 2001 From: nlancon <53610122+nlancon@users.noreply.github.com> Date: Thu, 30 Jan 2025 15:29:45 +0100 Subject: [PATCH 13/18] Update vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théo Bertin --- .../api_parser/beyondtrust_reportings/beyondtrust_reportings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py index dbf53c907..4b4c4d0a6 100644 --- a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py +++ b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py @@ -256,7 +256,7 @@ def get_logs(self, resource: str, requested_type: str, since: datetime, tries: i logger.info(f"[{__parser__}] get_logs: {msg}", extra={'frontend': str(self.frontend)}) res = self._execute_query(url, query=parameters) - + res = self._execute_query(url, query=parameters, tries=tries) logs = xmltodict_parse(res.text) if error := logs.get('error'): From fdd04525618ae4cb3ec8363185959e3afcc3a94f Mon Sep 17 00:00:00 2001 From: nlancon <53610122+nlancon@users.noreply.github.com> Date: Thu, 30 Jan 2025 15:30:19 +0100 Subject: [PATCH 14/18] Update vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théo Bertin --- .../api_parser/beyondtrust_reportings/beyondtrust_reportings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py index 4b4c4d0a6..15c774e6a 100644 --- a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py +++ b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py @@ -315,7 +315,6 @@ def execute(self): while since < timezone.now() - timedelta(hours=1) and not self.evt_stop.is_set(): - logs, last_datetime = self.get_logs("reporting", report_type, since) logs, last_datetime = self.get_logs("reporting", report_type, since, tries=2) if logs: self.write_to_file([self.format_log(log, "reporting", report_type) for log in logs]) From 534b66f6dd17949c60a3bff12b1096732911f1bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Lan=C3=A7on?= Date: Fri, 31 Jan 2025 15:11:14 +0100 Subject: [PATCH 15/18] BEYONDTRUST_REPORTINGS:: Update while loop condition --- .../api_parser/beyondtrust_reportings/beyondtrust_reportings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py index 15c774e6a..92d15bf51 100644 --- a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py +++ b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py @@ -313,7 +313,7 @@ def execute(self): msg = f"Parser starting to get {report_type} logs from {since}" logger.info(f"[{__parser__}]:execute: {msg}", extra={'frontend': str(self.frontend)}) - while since < timezone.now() - timedelta(hours=1) and not self.evt_stop.is_set(): + while since < timezone.now() and not self.evt_stop.is_set(): logs, last_datetime = self.get_logs("reporting", report_type, since, tries=2) if logs: From 8042f5a28e50cf90e2471da3fb5b5aec1156d872 Mon Sep 17 00:00:00 2001 From: nlancon <53610122+nlancon@users.noreply.github.com> Date: Fri, 31 Jan 2025 15:34:13 +0100 Subject: [PATCH 16/18] Update vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théo Bertin --- .../beyondtrust_reportings/beyondtrust_reportings.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py index 92d15bf51..2a9feeb70 100644 --- a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py +++ b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py @@ -313,7 +313,7 @@ def execute(self): msg = f"Parser starting to get {report_type} logs from {since}" logger.info(f"[{__parser__}]:execute: {msg}", extra={'frontend': str(self.frontend)}) - while since < timezone.now() and not self.evt_stop.is_set(): + while not self.evt_stop.is_set(): logs, last_datetime = self.get_logs("reporting", report_type, since, tries=2) if logs: @@ -338,4 +338,7 @@ def execute(self): logger.info(f"[{__parser__}]:execute: {msg}", extra={'frontend': str(self.frontend)}) since = self.last_collected_timestamps[f"beyondtrust_reportings_{report_type}"] + if not logs: + break + logger.info(f"[{__parser__}]:execute: Parsing done.", extra={'frontend': str(self.frontend)}) From d24dba12d9a966e6a8af42e9675cc986f723161d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Bertin?= Date: Fri, 31 Jan 2025 09:36:17 +0100 Subject: [PATCH 17/18] clean(beyondtrust_reportings): Change some log levels --- .../beyondtrust_reportings/beyondtrust_reportings.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py index 2a9feeb70..618a6b733 100644 --- a/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py +++ b/vulture_os/toolkit/api_parser/beyondtrust_reportings/beyondtrust_reportings.py @@ -194,7 +194,7 @@ def format_team_logs(self, logs): } formated_logs.append(event) except KeyError as e: - logger.info(f"[{__parser__}] An error occurred while formating Team logs: {e}", + logger.error(f"[{__parser__}] An error occurred while formating Team logs: {e}", extra={'frontend': str(self.frontend)}) return formated_logs @@ -211,7 +211,7 @@ def format_access_session_logs(self, logs): log["@timestamp"] = log["start_time"]["@timestamp"] formated_logs.append(log) except KeyError as e: - logger.info(f"[{__parser__}] An error occurred while formating AccessSession logs: {e}", + logger.error(f"[{__parser__}] An error occurred while formating AccessSession logs: {e}", extra={'frontend': str(self.frontend)}) return formated_logs @@ -222,7 +222,7 @@ def format_vault_account_activity_list_logs(self, logs): if not isinstance(formated_logs, list): formated_logs = [formated_logs] except KeyError as e: - logger.info(f"[{__parser__}] An error occurred while formating AccountActivityList logs: {e}", + logger.error(f"[{__parser__}] An error occurred while formating AccountActivityList logs: {e}", extra={'frontend': str(self.frontend)}) else: return formated_logs @@ -238,7 +238,7 @@ def format_support_session_logs(self, logs): del log['session_details'] # Avoid too long session log log["@timestamp"] = log["start_time"]["@timestamp"] except KeyError as e: - logger.info(f"[{__parser__}] An error occurred while formating AccountActivityList logs: {e}", + logger.error(f"[{__parser__}] An error occurred while formating AccountActivityList logs: {e}", extra={'frontend': str(self.frontend)}) else: return formated_logs @@ -255,7 +255,6 @@ def get_logs(self, resource: str, requested_type: str, since: datetime, tries: i msg = f"Getting {self.DURATION} seconds of {requested_type} logs, starting from {since}" logger.info(f"[{__parser__}] get_logs: {msg}", extra={'frontend': str(self.frontend)}) - res = self._execute_query(url, query=parameters) res = self._execute_query(url, query=parameters, tries=tries) logs = xmltodict_parse(res.text) From 03e1f1ba6107dcb5933f37ef72e92ff6f084062a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Bertin?= Date: Fri, 31 Jan 2025 16:52:35 +0100 Subject: [PATCH 18/18] chore(CHANGELOG): Add entry for new Beyondtrust Reportings collector --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 9caa02c14..5f52e3097 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - [API_PARSER] [CATONETWORKS] New collector - [API_PARSER] [INFOBLOX_THREAT_DEFENSE] New collector +- [API_PARSER] [BEYONDTRUST_REPORTINGS] New collector ## [2.19.0] - 2025-01-16 ### Added