Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[API_PARSER][PROOFPOINT TRAP] Add timeout choice #487

Open
wants to merge 2 commits into
base: dev
Choose a base branch
from
Open

[API_PARSER][PROOFPOINT TRAP] Add timeout choice #487

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
20d8a4f
Add choice timeout for requests of proofpoint_trap
DBischoff0 Jan 28, 2025
b365277
change for review
DBischoff0 Jan 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions vulture_os/services/frontend/form.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ def __init__(self, *args, **kwargs):
'safenet_tenant_code', 'safenet_apikey',
'signalsciences_ngwaf_email', 'signalsciences_ngwaf_token', 'signalsciences_ngwaf_corp_name', 'signalsciences_ngwaf_site_name',
'proofpoint_casb_api_key','proofpoint_casb_client_id','proofpoint_casb_client_secret',
'proofpoint_trap_host', 'proofpoint_trap_apikey',
'proofpoint_trap_host', 'proofpoint_trap_apikey', 'proofpoint_trap_timeout',
'waf_cloud_protector_host', 'waf_cloud_protector_api_key_pub', 'waf_cloud_protector_api_key_priv',
'waf_cloud_protector_provider', 'waf_cloud_protector_tenant', 'waf_cloud_protector_servers',
'trendmicro_visionone_token',
Expand Down Expand Up @@ -364,7 +364,7 @@ class Meta:
'trendmicro_worryfree_server_port', 'safenet_tenant_code', 'safenet_apikey',
'signalsciences_ngwaf_email', 'signalsciences_ngwaf_token', 'signalsciences_ngwaf_corp_name', 'signalsciences_ngwaf_site_name',
'proofpoint_casb_api_key','proofpoint_casb_client_id','proofpoint_casb_client_secret',
'proofpoint_trap_host', 'proofpoint_trap_apikey',
'proofpoint_trap_host', 'proofpoint_trap_apikey', 'proofpoint_trap_timeout',
'waf_cloud_protector_host', 'waf_cloud_protector_api_key_pub', 'waf_cloud_protector_api_key_priv',
'waf_cloud_protector_provider', 'waf_cloud_protector_tenant', 'waf_cloud_protector_servers',
'trendmicro_visionone_token',
Expand Down Expand Up @@ -550,6 +550,7 @@ class Meta:
'proofpoint_casb_client_secret': TextInput(attrs={'type': 'password','class': 'form-control'}),
'proofpoint_trap_host': TextInput(attrs={'class': 'form-control'}),
'proofpoint_trap_apikey': TextInput(attrs={'class': 'form-control'}),
'proofpoint_trap_timeout': TextInput(attrs={'class': 'form-control'}),
'waf_cloud_protector_host': TextInput(attrs={'class': 'form-control'}),
'waf_cloud_protector_api_key_pub': Textarea(attrs={'class': 'form-control'}),
'waf_cloud_protector_api_key_priv': Textarea(attrs={'class': 'form-control'}),
Expand Down
5 changes: 5 additions & 0 deletions vulture_os/services/frontend/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1172,6 +1172,11 @@
help_text = _("ProofPoint TRAP API key"),
default = "",
)
proofpoint_trap_timeout = models.PositiveIntegerField(
default=20,
help_text=_("ProofPoint TRAP API timeout"),
verbose_name=_("ProofPoint TRAP API timeout")
)
# WAF Cloud Protector attributes
waf_cloud_protector_host = models.TextField(
verbose_name = _("WAF CloudProtector host"),
Expand Down Expand Up @@ -1491,7 +1496,7 @@
# Test self.pk to prevent M2M errors when object isn't saved in DB
if self.pk:
for listener in self.listener_set.all():
l = listener.to_template()

Check failure on line 1499 in vulture_os/services/frontend/models.py

View workflow job for this annotation

GitHub Actions / check

Ruff (E741) 

vulture_os/services/frontend/models.py:1499:21: E741 Ambiguous variable name: `l`
# Remove frontend to prevent infinite loop
del l['frontend']
result['listeners'].append(l)
Expand Down Expand Up @@ -1535,7 +1540,7 @@
for log_fwd in self.log_forwarders.all().only('id')]

if not fields or "api_parser_custom_certificate" in fields:
if result['api_parser_custom_certificate'] == None:

Check failure on line 1543 in vulture_os/services/frontend/models.py

View workflow job for this annotation

GitHub Actions / check

Ruff (E711) 

vulture_os/services/frontend/models.py:1543:59: E711 Comparison to `None` should be `cond is None`
result['api_parser_custom_certificate'] = {}

return result
Expand All @@ -1557,9 +1562,9 @@
listeners_list = ["Kafka:"] + self.kafka_brokers
else:
# Test self.pk to prevent M2M errors when object isn't saved in DB
listeners_list = [str(l) for l in self.listener_set.all().only(*Listener.str_attrs())] if self.pk else []

Check failure on line 1565 in vulture_os/services/frontend/models.py

View workflow job for this annotation

GitHub Actions / check

Ruff (E741) 

vulture_os/services/frontend/models.py:1565:42: E741 Ambiguous variable name: `l`

log_forwarders = [str(l) for l in self.log_forwarders.all()]

Check failure on line 1567 in vulture_os/services/frontend/models.py

View workflow job for this annotation

GitHub Actions / check

Ruff (E741) 

vulture_os/services/frontend/models.py:1567:38: E741 Ambiguous variable name: `l`

mode = "UNKNOWN"
for m in MODE_CHOICES:
Expand Down Expand Up @@ -2274,7 +2279,7 @@
if self.rsyslog_port == 10000:
try:
self.rsyslog_port = Listener.objects.latest('rsyslog_port').rsyslog_port + 1
except:

Check failure on line 2282 in vulture_os/services/frontend/models.py

View workflow job for this annotation

GitHub Actions / check

Ruff (E722) 

vulture_os/services/frontend/models.py:2282:13: E722 Do not use bare `except`
# Let port 1000 if no listener defined yet
pass
super().save(*args, **kwargs)
Expand Down
7 changes: 7 additions & 0 deletions vulture_os/services/templates/services/frontend_edit.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1799,6 +1799,13 @@ <h4 class="panel-title"><i class="icon fa fa-ban"></i> {% translate "Form errors
{{ form.proofpoint_trap_apikey.errors|safe }}
</div>
</div>
<div class="form-group">
<label class="col-sm-3 control-label">{{ form.proofpoint_trap_timeout.label }}</label>
<div class="col-sm-5">
{{ form.proofpoint_trap_timeout }}
{{ form.proofpoint_trap_timeout.errors|safe }}
</div>
</div>
</div>
<div class="col-md-12 api_clients_row" id="api_waf_cloud_protector_row">
<div class="form-group">
Expand Down
10 changes: 7 additions & 3 deletions vulture_os/toolkit/api_parser/proofpoint_trap/proofpoint_trap.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ def __init__(self, data):
self.proofpoint_trap_host = "https://" + self.proofpoint_trap_host

self.proofpoint_trap_apikey = data["proofpoint_trap_apikey"]
self.proofpoint_trap_timeout = int(data["proofpoint_trap_timeout"])

self.session = None

Expand All @@ -74,7 +75,10 @@ def _connect(self):
except Exception as err:
raise ProofpointTRAPAPIError(err)

def __execute_query(self, url, query=None, timeout=20):
def __execute_query(self, url, query=None):
if self.evt_stop.is_set():
return
self.update_lock()

self._connect()

Expand All @@ -84,7 +88,7 @@ def __execute_query(self, url, query=None, timeout=20):
response = self.session.get(
url,
params=query,
timeout=timeout,
timeout=self.proofpoint_trap_timeout,
proxies=self.proxies,
verify=self.api_parser_custom_certificate if self.api_parser_custom_certificate else self.api_parser_verify_ssl
)
Expand All @@ -94,7 +98,7 @@ def __execute_query(self, url, query=None, timeout=20):
logger.info(f"[{__parser__}]:execute: API Rate limit exceeded, waiting 10 seconds...",
extra={'frontend': str(self.frontend)})
time.sleep(10)
return self.__execute_query(url, query, timeout)
return self.__execute_query(url, query)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Mmmmh this recursive call makes me anxious, we don't need at all, could you please do it iteratively using a while for example ?
Please ensure to execute a self.update_lock() whenever you are entering the loop, to avoid multiple simultaneous executions, moreover you needs to set the following condition not self.event_stop.is_set() in your loop to avoid infinite loop blocking service shutdown/updates

elif response.status_code != 200:
raise ProofpointTRAPAPIError(
f"Error at Proofpoint TRAP API Call URL: {url} Code: {response.status_code} Content: {response.content}")
Expand Down