Merge branch 'main' into fix/check-org-existence-on-migration

.env-prod

@@ -21,7 +21,7 @@ SCHEDULER_API=http://scheduler:8000
 KEIKO_API=http://keiko:8000
 KATALOGUS_API=http://katalogus:8000
 XTDB_URI=http://crux:3000
-BOEFJE_API=http://boefje:8000
+BOEFJES_API=http://boefje:8000
 
 # Bytes uses JWT for authentication
 BYTES_API=http://bytes:8000

boefjes/boefjes/app.py

@@ -95,7 +95,7 @@ def _fill_queue(self, task_queue: Queue, queue_type: WorkerManager.Queue):
 
         # We do not target a specific queue since we start one runtime for all organisations
         # and queue ids contain the organisation_id
-        queues = [q for q in queues if q.id.startswith(queue_type.value)]
+        queues = [q for q in queues if q.id.startswith(queue_type.value) and q.size > 0]
 
         logger.debug("Found queues: %s", [queue.id for queue in queues])
 

boefjes/boefjes/job_handler.py

@@ -2,6 +2,7 @@
 import traceback
 from collections.abc import Callable
 from datetime import datetime, timezone
+from functools import cache
 from typing import cast
 
 import httpx
@@ -36,6 +37,25 @@ def get_octopoes_api_connector(org_code: str) -> OctopoesAPIConnector:
     return OctopoesAPIConnector(str(settings.octopoes_api), org_code, timeout=settings.outgoing_request_timeout)
 
 
+@cache
+def boefje_env_variables() -> dict:
+    """
+    Return all environment variables that start with BOEFJE_. The returned
+    keys have the BOEFJE_ prefix removed.
+    """
+
+    boefje_variables = {}
+    for key, value in os.environ.items():
+        if key.startswith("BOEFJE_"):
+            boefje_variables[key.removeprefix("BOEFJE_")] = value
+
+    return boefje_variables
+
+
+def get_system_env_settings_for_boefje(allowed_keys: list[str]) -> dict:
+    return {key: value for key, value in boefje_env_variables().items() if key in allowed_keys}
+
+
 def get_environment_settings(boefje_meta: BoefjeMeta, schema: dict | None = None) -> dict[str, str]:
     try:
         katalogus_api = str(settings.katalogus_api).rstrip("/")
@@ -49,11 +69,7 @@ def get_environment_settings(boefje_meta: BoefjeMeta, schema: dict | None = None
         raise
 
     allowed_keys = schema.get("properties", []) if schema else []
-    new_env = {
-        key.split("BOEFJE_", 1)[1]: value
-        for key, value in os.environ.items()
-        if key.startswith("BOEFJE_") and key in allowed_keys
-    }
+    new_env = get_system_env_settings_for_boefje(allowed_keys)
 
     settings_from_katalogus = response.json()
 

boefjes/boefjes/plugins/kat_manual/single_ooi/normalizer.json

@@ -1,6 +1,7 @@
 {
   "id": "kat_manual_ooi",
   "name": "Manual OOI normalizer",
+  "description": "Parses manually added objects.",
   "consumes": [
     "manual/ooi"
   ],

boefjes/boefjes/plugins/kat_shodan_internetdb/boefje.json

@@ -1,6 +1,6 @@
 {
   "id": "shodan_internetdb",
-  "name": "'Shodan InternetDB",
+  "name": "Shodan InternetDB",
   "description": "Use Shodan InternetDB to find open ports with vulnerabilities that are found on an IP.",
   "consumes": [
     "IPAddressV4",

boefjes/boefjes/plugins/kat_shodan_internetdb/normalizer.json

@@ -1,6 +1,7 @@
 {
   "id": "kat_shodan_internetdb_normalize",
   "name": "Shodan InternetDB normalizer",
+  "description": "Parses Shodan InternetDB into findings.",
   "consumes": [
     "boefje/shodan_internetdb"
   ],

boefjes/boefjes/plugins/pdio_subfinder/normalizer.json

@@ -1,6 +1,7 @@
 {
   "id": "pdio-subfinder-normalizer",
   "name": "PDIO subfinder",
+  "description": "Parses ProjectDiscovery subfinder data for finding subdomains.",
   "consumes": [
     "boefje/pdio-subfinder"
   ],

boefjes/pyproject.toml

@@ -96,7 +96,7 @@ addopts = "-m 'not slow'"
 env = [
     "D:KATALOGUS_DB_URI=postgresql://postgres:postgres@ci_katalogus-db:5432/ci_katalogus",
     "D:QUEUE_URI=amqp://placeholder",
-    "D:BOEFJE_API=http://placeholder:8006",
+    "D:BOEFJES_API=http://placeholder:8006",
     "D:KATALOGUS_API=http://placeholder:8000",
     "D:OCTOPOES_API=http://placeholder:8001",
     "D:SCHEDULER_API=http://placeholder:8002",

boefjes/tests/test_job_handler.py

@@ -0,0 +1,29 @@
+from boefjes.job_handler import boefje_env_variables, get_system_env_settings_for_boefje
+
+
+def test_boefje_systems_vars(monkeypatch):
+    boefje_env_variables.cache_clear()
+
+    monkeypatch.setenv("BOEFJE_TEST1", "Test")
+
+    env = get_system_env_settings_for_boefje(["TEST1", "TEST2"])
+
+    assert env == {"TEST1": "Test"}
+
+
+def test_boefje_system_vars_no_vars():
+    boefje_env_variables.cache_clear()
+
+    env = get_system_env_settings_for_boefje(["TEST1", "TEST2"])
+
+    assert env == {}
+
+
+def test_boefje_systems_vars_no_allowed_keys(monkeypatch):
+    boefje_env_variables.cache_clear()
+
+    monkeypatch.setenv("BOEFJE_TEST1", "Test")
+
+    env = get_system_env_settings_for_boefje([])
+
+    assert env == {}

mula/.ci/.env.test

@@ -8,7 +8,7 @@ KATALOGUS_API=http://ci_katalogus:8000
 OCTOPOES_API=http://ci_octopoes:80
 BYTES_API=http://ci_bytes:8000
 SCHEDULER_API=http://ci_scheduler:8000
-BOEFJE_API=http://boefje:8000
+BOEFJES_API=http://boefje:8000
 
 SCHEDULER_API_HOST="0.0.0.0"
 SCHEDULER_API_PORT="8000"

octopoes/octopoes/connector/katalogus.py

@@ -1,7 +1,7 @@
 import httpx
 
 
-class KATalogusClientV1:
+class KATalogusClient:
     def __init__(self, base_uri: str):
         self.base_uri = f"{base_uri.rstrip('/')}/v1"
 

octopoes/octopoes/core/service.py

@@ -158,7 +158,7 @@ def _delete_ooi(self, reference: Reference, valid_time: datetime) -> None:
                 or (origin.origin_type == OriginType.INFERENCE and origin.source == reference)
             )
         ):
-            self.ooi_repository.delete(reference, valid_time)
+            self.ooi_repository.delete_if_exists(reference, valid_time)
 
     def save_origin(
         self, origin: Origin, oois: list[OOI], valid_time: datetime, end_valid_time: datetime | None = None
@@ -188,7 +188,7 @@ def save_origin(
             if not (other_origin.origin_type == OriginType.INFERENCE and [other_origin.source] == other_origin.result)
         ):
             logger.debug("Affirmation source %s seems dangling, deleting", origin.source)
-            self.ooi_repository.delete(origin.source, valid_time)
+            self.ooi_repository.delete_if_exists(origin.source, valid_time)
             return
 
         for ooi in oois:

octopoes/octopoes/repositories/ooi_repository.py

@@ -116,6 +116,9 @@ def list_neighbours(self, references: set[Reference], paths: set[Path], valid_ti
     def save(self, ooi: OOI, valid_time: datetime, end_valid_time: datetime | None = None) -> None:
         raise NotImplementedError
 
+    def delete_if_exists(self, reference: Reference, valid_time: datetime) -> None:
+        raise NotImplementedError
+
     def delete(self, reference: Reference, valid_time: datetime) -> None:
         raise NotImplementedError
 
@@ -613,13 +616,15 @@ def save(self, ooi: OOI, valid_time: datetime, end_valid_time: datetime | None =
         # After transaction, send event
         self.session.listen_post_commit(lambda: self.event_manager.publish(event))
 
-    def delete(self, reference: Reference, valid_time: datetime) -> None:
-        # retrieve old ooi
+    def delete_if_exists(self, reference: Reference, valid_time: datetime) -> None:
         try:
-            ooi = self.get(reference, valid_time=valid_time)
+            self.delete(reference, valid_time)
         except ObjectNotFoundException:
             return
 
+    def delete(self, reference: Reference, valid_time: datetime) -> None:
+        ooi = self.get(reference, valid_time=valid_time)
+
         self.session.add((XTDBOperationType.DELETE, str(reference), valid_time))
 
         event = OOIDBEvent(

octopoes/octopoes/tasks/tasks.py

@@ -12,7 +12,7 @@
 from pydantic import TypeAdapter
 
 from octopoes.config.settings import QUEUE_NAME_OCTOPOES, Settings
-from octopoes.connector.katalogus import KATalogusClientV1
+from octopoes.connector.katalogus import KATalogusClient
 from octopoes.core.app import bootstrap_octopoes, close_rabbit_channel, get_xtdb_client
 from octopoes.events.events import DBEvent, DBEventType
 from octopoes.events.manager import get_rabbit_channel
@@ -80,7 +80,7 @@ def handle_event(event: dict):
 @app.task(queue=QUEUE_NAME_OCTOPOES)
 def schedule_scan_profile_recalculations():
     try:
-        orgs = KATalogusClientV1(str(settings.katalogus_api)).get_organisations()
+        orgs = KATalogusClient(str(settings.katalogus_api)).get_organisations()
     except HTTPError:
         logger.exception("Failed getting organizations")
         raise

octopoes/tests/integration/test_api_connector.py

@@ -8,6 +8,7 @@
 from octopoes.api.models import Declaration, Observation
 from octopoes.connector.octopoes import OctopoesAPIConnector
 from octopoes.models import OOI, DeclaredScanProfile, Reference, ScanLevel
+from octopoes.models.exception import ObjectNotFoundException
 from octopoes.models.ooi.dns.records import DNSAAAARecord, DNSARecord, DNSMXRecord, DNSNSRecord
 from octopoes.models.ooi.dns.zone import Hostname
 from octopoes.models.ooi.findings import Finding, KATFindingType, RiskLevelSeverity
@@ -65,6 +66,9 @@ def test_bulk_operations(octopoes_api_connector: OctopoesAPIConnector, valid_tim
     )
     assert octopoes_api_connector.list_objects(types={Network, Hostname}, valid_time=valid_time).count == 6
 
+    with pytest.raises(ObjectNotFoundException):
+        octopoes_api_connector.delete_many(["bla"], valid_time=valid_time)
+
 
 def test_history(octopoes_api_connector: OctopoesAPIConnector):
     network = Network(name="test")

octopoes/tests/integration/test_ooi_deletion.py

@@ -95,7 +95,7 @@ def test_events_created_through_crud(xtdb_octopoes_service: OctopoesService, eve
     assert call2.new_data == origin
     assert call2.operation_type.value == "create"
 
-    xtdb_octopoes_service.ooi_repository.delete(network.reference, valid_time)
+    xtdb_octopoes_service.ooi_repository.delete_if_exists(network.reference, valid_time)
     xtdb_octopoes_service.commit()
 
     assert len(event_manager.queue) == 3  # Origin will be deleted by the worker due to the OOI delete event
@@ -118,7 +118,7 @@ def test_events_created_in_worker_during_handling(
     )
     xtdb_octopoes_service.save_origin(origin, [network], valid_time)
     xtdb_octopoes_service.commit()
-    xtdb_octopoes_service.ooi_repository.delete(network.reference, valid_time)
+    xtdb_octopoes_service.ooi_repository.delete_if_exists(network.reference, valid_time)
     xtdb_octopoes_service.commit()
 
     assert len(event_manager.queue) == 3
@@ -169,8 +169,8 @@ def test_events_deletion_after_bits(xtdb_octopoes_service: OctopoesService, even
     printer("ORIGINS", xtdb_octopoes_service.origin_repository.list_origins(valid_time))
     printer("EVENTS", event_manager.queue)
 
-    xtdb_octopoes_service.ooi_repository.delete(network.reference, valid_time)
-    xtdb_octopoes_service.ooi_repository.delete(hostname.reference, valid_time)
+    xtdb_octopoes_service.ooi_repository.delete_if_exists(network.reference, valid_time)
+    xtdb_octopoes_service.ooi_repository.delete_if_exists(hostname.reference, valid_time)
 
     print(3)
     print(f"PROCESSED {event_manager.complete_process_events(xtdb_octopoes_service)}")
@@ -393,7 +393,7 @@ def chain(source, results):
 
     count = xtdb_octopoes_service.ooi_repository.list_oois({OOI}, valid_time).count
 
-    xtdb_octopoes_service.ooi_repository.delete(ip[0].reference, valid_time)
+    xtdb_octopoes_service.ooi_repository.delete_if_exists(ip[0].reference, valid_time)
     event_manager.complete_process_events(xtdb_octopoes_service)
 
     assert xtdb_octopoes_service.ooi_repository.list_oois({OOI}, valid_time).count < count
@@ -421,7 +421,7 @@ def chain(source, results):
 
     chain(software1, [Software(name="ACME", version="v2")])
 
-    xtdb_octopoes_service.ooi_repository.delete(software1.reference, valid_time)
+    xtdb_octopoes_service.ooi_repository.delete_if_exists(software1.reference, valid_time)
     event_manager.complete_process_events(xtdb_octopoes_service)
 
     assert xtdb_octopoes_service.ooi_repository.list_oois({OOI}, valid_time).count == 0

octopoes/tests/test_octopoes_service.py

@@ -75,7 +75,7 @@ def test_on_update_origin(octopoes_service, valid_time):
     octopoes_service.process_event(event)
 
     # the ooi should be deleted
-    octopoes_service.ooi_repository.delete.assert_called_once_with(
+    octopoes_service.ooi_repository.delete_if_exists.assert_called_once_with(
         Reference.from_str("IPAddress|internet|1.1.1.1"), valid_time
     )
 

rocky/account/mixins.py

@@ -9,6 +9,7 @@
 from django.http import Http404
 from django.utils.translation import gettext_lazy as _
 from django.views import View
+from katalogus.client import KATalogus, get_katalogus
 from rest_framework.exceptions import ValidationError
 from rest_framework.request import Request
 from tools.models import Indemnification, Organization, OrganizationMember
@@ -114,6 +115,9 @@ def setup(self, request, *args, **kwargs):
         )
         self.bytes_client = get_bytes_client(organization_code)
 
+    def get_katalogus(self) -> KATalogus:
+        return get_katalogus(self.organization_member)
+
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
         context["organization"] = self.organization

rocky/account/templates/account_detail.html

@@ -14,7 +14,7 @@
                         <h1>{% translate "Account details" %}</h1>
                     </div>
                     <div class="horizontal-view toolbar">
-                        <a class="button" href="{% url 'password_reset' %}">{% translate "Reset password" %}</a>
+                        <a class="button ghost" href="{% url 'password_reset' %}">{% translate "Reset password" %}</a>
                     </div>
                 </div>
                 <dl>

rocky/assets/css/components/plugins.scss

@@ -95,20 +95,4 @@ $normalizer-bg: #e17000;
   .tile-description {
     font-size: var(--tile-font-size);
   }
-
-  button,
-  a.button {
-    min-width: auto;
-    min-height: auto;
-    font-weight: 700;
-    line-height: var(--form-button-line-height);
-
-    &.plugin-disabled {
-      color: var(--color-alert-negative);
-
-      &.plugin-disabled:hover {
-        border-color: var(--color-alert-negative);
-      }
-    }
-  }
 }