diff --git a/boefjes/boefjes/dependencies/plugins.py b/boefjes/boefjes/dependencies/plugins.py index c6f922ce9d4..ccb3187c0e2 100644 --- a/boefjes/boefjes/dependencies/plugins.py +++ b/boefjes/boefjes/dependencies/plugins.py @@ -158,7 +158,12 @@ def delete_settings(self, organisation_id: str, plugin_id: str): self.set_enabled_by_id(plugin_id, organisation_id, False) def schema(self, plugin_id: str) -> dict | None: - return self.local_repo.schema(plugin_id) + try: + boefje = self.plugin_storage.boefje_by_id(plugin_id) + + return boefje.schema + except PluginNotFound: + return self.local_repo.schema(plugin_id) def cover(self, plugin_id: str) -> Path: try: diff --git a/boefjes/boefjes/katalogus/plugins.py b/boefjes/boefjes/katalogus/plugins.py index 0928280af4a..134243ad963 100644 --- a/boefjes/boefjes/katalogus/plugins.py +++ b/boefjes/boefjes/katalogus/plugins.py @@ -3,7 +3,8 @@ from fastapi import APIRouter, Body, Depends, HTTPException, status from fastapi.responses import FileResponse, JSONResponse, Response -from pydantic import BaseModel, Field +from jsonschema.validators import Draft202012Validator +from pydantic import BaseModel, Field, field_validator from boefjes.dependencies.plugins import ( PluginService, @@ -90,6 +91,8 @@ def get_plugin( @router.post("/plugins", status_code=status.HTTP_201_CREATED) def add_plugin(plugin: PluginType, plugin_service: PluginService = Depends(get_plugin_service)): with plugin_service as service: + plugin.static = False # Creation through the API implies that these cannot be static + if plugin.type == "boefje": return service.create_boefje(plugin) @@ -124,9 +127,19 @@ class BoefjeIn(BaseModel): scan_level: int = 1 consumes: set[str] = Field(default_factory=set) produces: set[str] = Field(default_factory=set) + schema: dict | None = None oci_image: str | None = None oci_arguments: list[str] = Field(default_factory=list) + @field_validator("schema") + @classmethod + def json_schema_valid(cls, schema: dict | None) -> dict | None: + if schema is not None: + Draft202012Validator.check_schema(schema) + return schema + + return None + @router.patch("/boefjes/{boefje_id}", status_code=status.HTTP_204_NO_CONTENT) def update_boefje( diff --git a/boefjes/boefjes/katalogus/root.py b/boefjes/boefjes/katalogus/root.py index 5c62cea744b..8aa0c1683c5 100644 --- a/boefjes/boefjes/katalogus/root.py +++ b/boefjes/boefjes/katalogus/root.py @@ -5,6 +5,7 @@ import structlog from fastapi import APIRouter, FastAPI, Request, status from fastapi.responses import JSONResponse, RedirectResponse +from jsonschema.exceptions import SchemaError from opentelemetry import trace from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor @@ -96,6 +97,14 @@ def storage_error_handler(request: Request, exc: StorageError): ) +@app.exception_handler(SchemaError) +def schema_error_handler(request: Request, exc: StorageError): + return JSONResponse( + status_code=status.HTTP_400_BAD_REQUEST, + content={"message": "Invalid jsonschema provided"}, + ) + + class ServiceHealth(BaseModel): service: str healthy: bool = False diff --git a/boefjes/boefjes/local_repository.py b/boefjes/boefjes/local_repository.py index 4e53054fe43..a29fb92ba35 100644 --- a/boefjes/boefjes/local_repository.py +++ b/boefjes/boefjes/local_repository.py @@ -11,6 +11,7 @@ BOEFJES_DIR, ENTRYPOINT_NORMALIZERS, NORMALIZER_DEFINITION_FILE, + SCHEMA_FILE, BoefjeResource, ModuleException, NormalizerResource, @@ -52,10 +53,10 @@ def schema(self, id_: str) -> dict | None: if id_ not in boefjes: return None - path = boefjes[id_].path / "schema.json" + path = boefjes[id_].path / SCHEMA_FILE if not path.exists(): - logger.debug("Did not find schema for boefje %s", boefjes[id_]) + logger.debug("Did not find schema for boefje %s", id_) return None return json.loads(path.read_text()) diff --git a/boefjes/boefjes/migrations/versions/5be152459a7b_introduce_schema_field_to_boefje_model.py b/boefjes/boefjes/migrations/versions/5be152459a7b_introduce_schema_field_to_boefje_model.py new file mode 100644 index 00000000000..2cd63145aa5 --- /dev/null +++ b/boefjes/boefjes/migrations/versions/5be152459a7b_introduce_schema_field_to_boefje_model.py @@ -0,0 +1,62 @@ +"""Introduce schema field to Boefje model + +Revision ID: 5be152459a7b +Revises: f9de6eb7824b +Create Date: 2024-08-08 14:47:12.582017 + +""" + +import logging + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.orm import sessionmaker + +from boefjes.local_repository import get_local_repository +from boefjes.sql.plugin_storage import create_plugin_storage +from boefjes.storage.interfaces import PluginNotFound + +# revision identifiers, used by Alembic. +revision = "5be152459a7b" +down_revision = "f9de6eb7824b" +branch_labels = None +depends_on = None + +logger = logging.getLogger(__name__) + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("boefje", sa.Column("schema", sa.JSON(), nullable=True)) + + local_repo = get_local_repository() + session = sessionmaker(bind=op.get_bind())() + + with create_plugin_storage(session) as storage: + plugins = local_repo.get_all() + logger.info("Found %s plugins", len(plugins)) + + for plugin in local_repo.get_all(): + schema = local_repo.schema(plugin.id) + + if schema: + try: + # This way we avoid the safeguard that updating static boefjes is not allowed + instance = storage._db_boefje_instance_by_id(plugin.id) + instance.schema = schema + storage.session.add(instance) + logger.info("Updated database entry for plugin %s", plugin.id) + except PluginNotFound: + logger.info("No database entry for plugin %s", plugin.id) + continue + else: + logger.info("No schema present for plugin %s", plugin.id) + + session.close() + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("boefje", "schema") + # ### end Alembic commands ### diff --git a/boefjes/boefjes/migrations/versions/f9de6eb7824b_introduce_boefjeconfig_model.py b/boefjes/boefjes/migrations/versions/f9de6eb7824b_introduce_boefjeconfig_model.py index ffa35930368..40a44d504d3 100644 --- a/boefjes/boefjes/migrations/versions/f9de6eb7824b_introduce_boefjeconfig_model.py +++ b/boefjes/boefjes/migrations/versions/f9de6eb7824b_introduce_boefjeconfig_model.py @@ -10,11 +10,12 @@ import sqlalchemy as sa from alembic import op -from sqlalchemy.orm import sessionmaker +from psycopg2._json import Json +from psycopg2.extensions import register_adapter +from psycopg2.extras import execute_values from boefjes.local_repository import get_local_repository -from boefjes.sql.plugin_storage import create_plugin_storage -from boefjes.storage.interfaces import PluginNotFound +from boefjes.models import Boefje, Normalizer # revision identifiers, used by Alembic. revision = "f9de6eb7824b" @@ -57,72 +58,136 @@ def upgrade() -> None: op.add_column("boefje", sa.Column("static", sa.Boolean(), server_default="false", nullable=False)) op.add_column("normalizer", sa.Column("static", sa.Boolean(), server_default="false", nullable=False)) + register_adapter(dict, Json) + local_plugins = {plugin.id: plugin for plugin in get_local_repository().get_all()} connection = op.get_bind() - session = sessionmaker(bind=connection)() - - with create_plugin_storage(session) as storage: - # Get unique plugin_ids from the settings table for boefjes that do not exist yet in the database - query = """ - SELECT DISTINCT s.plugin_id FROM settings s left join boefje b on b.plugin_id = s.plugin_id - where b.plugin_id IS NULL - """ - for plugin_id_output in op.get_bind().execute(query).fetchall(): - plugin_id = plugin_id_output[0] - if plugin_id not in local_plugins: - raise ValueError(f"Invalid plugin id found: {plugin_id}") - - # Since settings are boefje-only at this moment - if local_plugins[plugin_id].type != "boefje": - raise ValueError(f"Settings for normalizer or bit found: {plugin_id}. Remove these entries first.") - - try: - storage.boefje_by_id(plugin_id) - continue # The Boefje already exists - except PluginNotFound: - pass # The raw query bypasses the session "cache", so this just checks for duplicates - - storage.create_boefje(local_plugins[plugin_id]) # type: ignore - - query = """ - SELECT DISTINCT p.plugin_id FROM plugin_state p left join boefje b on b.plugin_id = p.plugin_id - where b.plugin_id IS NULL - """ - - for plugin_id_output in op.get_bind().execute(query).fetchall(): - plugin_id = plugin_id_output[0] - if plugin_id not in local_plugins: - logger.warning("Unknown plugin id found: %s. You might have to re-enable the plugin!", plugin_id) - continue - - try: - storage.boefje_by_id(plugin_id) - continue # The Boefje already exists - except PluginNotFound: - pass # The raw query bypasses the session "cache", so this just checks for duplicates - - if local_plugins[plugin_id].type == "boefje": - storage.create_boefje(local_plugins[plugin_id]) # type: ignore - - query = """ - SELECT DISTINCT p.plugin_id FROM plugin_state p left join normalizer n on n.plugin_id = p.plugin_id - where n.plugin_id IS NULL - """ - - for plugin_id_output in op.get_bind().execute(query).fetchall(): - plugin_id = plugin_id_output[0] - if plugin_id not in local_plugins: - logger.warning("Unknown plugin id found: %s. You might have to re-enable the plugin!", plugin_id) - continue - - try: - storage.normalizer_by_id(plugin_id) - continue # The Normalizer already exists - except PluginNotFound: - pass # The raw query bypasses the session "cache", so this just checks for duplicates - - if local_plugins[plugin_id].type == "normalizer": - storage.create_normalizer(local_plugins[plugin_id]) # type: ignore + + # Get unique plugin_ids from the settings table for boefjes that do not exist yet in the database + query = """ + SELECT DISTINCT s.plugin_id FROM settings s left join boefje b on b.plugin_id = s.plugin_id + where b.plugin_id IS NULL + """ # noqa: S608 + + to_insert: list[Boefje] = [] + + for plugin_id_output in connection.execute(query).fetchall(): + plugin_id = plugin_id_output[0] + if plugin_id not in local_plugins: + raise ValueError(f"Invalid plugin id found: {plugin_id}") + + # Since settings are boefje-only at this moment + if local_plugins[plugin_id].type != "boefje": + raise ValueError(f"Settings for normalizer or bit found: {plugin_id}. Remove these entries first.") + + res = connection.execute(f"SELECT id FROM boefje where plugin_id = '{plugin_id}'") # noqa: S608 + if res.fetchone() is not None: + continue # The Boefje already exists + + if local_plugins[plugin_id].type == "boefje": + to_insert.append(local_plugins[plugin_id]) + + entries = [ + ( + boefje.id, + boefje.name, + boefje.description, + str(boefje.scan_level), + list(boefje.consumes), + list(boefje.produces), + boefje.environment_keys, + boefje.oci_image, + boefje.oci_arguments, + boefje.version, + ) + for boefje in to_insert + ] + query = """INSERT INTO boefje (plugin_id, name, description, scan_level, consumes, produces, environment_keys, + oci_image, oci_arguments, version) values %s""" + + with connection.begin(): + cursor = connection.connection.cursor() + execute_values(cursor, query, entries) + + to_insert = [] + + query = """ + SELECT DISTINCT p.plugin_id FROM plugin_state p left join boefje b on b.plugin_id = p.plugin_id + where b.plugin_id IS NULL + """ + + for plugin_id_output in connection.execute(query).fetchall(): + plugin_id = plugin_id_output[0] + if plugin_id not in local_plugins: + logger.warning("Unknown plugin id found: %s. You might have to re-enable the plugin!", plugin_id) + continue + + res = connection.execute(f"SELECT id FROM boefje where plugin_id = '{plugin_id}'") # noqa: S608 + if res.fetchone() is not None: + continue # The Boefje already exists + + if local_plugins[plugin_id].type == "boefje": + to_insert.append(local_plugins[plugin_id]) + + entries = [ + ( + boefje.id, + boefje.name, + boefje.description, + str(boefje.scan_level), + list(boefje.consumes), + list(boefje.produces), + boefje.environment_keys, + boefje.oci_image, + boefje.oci_arguments, + boefje.version, + ) + for boefje in to_insert + ] + query = """INSERT INTO boefje (plugin_id, name, description, scan_level, consumes, produces, environment_keys, + oci_image, oci_arguments, version) values %s""" # noqa: S608 + + with connection.begin(): + cursor = connection.connection.cursor() + execute_values(cursor, query, entries) + + normalizers_to_insert: list[Normalizer] = [] + query = """ + SELECT DISTINCT p.plugin_id FROM plugin_state p left join normalizer n on n.plugin_id = p.plugin_id + where n.plugin_id IS NULL + """ # noqa: S608 + + for plugin_id_output in connection.execute(query).fetchall(): + plugin_id = plugin_id_output[0] + if plugin_id not in local_plugins: + logger.warning("Unknown plugin id found: %s. You might have to re-enable the plugin!", plugin_id) + continue + + res = connection.execute(f"SELECT id FROM normalizer where plugin_id = '{plugin_id}'") # noqa: S608 + if res.fetchone() is not None: + continue # The Normalizer already exists + + if local_plugins[plugin_id].type == "normalizer": + normalizers_to_insert.append(local_plugins[plugin_id]) + + normalizer_entries = [ + ( + normalizer.id, + normalizer.name, + normalizer.description, + normalizer.consumes, + normalizer.produces, + normalizer.environment_keys, + normalizer.version, + ) + for normalizer in normalizers_to_insert + ] + query = """INSERT INTO normalizer (plugin_id, name, description, consumes, produces, environment_keys, version) + values %s""" # noqa: S608 + + with connection.begin(): + cursor = connection.connection.cursor() + execute_values(cursor, query, normalizer_entries) with connection.begin(): connection.execute(""" diff --git a/boefjes/boefjes/models.py b/boefjes/boefjes/models.py index 69d8cc2c637..4881b26008a 100644 --- a/boefjes/boefjes/models.py +++ b/boefjes/boefjes/models.py @@ -2,7 +2,8 @@ from enum import Enum from typing import Literal -from pydantic import BaseModel, Field +from jsonschema.validators import Draft202012Validator +from pydantic import BaseModel, Field, field_validator class Organisation(BaseModel): @@ -29,10 +30,21 @@ class Boefje(Plugin): scan_level: int = 1 consumes: set[str] = Field(default_factory=set) produces: set[str] = Field(default_factory=set) + schema: dict | None = None runnable_hash: str | None = None oci_image: str | None = None oci_arguments: list[str] = Field(default_factory=list) + @field_validator("schema") + @classmethod + def json_schema_valid(cls, schema: dict) -> dict: + if schema is not None: + Draft202012Validator.check_schema(schema) + return schema + + class Config: + validate_assignment = True + class Normalizer(Plugin): type: Literal["normalizer"] = "normalizer" diff --git a/boefjes/boefjes/plugins/models.py b/boefjes/boefjes/plugins/models.py index 5a2669ffbc4..91af854f576 100644 --- a/boefjes/boefjes/plugins/models.py +++ b/boefjes/boefjes/plugins/models.py @@ -1,15 +1,20 @@ import hashlib +import json from enum import Enum from importlib import import_module from inspect import isfunction, signature +from json import JSONDecodeError from pathlib import Path from typing import Protocol +from jsonschema.exceptions import SchemaError + from boefjes.models import Boefje, Normalizer BOEFJES_DIR = Path(__file__).parent BOEFJE_DEFINITION_FILE = "boefje.json" +SCHEMA_FILE = "schema.json" NORMALIZER_DEFINITION_FILE = "normalizer.json" ENTRYPOINT_BOEFJES = "main.py" ENTRYPOINT_NORMALIZERS = "normalize.py" @@ -62,6 +67,14 @@ def __init__(self, path: Path, package: str): if (path / ENTRYPOINT_BOEFJES).exists(): self.module = get_runnable_module_from_package(package, ENTRYPOINT_BOEFJES, parameter_count=1) + if (path / SCHEMA_FILE).exists(): + try: + self.boefje.schema = json.load((path / SCHEMA_FILE).open()) + except JSONDecodeError as e: + raise ModuleException("Invalid schema file") from e + except SchemaError as e: + raise ModuleException("Invalid schema") from e + class NormalizerResource: """Represents a Normalizer package that we can run. Throws a ModuleException if any validation fails.""" diff --git a/boefjes/boefjes/sql/db_models.py b/boefjes/boefjes/sql/db_models.py index 1ecd0cc0086..e8e9740f8f4 100644 --- a/boefjes/boefjes/sql/db_models.py +++ b/boefjes/boefjes/sql/db_models.py @@ -76,6 +76,7 @@ class BoefjeInDB(SQL_BASE): consumes = Column(types.ARRAY(types.String(length=128)), default=lambda: [], nullable=False) produces = Column(types.ARRAY(types.String(length=128)), default=lambda: [], nullable=False) environment_keys = Column(types.ARRAY(types.String(length=128)), default=lambda: [], nullable=False) + schema = Column(types.JSON(), nullable=True) # Image specifications oci_image = Column(types.String(length=256), nullable=True) diff --git a/boefjes/boefjes/sql/plugin_storage.py b/boefjes/boefjes/sql/plugin_storage.py index 2dddd0ad822..c9bcffbaa2f 100644 --- a/boefjes/boefjes/sql/plugin_storage.py +++ b/boefjes/boefjes/sql/plugin_storage.py @@ -110,6 +110,7 @@ def to_boefje_in_db(boefje: Boefje) -> BoefjeInDB: scan_level=str(boefje.scan_level), consumes=boefje.consumes, produces=boefje.produces, + schema=boefje.schema, environment_keys=boefje.environment_keys, oci_image=boefje.oci_image, oci_arguments=boefje.oci_arguments, @@ -142,6 +143,7 @@ def to_boefje(boefje_in_db: BoefjeInDB) -> Boefje: scan_level=int(boefje_in_db.scan_level), consumes=boefje_in_db.consumes, produces=boefje_in_db.produces, + schema=boefje_in_db.schema, environment_keys=boefje_in_db.environment_keys, oci_image=boefje_in_db.oci_image, oci_arguments=boefje_in_db.oci_arguments, diff --git a/boefjes/tests/integration/test_api.py b/boefjes/tests/integration/test_api.py index b29060cf316..9c32af42f8a 100644 --- a/boefjes/tests/integration/test_api.py +++ b/boefjes/tests/integration/test_api.py @@ -60,7 +60,7 @@ def test_cannot_add_plugin_reserved_id(self): self.assertEqual(response.status_code, 400) self.assertEqual(response.json(), {"message": "Plugin id 'dns-records' is already used"}) - normalizer = Normalizer(id="kat_nmap_normalize", name="My test normalizer", static=False) + normalizer = Normalizer(id="kat_nmap_normalize", name="My test normalizer") response = self.client.post(f"/v1/organisations/{self.org.id}/plugins", content=normalizer.json()) self.assertEqual(response.status_code, 400) self.assertEqual(response.json(), {"message": "Plugin id 'kat_nmap_normalize' is already used"}) @@ -115,8 +115,8 @@ def test_delete_normalizer(self): self.assertEqual(response.status_code, 404) def test_update_plugins(self): - normalizer = Normalizer(id="norm_id", name="My test normalizer", static=False) - boefje = Boefje(id="test_plugin", name="My test boefje", description="123", static=False) + normalizer = Normalizer(id="norm_id", name="My test normalizer") + boefje = Boefje(id="test_plugin", name="My test boefje", description="123") self.client.post(f"/v1/organisations/{self.org.id}/plugins", content=boefje.json()) self.client.patch(f"/v1/organisations/{self.org.id}/boefjes/{boefje.id}", json={"description": "4"}) @@ -126,6 +126,50 @@ def test_update_plugins(self): self.assertEqual(response.json()["description"], "4") self.assertTrue(response.json()["enabled"]) + self.client.post(f"/v1/organisations/{self.org.id}/plugins", content=normalizer.json()) + self.client.patch(f"/v1/organisations/{self.org.id}/normalizers/{normalizer.id}", json={"version": "v1.2"}) + + response = self.client.get(f"/v1/organisations/{self.org.id}/plugins/{normalizer.id}") + self.assertEqual(response.json()["version"], "v1.2") + + def test_cannot_create_boefje_with_invalid_schema(self): + boefje = Boefje(id="test_plugin", name="My test boefje", description="123").model_dump(mode="json") + boefje["schema"] = {"$schema": 3} + + r = self.client.post(f"/v1/organisations/{self.org.id}/plugins", json=boefje) + self.assertEqual(r.status_code, 400) + + def test_update_boefje_schema(self): + boefje = Boefje(id="test_plugin", name="My test boefje", description="123") + self.client.post(f"/v1/organisations/{self.org.id}/plugins", content=boefje.json()) + + r = self.client.patch(f"/v1/organisations/{self.org.id}/boefjes/{boefje.id}", json={"schema": {"$schema": 3}}) + self.assertEqual(r.status_code, 400) + + valid_schema = { + "title": "Arguments", + "type": "object", + "properties": { + "MY_KEY": { + "title": "MY_KEY", + "type": "integer", + } + }, + "required": [], + } + r = self.client.patch(f"/v1/organisations/{self.org.id}/boefjes/{boefje.id}", json={"schema": valid_schema}) + self.assertEqual(r.status_code, 204) + + schema = self.client.get(f"/v1/organisations/{self.org.id}/plugins/{boefje.id}/schema.json").json() + assert schema == valid_schema + + api_boefje = self.client.get(f"/v1/organisations/{self.org.id}/plugins/{boefje.id}").json() + assert api_boefje["schema"] == valid_schema + + r = self.client.patch(f"/v1/organisations/{self.org.id}/boefjes/dns-records", json={"schema": valid_schema}) + self.assertEqual(r.status_code, 404) + + def test_cannot_update_static_plugins(self): r = self.client.patch(f"/v1/organisations/{self.org.id}/boefjes/dns-records", json={"id": "4", "version": "s"}) self.assertEqual(r.status_code, 404) r = self.client.patch(f"/v1/organisations/{self.org.id}/boefjes/dns-records", json={"name": "Overwrite name"}) @@ -136,13 +180,6 @@ def test_update_plugins(self): self.assertIsNone(response.json()["version"]) self.assertEqual(response.json()["id"], "dns-records") - self.client.post(f"/v1/organisations/{self.org.id}/plugins", content=normalizer.json()) - self.client.patch(f"/v1/organisations/{self.org.id}/normalizers/{normalizer.id}", json={"version": "v1.2"}) - - response = self.client.get(f"/v1/organisations/{self.org.id}/plugins/{normalizer.id}") - self.assertEqual(response.json()["version"], "v1.2") - - def test_cannot_update_static_plugins(self): self.client.patch(f"/v1/organisations/{self.org.id}/plugins/dns-records", json={"enabled": True}) response = self.client.get(f"/v1/organisations/{self.org.id}/plugins/dns-records") self.assertTrue(response.json()["enabled"]) diff --git a/boefjes/tests/integration/test_migration_add_schema_field.py b/boefjes/tests/integration/test_migration_add_schema_field.py new file mode 100644 index 00000000000..26fa13f0a70 --- /dev/null +++ b/boefjes/tests/integration/test_migration_add_schema_field.py @@ -0,0 +1,166 @@ +import os +from unittest import TestCase, skipIf + +import alembic.config +from psycopg2.extras import execute_values +from sqlalchemy.orm import sessionmaker + +from boefjes.local_repository import get_local_repository +from boefjes.sql.db import SQL_BASE, get_engine + + +@skipIf(os.environ.get("CI") != "1", "Needs a CI database.") +class TestSettingsToBoefjeConfig(TestCase): + def setUp(self) -> None: + self.engine = get_engine() + + # To reset autoincrement ids + alembic.config.main(argv=["--config", "/app/boefjes/boefjes/alembic.ini", "downgrade", "base"]) + # Set state to revision 6f99834a4a5a + alembic.config.main(argv=["--config", "/app/boefjes/boefjes/alembic.ini", "upgrade", "f9de6eb7824b"]) + + dns_records = get_local_repository().by_id("dns-records") + nmap_udp = get_local_repository().by_id("nmap-udp") + + entries = [ + ( + boefje.id, + boefje.name, + boefje.description, + str(boefje.scan_level), + list(sorted(boefje.consumes)), + list(sorted(boefje.produces)), + boefje.environment_keys, + boefje.oci_image, + boefje.oci_arguments, + boefje.version, + ) + for boefje in [dns_records, nmap_udp] + ] + + connection = self.engine.connect() + query = """INSERT INTO boefje (plugin_id, name, description, scan_level, consumes, produces, environment_keys, + oci_image, oci_arguments, version) values %s""" + + with connection.begin(): + execute_values(connection.connection.cursor(), query, entries) + + def test_fail_on_wrong_plugin_ids(self): + assert self.engine.execute("SELECT * from boefje").fetchall() == [ + ( + 1, + "dns-records", + None, + "DnsRecords", + "Fetch the DNS record(s) of a hostname", + "1", + ["Hostname"], + ["boefje/dns-records"], + ["RECORD_TYPES", "REMOTE_NS"], + None, + [], + None, + False, + ), + ( + 2, + "nmap-udp", + None, + "Nmap UDP", + "Defaults to top 250 UDP ports. Includes service detection.", + "2", + ["IPAddressV4", "IPAddressV6"], + ["boefje/nmap-udp"], + ["TOP_PORTS_UDP"], + "ghcr.io/minvws/openkat/nmap:latest", + ["--open", "-T4", "-Pn", "-r", "-v10", "-sV", "-sU"], + None, + False, + ), + ] + + alembic.config.main(argv=["--config", "/app/boefjes/boefjes/alembic.ini", "upgrade", "5be152459a7b"]) + + schema_dns = { + "title": "Arguments", + "type": "object", + "properties": { + "RECORD_TYPES": { + "title": "RECORD_TYPES", + "type": "string", + "description": "List of comma separated DNS record types to query for.", + "default": "A,AAAA,CAA,CERT,RP,SRV,TXT,MX,NS,CNAME,DNAME", + }, + "REMOTE_NS": { + "title": "REMOTE_NS", + "maxLength": 45, + "type": "string", + "description": "The IP address of the DNS resolver you want to use.", + "default": "1.1.1.1", + }, + }, + } + + schema_udp = { + "title": "Arguments", + "type": "object", + "properties": { + "TOP_PORTS_UDP": { + "title": "TOP_PORTS_UDP", + "type": "integer", + "minimum": 1, + "maximum": 65535, + "description": "Scan TOP_PORTS_UDP most common ports. Defaults to 250.", + } + }, + "required": [], + } + + self.assertListEqual( + self.engine.execute("SELECT * from boefje").fetchall(), + [ + ( + 1, + "dns-records", + None, + "DnsRecords", + "Fetch the DNS record(s) of a hostname", + "1", + ["Hostname"], + ["boefje/dns-records"], + ["RECORD_TYPES", "REMOTE_NS"], + None, + [], + None, + False, + schema_dns, + ), + ( + 2, + "nmap-udp", + None, + "Nmap UDP", + "Defaults to top 250 UDP ports. Includes service detection.", + "2", + ["IPAddressV4", "IPAddressV6"], + ["boefje/nmap-udp"], + ["TOP_PORTS_UDP"], + "ghcr.io/minvws/openkat/nmap:latest", + ["--open", "-T4", "-Pn", "-r", "-v10", "-sV", "-sU"], + None, + False, + schema_udp, + ), + ], + ) + + def tearDown(self) -> None: + alembic.config.main(argv=["--config", "/app/boefjes/boefjes/alembic.ini", "upgrade", "head"]) + + session = sessionmaker(bind=get_engine())() + + for table in SQL_BASE.metadata.tables: + session.execute(f"DELETE FROM {table} CASCADE") # noqa: S608 + + session.commit() + session.close() diff --git a/boefjes/tests/integration/test_remove_repository_migration.py b/boefjes/tests/integration/test_remove_repository_migration.py index dd2ee7c8034..33379d42714 100644 --- a/boefjes/tests/integration/test_remove_repository_migration.py +++ b/boefjes/tests/integration/test_remove_repository_migration.py @@ -27,6 +27,8 @@ def setUp(self) -> None: storage.create(Organisation(id="dev1", name="Test 1 ")) storage.create(Organisation(id="dev2", name="Test 2 ")) + session.close() + entries = [(1, "LOCAL", "Repository Local", "https://local.com/")] query = f"INSERT INTO repository (pk, id, name, base_url) values {','.join(map(str, entries))}" # noqa: S608 self.engine.execute(text(query)) @@ -44,7 +46,6 @@ def setUp(self) -> None: f"INSERT INTO organisation_repository (repository_pk, organisation_pk) values {','.join(map(str, entries))}" # noqa: S608 ) self.engine.execute(text(query)) - session.close() def test_fail_on_non_unique(self): session = sessionmaker(bind=self.engine)() @@ -76,8 +77,6 @@ def test_fail_on_non_unique(self): session.close() def test_downgrade(self): - session = sessionmaker(bind=self.engine)() - self.engine.execute(text("DELETE FROM plugin_state WHERE id = 2")) # Fix unique constraint fails alembic.config.main(argv=["--config", "/app/boefjes/boefjes/alembic.ini", "upgrade", "7c88b9cd96aa"]) alembic.config.main(argv=["--config", "/app/boefjes/boefjes/alembic.ini", "downgrade", "-1"]) @@ -88,8 +87,6 @@ def test_downgrade(self): (1, "LOCAL", "Local Plugin Repository", "http://dev/null") ] - session.close() - def tearDown(self) -> None: self.engine.execute(text("DELETE FROM plugin_state")) # Fix unique constraint fails diff --git a/boefjes/tests/integration/test_settings_to_boefje_config_migration.py b/boefjes/tests/integration/test_settings_to_boefje_config_migration.py index 35e7670f614..261a965240c 100644 --- a/boefjes/tests/integration/test_settings_to_boefje_config_migration.py +++ b/boefjes/tests/integration/test_settings_to_boefje_config_migration.py @@ -10,7 +10,6 @@ from boefjes.sql.config_storage import SQLConfigStorage, create_encrypter from boefjes.sql.db import SQL_BASE, get_engine from boefjes.sql.organisation_storage import SQLOrganisationStorage -from boefjes.sql.plugin_storage import SQLPluginStorage @skipIf(os.environ.get("CI") != "1", "Needs a CI database.") @@ -29,6 +28,8 @@ def setUp(self) -> None: storage.create(Organisation(id="dev1", name="Test 1 ")) storage.create(Organisation(id="dev2", name="Test 2 ")) + session.close() + encrypter = create_encrypter() entries = [ (1, encrypter.encode('{"key1": "val1"}'), "dns-records", 1), @@ -44,8 +45,6 @@ def setUp(self) -> None: ) self.engine.execute(text(query)) - session.close() - def test_fail_on_wrong_plugin_ids(self): session = sessionmaker(bind=self.engine)() @@ -73,16 +72,17 @@ def test_fail_on_wrong_plugin_ids(self): alembic.config.main(argv=["--config", "/app/boefjes/boefjes/alembic.ini", "upgrade", "f9de6eb7824b"]) - assert SQLPluginStorage(session, settings).boefje_by_id("dns-records").id == "dns-records" + assert self.engine.execute(text("SELECT id FROM boefje WHERE plugin_id = 'dns-records'")).fetchall() == [(2,)] config_storage = SQLConfigStorage(session, encrypter) assert config_storage.get_all_settings("dev1", "dns-records") == {"key1": "val1"} - assert config_storage.get_all_settings("dev2", "dns-records") == {"key1": "val1", "key2": "val2"} assert config_storage.get_all_settings("dev1", "nmap-udp") == {} + assert config_storage.get_all_settings("dev2", "dns-records") == {"key1": "val1", "key2": "val2"} assert config_storage.is_enabled_by_id("dns-records", "dev1") assert config_storage.is_enabled_by_id("nmap-udp", "dev1") + session.commit() session.close() def test_downgrade(self):