Merge branch 'feature/nibbles' into feature/scan_level_nibbles_with_p…

…hantom_results

.env-defaults

@@ -10,7 +10,6 @@ TWOFACTOR_ENABLED=False
 # --- Endpoints --- #
 OCTOPOES_API=http://octopoes_api:80
 SCHEDULER_API=http://scheduler:8000
-KEIKO_API=http://keiko:8000
 KATALOGUS_API=http://katalogus:8000
 XTDB_URI=http://crux:3000
 BOEFJES_API=http://boefje:8000

.env-dist

@@ -59,9 +59,6 @@ BYTES_DB_PASSWORD={%BYTES_DB_PASSWORD}
 BYTES_DB_URI=postgresql://${BYTES_DB_USER}:${BYTES_DB_PASSWORD}@postgres:5432/${BYTES_DB}
 
 
-# --- Keiko --- #
-# See `keiko/keiko/settings.py`
-
 # --- Octopoes --- #
 # See `octopoes/octopoes/config/settings.py`
 

.env-prod

@@ -18,7 +18,6 @@ TWOFACTOR_ENABLED=True
 # --- Endpoints --- #
 OCTOPOES_API=http://octopoes_api:80
 SCHEDULER_API=http://scheduler:8000
-KEIKO_API=http://keiko:8000
 KATALOGUS_API=http://katalogus:8000
 XTDB_URI=http://crux:3000
 BOEFJES_API=http://boefje:8000

.github/workflows/build-rdo-package.yml

@@ -123,7 +123,6 @@ jobs:
           BYTES_PASSWORD: password
           BYTES_USERNAME: username
           KATALOGUS_API: http://katalogus:8000
-          KEIKO_API: http://keiko:8000
           OCTOPOES_API: http://octopoes_api:80
           SCHEDULER_API: http://scheduler:8000
           SECRET_KEY: whatever
@@ -200,28 +199,6 @@ jobs:
         run: tar -cvzf ${{ env.PKGDIR }}/boefjes_${{ env.RELEASE_VERSION }}.tar.gz .
         working-directory: ./boefjes
 
-      # Keiko
-      - name: Keiko Create /var/lib/html
-        run: sudo rm -rfv /var/lib/html; sudo mkdir -p /var/www/html ; sudo chown `id -u`:`id -g` /var/www/html
-
-      - name: Keiko copy project to run location.
-        run: cp -rv * /var/www/html
-        working-directory: ./keiko
-
-      - name: Keiko Create env
-        run: python${{ matrix.python_version }} -m venv /var/www/html/.venv
-
-      - name: Keiko Install requirements
-        run: source .venv/bin/activate; pip install --upgrade pip; find . -name requirements.txt | xargs -L 1 pip install -r
-        working-directory: /var/www/html
-
-      - name: Keiko Create archive
-        run: tar -zcvf ${{ env.PKGDIR }}/katkeiko_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz -C /var/www/html/ .venv
-
-      - name: Keiko Create katkeiko release
-        run: tar -cvzf ${{ env.PKGDIR }}/katkeiko_${{ env.RELEASE_VERSION }}.tar.gz .
-        working-directory: ./keiko
-
       # Common / Uploads
       - name: Octopoes Upload whl package
         uses: actions/upload-artifact@v4
@@ -288,15 +265,3 @@ jobs:
         with:
           name: boefjes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}
           path: ${{ env.PKGDIR }}/boefjes_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz
-
-      - name: Keiko Upload tar
-        uses: actions/upload-artifact@v4
-        with:
-          name: katkeiko_${{ env.RELEASE_VERSION }}
-          path: ${{ env.PKGDIR }}/katkeiko_${{ env.RELEASE_VERSION }}.tar.gz
-
-      - name: Keiko Upload venv tar
-        uses: actions/upload-artifact@v4
-        with:
-          name: katkeiko_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}
-          path: ${{ env.PKGDIR }}/katkeiko_venv_${{ env.RELEASE_VERSION }}_python${{ matrix.python_version }}.tar.gz

.github/workflows/build_docs_on_pr.yml

@@ -3,12 +3,11 @@ name: Build docs artifact for PR
 on:
   pull_request:
     paths:
-      # We generate documentation for boefje, bytes, keiko, mula and octopoes
+      # We generate documentation for boefje, bytes, mula and octopoes
       # from code so the workflow should also depend on it.
       - "boefje/**"
       - "bytes/**"
       - "docs/**"
-      - "keiko/**"
       - "mula/**"
       - "octopoes/**"
       - "requirements.txt"

.github/workflows/rocky_makelang.yml

@@ -48,7 +48,6 @@ jobs:
           BYTES_PASSWORD: password
           BYTES_USERNAME: username
           KATALOGUS_API: http://katalogus:8000
-          KEIKO_API: http://keiko:8000
           OCTOPOES_API: http://octopoes_api:80
           SCHEDULER_API: http://scheduler:8000
           SECRET_KEY: whatever

.github/workflows/test_debian_packages_on_ubuntu.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       matrix:
         dist: [ubuntu]
-        package: [bytes, boefjes, rocky, keiko, octopoes, mula]
+        package: [bytes, boefjes, rocky, octopoes, mula]
     runs-on: ubuntu-latest
 
     steps:
@@ -160,11 +160,6 @@ jobs:
         with:
           name: kat-mula_${{ env.RELEASE_VERSION }}_ubuntu.deb
 
-      - name: Download Keiko artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: kat-keiko_${{ env.RELEASE_VERSION }}_ubuntu.deb
-
       - name: Download Octopoes artifact
         uses: actions/download-artifact@v4
         with:
@@ -213,18 +208,13 @@ jobs:
           sudo sed -i "s/BYTES_PASSWORD=\$/BYTES_PASSWORD=$(sudo grep BYTES_PASSWORD /etc/kat/bytes.conf | awk -F'=' '{ print $2 }')/" /etc/kat/mula.conf
 
       - name: Restart KAT
-        run: sudo systemctl restart kat-rocky kat-rocky-worker kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
+        run: sudo systemctl restart kat-rocky kat-rocky-worker kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-octopoes kat-octopoes-worker
 
       - name: Setup accounts in Rocky
         run: |
           DJANGO_SUPERUSER_PASSWORD=robotpassword sudo -E -u kat rocky-cli createsuperuser --noinput --email robot@localhost --full_name "Mr. Robot"
           sudo -u kat rocky-cli setup_dev_account
 
-      - name: Check Keiko API health or print response and logs
-        run: |
-          for i in {1..15}; do curl -s http://localhost:8005/health | jq .healthy | grep true -q && s=0 && break || s=$? && sleep 1 ; done
-          if [ $s != 0 ]; then echo $(curl -v http://localhost:8005/health) || true && journalctl --no-pager -u kat-keiko.service && exit $s ; fi
-
       - name: Check Bytes API health or print response and logs
         run: |
           for i in {1..15}; do curl -s http://localhost:8002/health | jq .healthy | grep true -q && s=0 && break || s=$? && sleep 1 ; done

.gitignore

@@ -439,7 +439,6 @@ nl-kat-*
 # Automatically generated markdown files for the environment variables
 docs/source/installation-and-deployment/environment-settings/boefjes.md
 docs/source/installation-and-deployment/environment-settings/bytes.md
-docs/source/installation-and-deployment/environment-settings/keiko.md
 docs/source/installation-and-deployment/environment-settings/mula.md
 docs/source/installation-and-deployment/environment-settings/octopoes.md
 

.pre-commit-config.yaml

@@ -100,7 +100,6 @@ repos:
         exclude: |
           (?x)(
           ^boefjes/tools |
-          ^keiko/templates |
           ^mula/whitelist\.py$ |
           ^mula/scripts |
           ^octopoes/tools |
@@ -129,8 +128,6 @@ repos:
           ^boefjes/boefjes/plugins/kat_wappalyzer/data/.*.json |
           ^boefjes/tests/examples/inputs/cve-result-without-cvss.json |
           ^boefjes/tests/examples |
-          ^keiko/glossaries |
-          ^keiko/templates/.*/template.tex$ |
           ^rocky/assets/js/vendor |
           ^rocky/assets/css/themes/soft/fonts/tabler-icons/tabler-icons.scss$ |
           ^rocky/tests/stubs |

Makefile

@@ -112,7 +112,6 @@ ubuntu22.04-build-image:
 CHECKSUM_CMD = $(if $(filter $(UNAME), Darwin), shasum -a 256, sha256sum --quiet)
 
 docs:
-	$(call build-settings-doc,keiko,keiko.settings,keiko,Keiko)
 	$(call build-settings-doc,octopoes,octopoes.config.settings,octopoes,Octopoes)
 	$(call build-settings-doc,boefjes,boefjes.config,boefjes,Boefjes)
 	$(call build-settings-doc,bytes,bytes.config,bytes,Bytes)
@@ -124,7 +123,7 @@ docs:
 	echo "f2094bbf6141b359722c4fe454eb6c4b0f0e42cc10cc7af921fc158fceb86539  docs/source/_static/d3.min.js" | $(CHECKSUM_CMD) --check || exit 1
 	echo "0d2b6f2361e7e0ce466a6ed458e03daa5584b42ef6926c3beb62eb64670ca261  docs/source/_static/mermaid.min.js" | $(CHECKSUM_CMD) --check || exit 1
 
-	PYTHONPATH=$(PYTHONPATH):boefjes/:bytes/:keiko/:mula/:octopoes/ sphinx-build -b html --fail-on-warning docs/source docs/_build
+	PYTHONPATH=$(PYTHONPATH):boefjes/:bytes/:mula/:octopoes/ sphinx-build -b html --fail-on-warning docs/source docs/_build
 
 
 poetry-dependencies:

boefjes/boefjes/katalogus/plugins.py

@@ -15,6 +15,7 @@
     get_plugins_filter_parameters,
 )
 from boefjes.models import FilterParameters, PaginationParameters, PluginType
+from boefjes.sql.db_models import RunOn
 from boefjes.sql.plugin_storage import get_plugin_storage
 from boefjes.storage.interfaces import DuplicatePlugin, IntegrityError, NotAllowed, PluginStorage
 
@@ -130,6 +131,7 @@ class BoefjeIn(BaseModel):
     boefje_schema: dict | None = None
     cron: str | None = None
     interval: int | None = None
+    run_on: list[RunOn] | None = None
     oci_image: str | None = None
     oci_arguments: list[str] = Field(default_factory=list)
 

boefjes/boefjes/migrations/versions/fc0295b38184_add_run_on_field_to_boefje.py

@@ -0,0 +1,33 @@
+"""Add run on field to boefje
+
+Revision ID: fc0295b38184
+Revises: 9f48560b0000
+Create Date: 2025-02-04 16:43:59.171960
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "fc0295b38184"
+down_revision = "9f48560b0000"
+branch_labels = None
+depends_on = None
+
+
+run_on = sa.Enum("create", "update", "create_update", name="run_on")
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    run_on.create(op.get_bind())
+    op.add_column("boefje", sa.Column("run_on", run_on, nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("boefje", "run_on")
+    run_on.drop(op.get_bind(), checkfirst=False)
+    # ### end Alembic commands ###

boefjes/boefjes/models.py

@@ -1,5 +1,6 @@
 import datetime
 from enum import Enum
+from functools import total_ordering
 from typing import Literal
 
 from croniter import croniter
@@ -8,6 +9,18 @@
 from pydantic import BaseModel, Field, field_validator
 
 
+# This makes the RunOn sortable when in a list. This is convenient for e.g. the RunOnDB.from_run_ons method, that now
+# does not have to take the ordering of a boefje.run_on into account in its match statement. This is especially handy
+# once we introduce more RunOn values such as DELETE.
+@total_ordering
+class RunOn(Enum):
+    CREATE = "create"
+    UPDATE = "update"
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+
 class Organisation(BaseModel):
     id: str
     name: str
@@ -34,6 +47,7 @@ class Boefje(Plugin):
     boefje_schema: dict | None = None
     cron: str | None = None
     interval: int | None = None
+    run_on: list[RunOn] | None = None
     runnable_hash: str | None = None
     oci_image: str | None = None
     oci_arguments: list[str] = Field(default_factory=list)