Refactoring for Report Recipe, Report Task Runner and Scheduling (#3597)

Co-authored-by: Jan Klopper <[email protected]> Co-authored-by: ammar92 <[email protected]> Co-authored-by: noamblitz <[email protected]> Co-authored-by: Donny Peeters <[email protected]> Co-authored-by: JP Bruins Slot <[email protected]>
minvws · Oct 1, 2024 · 2959a16 · 2959a16
1 parent b3b052a
commit 2959a16
Show file tree

Hide file tree

Showing 55 changed files with 1,615 additions and 778 deletions.
diff --git a/docs/source/developer_documentation/development_tutorial/creating_a_report.md b/docs/source/developer_documentation/development_tutorial/creating_a_report.md
@@ -50,7 +50,7 @@ class GreetingsReport(Report):
     id = "greetings-report"
     name = _("Greetings report")
     description = _("Makes a nice report about the selected greeting objects")
-    plugins = {"required": [], "optional": []}
+    plugins = {"required": set(), "optional": set()}
     input_ooi_types = {Greeting, IPAddressV4, IPAddressV6}
     template_path = "greetings_report/report.html"
 

diff --git a/octopoes/octopoes/models/ooi/reports.py b/octopoes/octopoes/models/ooi/reports.py
@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import Literal
+from typing import Any, Literal
 from uuid import UUID
 
 from octopoes.models import OOI, Reference
@@ -39,10 +39,27 @@ class Report(OOI):
 
     observed_at: datetime
     parent_report: Reference | None = ReferenceField("Report", default=None)
+    report_recipe: Reference | None = ReferenceField("ReportRecipe", default=None)
     has_parent: bool
 
     _natural_key_attrs = ["report_id"]
 
     @classmethod
     def format_reference_human_readable(cls, reference: Reference) -> str:
         return f"Report {reference.tokenized.report_id}"
+
+
+class ReportRecipe(OOI):
+    object_type: Literal["ReportRecipe"] = "ReportRecipe"
+
+    recipe_id: UUID
+
+    report_name_format: str
+    subreport_name_format: str
+
+    input_recipe: dict[str, Any]  # can contain a query which maintains a live set of OOIs or manually picked OOIs.
+    report_types: list[str]
+
+    cron_expression: str
+
+    _natural_key_attrs = ["recipe_id"]
diff --git a/octopoes/octopoes/models/types.py b/octopoes/octopoes/models/types.py
@@ -59,7 +59,7 @@
     Network,
 )
 from octopoes.models.ooi.question import Question
-from octopoes.models.ooi.reports import Report, ReportData
+from octopoes.models.ooi.reports import Report, ReportData, ReportRecipe
 from octopoes.models.ooi.scans import ExternalScan
 from octopoes.models.ooi.service import IPService, Service, TLSCipher
 from octopoes.models.ooi.software import Software, SoftwareInstance
@@ -163,6 +163,7 @@
     | ScanType
     | Report
     | GeographicPoint
+    | ReportRecipe
 )
 
 OOIType = ConcreteOOIType | NetworkType | FindingTypeType

diff --git a/rocky/katalogus/client.py b/rocky/katalogus/client.py
@@ -106,7 +106,7 @@ def delete_organization(self):
 
         logger.info("Deleted organization", organization_code=self.organization)
 
-    def get_plugins(self, **params):
+    def get_plugins(self, **params) -> list[Plugin]:
         try:
             response = self.session.get(f"{self.organization_uri}/plugins", params=params)
             response.raise_for_status()

diff --git a/rocky/katalogus/views/change_clearance_level.py b/rocky/katalogus/views/change_clearance_level.py
@@ -12,6 +12,7 @@
 class ChangeClearanceLevel(OrganizationPermissionRequiredMixin, SchedulerView, SinglePluginView, TemplateView):
     template_name = "change_clearance_level.html"
     permission_required = "tools.can_set_clearance_level"
+    task_type = "boefje"
 
     def setup(self, request, *args, **kwargs):
         super().setup(request, *args, **kwargs)

diff --git a/rocky/onboarding/views.py b/rocky/onboarding/views.py
@@ -364,11 +364,15 @@ class OnboardingReportView(
 
     def setup(self, request, *args, **kwargs):
         super().setup(request, *args, **kwargs)
+        self.selected_oois = self.get_ooi_pks()
+        self.selected_report_types = self.get_report_type_ids()
+
+    def get_ooi_pks(self) -> list[str]:
         ooi = self.get_ooi(self.request.GET.get("ooi", ""))
-        self.oois = [Hostname(name=ooi.web_url.tokenized["netloc"]["name"], network=ooi.network)]
-        self.selected_oois = [self.oois[0].primary_key]
+        hostname_ooi = [Hostname(name=ooi.web_url.tokenized["netloc"]["name"], network=ooi.network)]
+        return [hostname_ooi[0].primary_key]
 
-    def get_report_type_selection(self) -> list[str]:
+    def get_report_type_ids(self) -> list[str]:
         return [self.request.GET.get("report_type", "")]
 
     def post(self, request, *args, **kwargs):

diff --git a/rocky/reports/forms.py b/rocky/reports/forms.py
@@ -1,6 +1,8 @@
+from datetime import datetime, timezone
+
 from django import forms
 from django.utils.translation import gettext_lazy as _
-from tools.forms.base import BaseRockyForm
+from tools.forms.base import BaseRockyForm, DateInput
 
 from reports.report_types.definitions import Report
 
@@ -28,3 +30,108 @@ def __init__(self, report_types: set[Report], *args, **kwargs):
         super().__init__(*args, **kwargs)
         report_types_choices = ((report_type.id, report_type.name) for report_type in report_types)
         self.fields["report_type"].choices = report_types_choices
+
+
+class ReportScheduleStartDateChoiceForm(BaseRockyForm):
+    choose_date = forms.ChoiceField(
+        label="",
+        required=False,
+        widget=forms.RadioSelect(attrs={"class": "submit-on-click"}),
+        choices=(("today", _("Today")), ("schedule", _("Different date"))),
+        initial="today",
+    )
+
+
+class ReportScheduleStartDateForm(BaseRockyForm):
+    start_date = forms.DateField(
+        label="",
+        widget=DateInput(format="%Y-%m-%d"),
+        initial=lambda: datetime.now(tz=timezone.utc).date(),
+        required=True,
+    )
+
+
+class ReportRecurrenceChoiceForm(BaseRockyForm):
+    choose_recurrence = forms.ChoiceField(
+        label="",
+        required=False,
+        widget=forms.RadioSelect(attrs={"class": "submit-on-click"}),
+        choices=(("once", _("No, just once")), ("repeat", _("Yes, repeat"))),
+        initial="once",
+    )
+
+
+class ReportScheduleRecurrenceForm(BaseRockyForm):
+    recurrence = forms.ChoiceField(
+        label="",
+        required=False,
+        widget=forms.Select(attrs={"form": "generate_report"}),
+        choices=[
+            ("daily", _("Daily")),
+            ("weekly", _("Weekly")),
+            ("monthly", _("Monthly")),
+            ("yearly", _("Yearly")),
+        ],
+    )
+
+
+class CustomReportScheduleForm(BaseRockyForm):
+    start_date = forms.DateField(
+        label=_("Start date"),
+        widget=DateInput(format="%Y-%m-%d"),
+        initial=lambda: datetime.now(tz=timezone.utc).date(),
+        required=False,
+    )
+    repeating_number = forms.IntegerField(initial=1, required=False, min_value=1, max_value=100)
+    repeating_term = forms.ChoiceField(
+        widget=forms.Select,
+        choices=[
+            ("day", _("day")),
+            ("week", _("week")),
+            ("month", _("month")),
+            ("year", _("year")),
+        ],
+    )
+    on_weekdays = forms.ChoiceField(
+        widget=forms.RadioSelect,
+        choices=[
+            ("monday", "M"),
+            ("tuesday", "T"),
+            ("wednesday", "W"),
+            ("thursday", "T"),
+            ("friday", "F"),
+            ("saturday", "S"),
+            ("sunday", "S"),
+        ],
+    )
+    recurrence_ends = forms.ChoiceField(
+        widget=forms.RadioSelect,
+        choices=[
+            ("never", _("Never")),
+            ("on", _("On")),  # user choses a specific date
+            ("after", _("After")),  # after how many occurrences? Shows drop down with occurrences
+        ],
+    )
+
+    end_date = forms.DateField(
+        label=_(""),
+        widget=forms.HiddenInput(),
+        initial=lambda: datetime.now(tz=timezone.utc).date(),
+        required=False,
+    )
+
+
+class ParentReportNameForm(BaseRockyForm):
+    parent_report_name = forms.CharField(
+        label=_("Report name format"), required=False, initial="{report type} for {ooi}"
+    )
+
+
+class ChildReportNameForm(BaseRockyForm):
+    child_report_name = forms.CharField(
+        label=_("Subreports name format"), required=True, initial="{report type} for {ooi}"
+    )
+
+
+class ReportNameForm(ParentReportNameForm, ChildReportNameForm):
+    pass
diff --git a/rocky/reports/report_types/aggregate_organisation_report/appendix.html b/rocky/reports/report_types/aggregate_organisation_report/appendix.html
@@ -17,7 +17,7 @@ <h3 id="selected-oois">{% translate "Selected objects" %} ({{ total_oois }})</h3
         {% include "summary/ooi_selection.html" %}
 
     </div>
-    <h3 id="selected-report-types">{% translate "Selected Report Types" %}</h3>
+    <h3 id="selected-report-types">{% translate "Selected Report Types" %} ({{ data.report_types|length }})</h3>
     <table>
         <caption class="visually-hidden">{% translate "Selected report types" %}</caption>
         <thead>

diff --git a/rocky/reports/report_types/aggregate_organisation_report/introduction.html b/rocky/reports/report_types/aggregate_organisation_report/introduction.html
@@ -3,7 +3,7 @@
 <section class="introduction" id="introduction">
     <div>
         <div class="horizontal-view toolbar">
-            <h1>{{ report_name }}</h1>
+            <h1>{{ report_ooi.name }}</h1>
             <div class="dropdown">
                 <button aria-controls="export-add" class="dropdown-button ghost">
                     {% translate "Export" %}<span aria-hidden="true" class="icon ti-chevron-down"></span>
@@ -23,10 +23,10 @@ <h1>{{ report_name }}</h1>
         <div>
             <p>{% translate "This is the OpenKAT report for organization" %} {{ organization.name }}</p>
             <p>
-                {% translate "Created with data from:" %} <strong>{{ observed_at }} {{ TIME_ZONE }}</strong>
+                {% translate "Created with data from:" %} <strong>{{ report_ooi.observed_at }} {{ TIME_ZONE }}</strong>
             </p>
             <p>
-                {% translate "Created on:" %} <strong>{{ created_at }}</strong>
+                {% translate "Created on:" %} <strong>{{ report_ooi.date_generated }}</strong>
             </p>
             <p>
                 {% translate "Created by:" %} <strong>{{ organization_member.user.full_name }}</strong>

diff --git a/rocky/reports/report_types/definitions.py b/rocky/reports/report_types/definitions.py
@@ -13,8 +13,21 @@
 
 
 class ReportPlugins(TypedDict):
-    required: list[str]
-    optional: list[str]
+    required: set[str]
+    optional: set[str]
+
+
+def report_plugins_union(report_types: list[type["BaseReport"]]) -> ReportPlugins:
+    """Take the union of the required and optional plugin sets and remove optional plugins that are required"""
+
+    plugins: ReportPlugins = {"required": set(), "optional": set()}
+
+    for report_type in report_types:
+        plugins["required"].update(report_type.plugins["required"])
+        plugins["optional"].update(report_type.plugins["optional"])
+        plugins["optional"].difference_update(report_type.plugins["required"])
+
+    return plugins
 
 
 class BaseReport:

diff --git a/rocky/reports/report_types/dns_report/report.py b/rocky/reports/report_types/dns_report/report.py
@@ -18,7 +18,7 @@ class DNSReport(Report):
     id = "dns-report"
     name = _("DNS Report")
     description = _("DNS reports focus on domain name system configuration and potential weaknesses.")
-    plugins = {"required": ["dns-records", "dns-sec"], "optional": ["dns-zone"]}
+    plugins = {"required": {"dns-records", "dns-sec"}, "optional": {"dns-zone"}}
     input_ooi_types = {Hostname}
     template_path = "dns_report/report.html"
 

diff --git a/rocky/reports/report_types/findings_report/report.py b/rocky/reports/report_types/findings_report/report.py
@@ -23,8 +23,8 @@ class FindingsReport(Report):
     id = "findings-report"
     name = _("Findings Report")
     description = _("Shows all the finding types and their occurrences.")
-    plugins: ReportPlugins = {"required": [], "optional": []}
-    input_ooi_types = _INPUT_OOI_TYPES
+    plugins: ReportPlugins = {"required": set(), "optional": set()}
+    input_ooi_types = ALL_TYPES
     template_path = "findings_report/report.html"
     label_style = "3-light"
 

diff --git a/rocky/reports/report_types/helpers.py b/rocky/reports/report_types/helpers.py
@@ -1,7 +1,7 @@
 from octopoes.models import OOI, Reference
 from reports.report_types.aggregate_organisation_report.report import AggregateOrganisationReport
 from reports.report_types.concatenated_report.report import ConcatenatedReport
-from reports.report_types.definitions import AggregateReport, MultiReport, Report
+from reports.report_types.definitions import AggregateReport, BaseReport, Report
 from reports.report_types.dns_report.report import DNSReport
 from reports.report_types.findings_report.report import FindingsReport
 from reports.report_types.ipv6_report.report import IPv6Report
@@ -36,6 +36,8 @@
 
 CONCATENATED_REPORTS = [ConcatenatedReport]
 
+ALL_REPORT_TYPES = REPORTS + AGGREGATE_REPORTS + MULTI_REPORTS + CONCATENATED_REPORTS
+
 
 def get_ooi_types_with_report() -> set[type[OOI]]:
     """
@@ -44,7 +46,7 @@ def get_ooi_types_with_report() -> set[type[OOI]]:
     return {ooi_type for report in REPORTS for ooi_type in report.input_ooi_types}
 
 
-def get_report_types_for_ooi(ooi_pk: str) -> list[type[Report]]:
+def get_report_types_for_ooi(ooi_pk: str) -> list[type[BaseReport]]:
     """
     Get all report types that can be generated for a given OOI
     """
@@ -53,26 +55,26 @@ def get_report_types_for_ooi(ooi_pk: str) -> list[type[Report]]:
     return [report for report in REPORTS if ooi_type in report.input_ooi_types]
 
 
-def get_report_types_for_oois(ooi_pks: list[str]) -> set[type[Report]]:
+def get_report_types_for_oois(oois: list[str]) -> set[type[BaseReport]]:
     """
     Get all report types that can be generated for a given list of OOIs
     """
-    return {report for ooi_pk in ooi_pks for report in get_report_types_for_ooi(ooi_pk)}
+
+    return {report for ooi_pk in oois for report in get_report_types_for_ooi(ooi_pk)}
 
 
-def get_report_by_id(report_id: str) -> type[Report] | type[MultiReport] | type[AggregateReport]:
+def get_report_by_id(report_id: str) -> type[BaseReport]:
     """
     Get report type by id
     """
-    if report_id is None:
-        return ConcatenatedReport
-    for report in REPORTS + MULTI_REPORTS + AGGREGATE_REPORTS + CONCATENATED_REPORTS:
+
+    for report in ALL_REPORT_TYPES:
         if report.id == report_id:
             return report
     raise ValueError(f"Report with id {report_id} not found")
 
 
-def get_reports(report_ids: list[str]) -> list[type[Report] | type[MultiReport] | type[AggregateReport]]:
+def get_reports(report_ids: list[str]) -> list[type[BaseReport]]:
     return [get_report_by_id(report_id) for report_id in report_ids]
 
 

diff --git a/rocky/reports/report_types/ipv6_report/report.py b/rocky/reports/report_types/ipv6_report/report.py
@@ -20,7 +20,7 @@ class IPv6Report(Report):
     id = "ipv6-report"
     name = _("IPv6 Report")
     description = _("Check whether hostnames point to IPv6 addresses.")
-    plugins = {"required": ["dns-records"], "optional": []}
+    plugins = {"required": {"dns-records"}, "optional": set()}
     input_ooi_types = {Hostname, IPAddressV4, IPAddressV6}
     template_path = "ipv6_report/report.html"
     label_style = "4-light"

diff --git a/rocky/reports/report_types/mail_report/report.py b/rocky/reports/report_types/mail_report/report.py
@@ -15,7 +15,7 @@ class MailReport(Report):
     id = "mail-report"
     name = _("Mail Report")
     description = _("System specific Mail Report that focusses on IP addresses and hostnames.")
-    plugins = {"required": ["dns-records"], "optional": []}
+    plugins = {"required": {"dns-records"}, "optional": set()}
     input_ooi_types = {Hostname, IPAddressV4, IPAddressV6}
     template_path = "mail_report/report.html"
     label_style = "2-light"

diff --git a/rocky/reports/report_types/multi_organization_report/report.py b/rocky/reports/report_types/multi_organization_report/report.py
@@ -23,7 +23,7 @@ class MultiOrganizationReport(MultiReport):
     id = "multi-organization-report"
     name = _("Multi Organization Report")
     description = _("Multi Organization Report")
-    plugins: ReportPlugins = {"required": [], "optional": []}
+    plugins: ReportPlugins = {"required": set(), "optional": set()}
     input_ooi_types = {ReportData}
     template_path = "multi_organization_report/report.html"
 

diff --git a/rocky/reports/report_types/name_server_report/report.py b/rocky/reports/report_types/name_server_report/report.py
@@ -52,12 +52,12 @@ class NameServerSystemReport(Report):
     name = _("Name Server Report")
     description = _("Name Server Report checks name servers on basic security standards.")
     plugins = {
-        "required": [
+        "required": {
             "nmap",
             "dns-records",
             "dns-sec",
-        ],
-        "optional": [],
+        },
+        "optional": set(),
     }
     input_ooi_types = {Hostname, IPAddressV4, IPAddressV6}
     template_path = "name_server_report/report.html"