Merge branch 'main' into remove-keiko

mula/scheduler/schedulers/scheduler.py

@@ -342,21 +342,22 @@ def post_push(self, item: models.Task, create_schedule: bool = True) -> models.T
 
         item.schedule_id = schedule_db.id
 
-        # Set the cron schedule based on the item, default this is None.
-        # We do this because we want to explicitly set the cron schedule. When
-        # a schedule already has a cron expression, this will not be updated
-        # unless this is specifically overridden in a subclass.
+        # Determine the cron expression, either from the overridden set_cron()
+        # or explicitly set.
         cron_expr = self.set_cron(item)
         if cron_expr is not None:
             schedule_db.schedule = cron_expr
 
-        # If the schedule has a cron schedule, we calculate the next run
-        # based on the cron schedule, otherwise we calculate the deadline
-        # based on the item.
+        # When a Schedule does not have a schedule (cron expression), we
+        # calculate the deadline when a Schedules specified a way to calculate
+        # this. Otherwise we set the deadline to None make sure the Schedule
+        # will not continue to be processed.
         if schedule_db.schedule is not None:
             schedule_db.deadline_at = cron.next_run(schedule_db.schedule)
         elif self.auto_calculate_deadline:
             schedule_db.deadline_at = self.calculate_deadline(item)
+        else:
+            schedule_db.deadline_at = None
 
         self.ctx.datastores.schedule_store.update_schedule(schedule_db)
         self.ctx.datastores.task_store.update_task(item)
@@ -418,6 +419,8 @@ def set_cron(self, task: models.Task) -> str | None:
         return None
 
     def calculate_deadline(self, task: models.Task) -> datetime:
+        """The default deadline calculation for a task, when the
+        auto_calculate_deadline attribute is set to True"""
         # We at least delay a job by the grace period
         minimum = self.ctx.config.pq_grace_period
         deadline = datetime.now(timezone.utc) + timedelta(seconds=minimum)

mula/scheduler/schedulers/schedulers/report.py

@@ -87,40 +87,6 @@ def push_tasks_for_rescheduling(self):
         ) as executor:
             for schedule in schedules:
                 report_task = ReportTask.model_validate(schedule.data)
-
-                # When the schedule has no schedule (cron expression), but a
-                # task is already executed for this schedule we should not run
-                # the task again
-                if schedule.schedule is None:
-                    try:
-                        _, count = self.ctx.datastores.task_store.get_tasks(
-                            scheduler_id=self.scheduler_id,
-                            task_type=report_task.type,
-                            filters=filters.FilterRequest(
-                                filters=[
-                                    filters.Filter(column="hash", operator="eq", value=report_task.hash),
-                                    filters.Filter(column="schedule_id", operator="eq", value=str(schedule.id)),
-                                ]
-                            ),
-                        )
-                        if count > 0:
-                            self.logger.debug(
-                                "Schedule has no schedule, but task already executed",
-                                schedule_id=schedule.id,
-                                scheduler_id=self.scheduler_id,
-                                organisation_id=self.organisation.id,
-                            )
-                            continue
-                    except storage.errors.StorageError as exc_db:
-                        self.logger.error(
-                            "Could not get latest task by hash %s",
-                            report_task.hash,
-                            scheduler_id=self.scheduler_id,
-                            organisation_id=self.organisation.id,
-                            exc_info=exc_db,
-                        )
-                        continue
-
                 executor.submit(self.push_report_task, report_task, self.push_tasks_for_rescheduling.__name__)
 
     def push_report_task(self, report_task: ReportTask, caller: str = "") -> None:

mula/scheduler/storage/stores/schedule.py

@@ -21,7 +21,7 @@ def get_schedules(
         self,
         scheduler_id: str | None = None,
         schedule_hash: str | None = None,
-        enabled: bool | None = True,  # FIXME: None?
+        enabled: bool | None = None,
         min_deadline_at: datetime | None = None,
         max_deadline_at: datetime | None = None,
         min_created_at: datetime | None = None,

mula/tests/integration/test_scheduler.py

@@ -314,6 +314,43 @@ def test_post_push_schedule_is_not_none(self):
             datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0) + timedelta(days=1),
         )
 
+    def test_post_push_schedule_is_none(self):
+        """When a schedule is not provided, the deadline should be set to None"""
+        # Arrange
+        first_item = functions.create_item(scheduler_id=self.scheduler.scheduler_id, priority=1)
+
+        schedule = models.Schedule(scheduler_id=self.scheduler.scheduler_id, hash=first_item.hash, data=first_item.data)
+        schedule_db = self.mock_ctx.datastores.schedule_store.create_schedule(schedule)
+
+        first_item.schedule_id = schedule_db.id
+        self.mock_ctx.datastores.task_store.update_task(first_item)
+
+        # Act
+        self.scheduler.push_item_to_queue(first_item)
+
+        # Assert:
+        self.assertIsNone(schedule_db.deadline_at)
+
+    def test_post_push_schedule_auto_calculate_deadline(self):
+        """When a schedule is not provided, and auto_calculate_deadline is True, the deadline should be set"""
+        # Arrange
+        self.scheduler.auto_calculate_deadline = True
+
+        first_item = functions.create_item(scheduler_id=self.scheduler.scheduler_id, priority=1)
+
+        schedule = models.Schedule(scheduler_id=self.scheduler.scheduler_id, hash=first_item.hash, data=first_item.data)
+        schedule_db = self.mock_ctx.datastores.schedule_store.create_schedule(schedule)
+
+        first_item.schedule_id = schedule_db.id
+        self.mock_ctx.datastores.task_store.update_task(first_item)
+
+        # Act
+        self.scheduler.push_item_to_queue(first_item)
+
+        # Assert: Check if the deadline_at is set correctly
+        schedule_db_updated = self.mock_ctx.datastores.schedule_store.get_schedule(first_item.schedule_id)
+        self.assertIsNotNone(schedule_db_updated.deadline_at)
+
     def test_post_pop(self):
         """When a task is popped from the queue, it should be removed from the database"""
         # Arrange

rocky/reports/templates/tasks/report_tasks_list.html

@@ -0,0 +1,34 @@
+{% load i18n %}
+{% load static %}
+
+<div class="horizontal-scroll sticky-column">
+    <table class="nowrap">
+        <caption class="visually-hidden">{% translate "Reports:" %}</caption>
+        <thead>
+            <tr>
+                <th scope="col" class="nowrap">{% translate "Status" %}</th>
+                <th scope="col">{% translate "Recipe ID" %}</th>
+                <th scope="col" class="nowrap">{% translate "Creation date" %}</th>
+                <th scope="col" class="nowrap">{% translate "Modified date" %}</th>
+            </tr>
+        </thead>
+        <tbody>
+            {% for report_task in report_task_list %}
+                {% with recipe_pk="ReportRecipe|"|add:report_task.data.report_recipe_id %}
+                    <tr>
+                        <td>
+                            <i class="icon {{ report_task.status.value }}"></i>{{ report_task.status.value|capfirst }}
+                        </td>
+                        <td>
+                            <a href="{% ooi_url 'ooi_detail' recipe_pk organization.code query=ooi.mandatory_fields %}">{{ report_task.data.report_recipe_id }}</a>
+                        </td>
+                        <td>{{ report_task.created_at }}</td>
+                        <td>{{ report_task.modified_at }}</td>
+                    </tr>
+                {% endwith %}
+            {% endfor %}
+        </tbody>
+    </table>
+    {% include "partials/list_paginator.html" %}
+
+</div>

rocky/rocky/locale/django.pot

@@ -9,7 +9,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2025-02-04 16:30+0000\n"
+"POT-Creation-Date: 2025-02-06 13:51+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <[email protected]>\n"
@@ -1156,13 +1156,13 @@ msgstr ""
 
 #: katalogus/templates/partials/plugins_navigation.html
 #: rocky/templates/scan.html rocky/templates/tasks/boefjes.html
-#: rocky/templates/tasks/partials/tab_navigation.html
+#: rocky/templates/tasks/partials/tab_navigation.html rocky/views/tasks.py
 msgid "Boefjes"
 msgstr ""
 
 #: katalogus/templates/partials/plugins_navigation.html
 #: rocky/templates/tasks/normalizers.html
-#: rocky/templates/tasks/partials/tab_navigation.html
+#: rocky/templates/tasks/partials/tab_navigation.html rocky/views/tasks.py
 msgid "Normalizers"
 msgstr ""
 
@@ -1218,6 +1218,7 @@ msgstr ""
 #: katalogus/templates/plugin_container_image.html
 #: reports/report_types/tls_report/report.html
 #: reports/templates/partials/plugin_overview_table.html
+#: reports/templates/tasks/report_tasks_list.html
 #: rocky/templates/organizations/organization_member_list.html
 #: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html
 #: rocky/templates/tasks/ooi_detail_task_list.html
@@ -1285,6 +1286,7 @@ msgstr ""
 #: reports/templates/report_overview/modal_partials/rerun_modal.html
 #: reports/templates/report_overview/report_history_table.html
 #: reports/templates/report_overview/subreports_table.html
+#: reports/templates/tasks/report_tasks_list.html
 msgid "Creation date"
 msgstr ""
 
@@ -1856,6 +1858,8 @@ msgstr ""
 #: onboarding/templates/step_2a_choose_report_info.html
 #: reports/templates/report_overview/report_overview_header.html
 #: reports/views/base.py rocky/templates/header.html
+#: rocky/templates/tasks/partials/tab_navigation.html
+#: rocky/templates/tasks/report_tasks.html rocky/views/tasks.py
 msgid "Reports"
 msgstr ""
 
@@ -4216,6 +4220,7 @@ msgid "Showing %(length)s of %(total)s reports"
 msgstr ""
 
 #: reports/templates/report_overview/report_history_table.html
+#: reports/templates/tasks/report_tasks_list.html
 msgid "Reports:"
 msgstr ""
 
@@ -4425,6 +4430,15 @@ msgid ""
 "types."
 msgstr ""
 
+#: reports/templates/tasks/report_tasks_list.html
+msgid "Recipe ID"
+msgstr ""
+
+#: reports/templates/tasks/report_tasks_list.html
+#: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html
+msgid "Modified date"
+msgstr ""
+
 #: reports/views/aggregate_report.py
 msgid "Aggregate report"
 msgstr ""
@@ -5572,8 +5586,8 @@ msgstr ""
 msgid "Crisis room"
 msgstr ""
 
-#: rocky/templates/header.html rocky/templates/tasks/boefjes.html
-#: rocky/templates/tasks/ooi_detail_task_list.html
+#: rocky/templates/header.html rocky/templates/tasks/ooi_detail_task_list.html
+#: rocky/templates/tasks/partials/tasks_overview_header.html
 #: rocky/templates/tasks/plugin_detail_task_list.html
 #: rocky/views/task_detail.py rocky/views/tasks.py
 msgid "Tasks"
@@ -6678,17 +6692,6 @@ msgstr ""
 msgid "Input object"
 msgstr ""
 
-#: rocky/templates/tasks/boefjes.html
-#, python-format
-msgid ""
-"\n"
-"                        An overview of the tasks for %(organization)s. Tasks "
-"are divided in Boefjes and Normalizers.\n"
-"                        Boefjes scan objects and Normalizers dispatch on the "
-"output mime-type.\n"
-"                    "
-msgstr ""
-
 #: rocky/templates/tasks/boefjes.html
 msgid "There are no tasks for boefjes"
 msgstr ""
@@ -6707,10 +6710,6 @@ msgstr ""
 msgid "Created date"
 msgstr ""
 
-#: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html
-msgid "Modified date"
-msgstr ""
-
 #: rocky/templates/tasks/normalizers.html
 msgid "There are no tasks for normalizers"
 msgstr ""
@@ -6763,6 +6762,19 @@ msgstr ""
 msgid "Download task data"
 msgstr ""
 
+#: rocky/templates/tasks/partials/tasks_overview_header.html
+#, python-format
+msgid ""
+"\n"
+"            An overview of the tasks for %(organization)s. Tasks are divided "
+"in Boefjes, Normalizers and Reports.\n"
+"            Boefjes scan objects and Normalizers dispatch on the output mime-"
+"type. Additionally, there is a Report\n"
+"            tasks. This task aggregates and presents findings from both "
+"Boefjes and Normalizers.\n"
+"        "
+msgstr ""
+
 #: rocky/templates/tasks/plugin_detail_task_list.html
 msgid "There are no tasks for"
 msgstr ""
@@ -6771,6 +6783,14 @@ msgstr ""
 msgid "List of tasks for"
 msgstr ""
 
+#: rocky/templates/tasks/report_tasks.html
+msgid "There are no tasks for reports"
+msgstr ""
+
+#: rocky/templates/tasks/report_tasks.html
+msgid "List of tasks for reports"
+msgstr ""
+
 #: rocky/templates/two_factor/_wizard_actions.html
 msgid "Log in"
 msgstr ""

rocky/rocky/templates/tasks/boefjes.html

@@ -9,15 +9,7 @@
 
     <main id="main-content">
         <section>
-            <div>
-                <h1>{% translate "Tasks" %}</h1>
-                <p class="emphasized">
-                    {% blocktranslate %}
-                        An overview of the tasks for {{ organization }}. Tasks are divided in Boefjes and Normalizers.
-                        Boefjes scan objects and Normalizers dispatch on the output mime-type.
-                    {% endblocktranslate%}
-                </p>
-            </div>
+            {% include "tasks/partials/tasks_overview_header.html" %}
             {% include "tasks/partials/tab_navigation.html" with view="boefjes_tasks" %}
 
             {% if not task_list %}