From e41cb86f8273992228ff00ae7733af502397c408 Mon Sep 17 00:00:00 2001 From: Rieven Date: Thu, 6 Feb 2025 15:02:54 +0100 Subject: [PATCH 1/2] Report Task List (#4059) Co-authored-by: Donny Peeters Co-authored-by: Madelon D Co-authored-by: JP Bruins Slot --- .../templates/tasks/report_tasks_list.html | 34 ++++ rocky/rocky/locale/django.pot | 60 ++++--- rocky/rocky/templates/tasks/boefjes.html | 10 +- rocky/rocky/templates/tasks/normalizers.html | 151 +++++++++--------- .../tasks/partials/tab_navigation.html | 5 + .../tasks/partials/tasks_overview_header.html | 12 ++ rocky/rocky/templates/tasks/report_tasks.html | 26 +++ rocky/rocky/urls.py | 2 + rocky/rocky/views/tasks.py | 40 ++++- rocky/tests/conftest.py | 41 ++++- rocky/tests/reports/test_report_tasks.py | 58 +++++++ 11 files changed, 332 insertions(+), 107 deletions(-) create mode 100644 rocky/reports/templates/tasks/report_tasks_list.html create mode 100644 rocky/rocky/templates/tasks/partials/tasks_overview_header.html create mode 100644 rocky/rocky/templates/tasks/report_tasks.html create mode 100644 rocky/tests/reports/test_report_tasks.py diff --git a/rocky/reports/templates/tasks/report_tasks_list.html b/rocky/reports/templates/tasks/report_tasks_list.html new file mode 100644 index 00000000000..0c19601ec97 --- /dev/null +++ b/rocky/reports/templates/tasks/report_tasks_list.html @@ -0,0 +1,34 @@ +{% load i18n %} +{% load static %} + +
+ + + + + + + + + + + + {% for report_task in report_task_list %} + {% with recipe_pk="ReportRecipe|"|add:report_task.data.report_recipe_id %} + + + + + + + {% endwith %} + {% endfor %} + +
{% translate "Status" %}{% translate "Recipe ID" %}{% translate "Creation date" %}{% translate "Modified date" %}
+ {{ report_task.status.value|capfirst }} + + {{ report_task.data.report_recipe_id }} + {{ report_task.created_at }}{{ report_task.modified_at }}
+ {% include "partials/list_paginator.html" %} + +
diff --git a/rocky/rocky/locale/django.pot b/rocky/rocky/locale/django.pot index 062b6562bdd..f7714e53631 100644 --- a/rocky/rocky/locale/django.pot +++ b/rocky/rocky/locale/django.pot @@ -9,7 +9,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2025-02-04 16:30+0000\n" +"POT-Creation-Date: 2025-02-06 13:51+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -1156,13 +1156,13 @@ msgstr "" #: katalogus/templates/partials/plugins_navigation.html #: rocky/templates/scan.html rocky/templates/tasks/boefjes.html -#: rocky/templates/tasks/partials/tab_navigation.html +#: rocky/templates/tasks/partials/tab_navigation.html rocky/views/tasks.py msgid "Boefjes" msgstr "" #: katalogus/templates/partials/plugins_navigation.html #: rocky/templates/tasks/normalizers.html -#: rocky/templates/tasks/partials/tab_navigation.html +#: rocky/templates/tasks/partials/tab_navigation.html rocky/views/tasks.py msgid "Normalizers" msgstr "" @@ -1218,6 +1218,7 @@ msgstr "" #: katalogus/templates/plugin_container_image.html #: reports/report_types/tls_report/report.html #: reports/templates/partials/plugin_overview_table.html +#: reports/templates/tasks/report_tasks_list.html #: rocky/templates/organizations/organization_member_list.html #: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html #: rocky/templates/tasks/ooi_detail_task_list.html @@ -1285,6 +1286,7 @@ msgstr "" #: reports/templates/report_overview/modal_partials/rerun_modal.html #: reports/templates/report_overview/report_history_table.html #: reports/templates/report_overview/subreports_table.html +#: reports/templates/tasks/report_tasks_list.html msgid "Creation date" msgstr "" @@ -1856,6 +1858,8 @@ msgstr "" #: onboarding/templates/step_2a_choose_report_info.html #: reports/templates/report_overview/report_overview_header.html #: reports/views/base.py rocky/templates/header.html +#: rocky/templates/tasks/partials/tab_navigation.html +#: rocky/templates/tasks/report_tasks.html rocky/views/tasks.py msgid "Reports" msgstr "" @@ -4216,6 +4220,7 @@ msgid "Showing %(length)s of %(total)s reports" msgstr "" #: reports/templates/report_overview/report_history_table.html +#: reports/templates/tasks/report_tasks_list.html msgid "Reports:" msgstr "" @@ -4425,6 +4430,15 @@ msgid "" "types." msgstr "" +#: reports/templates/tasks/report_tasks_list.html +msgid "Recipe ID" +msgstr "" + +#: reports/templates/tasks/report_tasks_list.html +#: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html +msgid "Modified date" +msgstr "" + #: reports/views/aggregate_report.py msgid "Aggregate report" msgstr "" @@ -5572,8 +5586,8 @@ msgstr "" msgid "Crisis room" msgstr "" -#: rocky/templates/header.html rocky/templates/tasks/boefjes.html -#: rocky/templates/tasks/ooi_detail_task_list.html +#: rocky/templates/header.html rocky/templates/tasks/ooi_detail_task_list.html +#: rocky/templates/tasks/partials/tasks_overview_header.html #: rocky/templates/tasks/plugin_detail_task_list.html #: rocky/views/task_detail.py rocky/views/tasks.py msgid "Tasks" @@ -6678,17 +6692,6 @@ msgstr "" msgid "Input object" msgstr "" -#: rocky/templates/tasks/boefjes.html -#, python-format -msgid "" -"\n" -" An overview of the tasks for %(organization)s. Tasks " -"are divided in Boefjes and Normalizers.\n" -" Boefjes scan objects and Normalizers dispatch on the " -"output mime-type.\n" -" " -msgstr "" - #: rocky/templates/tasks/boefjes.html msgid "There are no tasks for boefjes" msgstr "" @@ -6707,10 +6710,6 @@ msgstr "" msgid "Created date" msgstr "" -#: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html -msgid "Modified date" -msgstr "" - #: rocky/templates/tasks/normalizers.html msgid "There are no tasks for normalizers" msgstr "" @@ -6763,6 +6762,19 @@ msgstr "" msgid "Download task data" msgstr "" +#: rocky/templates/tasks/partials/tasks_overview_header.html +#, python-format +msgid "" +"\n" +" An overview of the tasks for %(organization)s. Tasks are divided " +"in Boefjes, Normalizers and Reports.\n" +" Boefjes scan objects and Normalizers dispatch on the output mime-" +"type. Additionally, there is a Report\n" +" tasks. This task aggregates and presents findings from both " +"Boefjes and Normalizers.\n" +" " +msgstr "" + #: rocky/templates/tasks/plugin_detail_task_list.html msgid "There are no tasks for" msgstr "" @@ -6771,6 +6783,14 @@ msgstr "" msgid "List of tasks for" msgstr "" +#: rocky/templates/tasks/report_tasks.html +msgid "There are no tasks for reports" +msgstr "" + +#: rocky/templates/tasks/report_tasks.html +msgid "List of tasks for reports" +msgstr "" + #: rocky/templates/two_factor/_wizard_actions.html msgid "Log in" msgstr "" diff --git a/rocky/rocky/templates/tasks/boefjes.html b/rocky/rocky/templates/tasks/boefjes.html index 527d064cbd7..c7da620277a 100644 --- a/rocky/rocky/templates/tasks/boefjes.html +++ b/rocky/rocky/templates/tasks/boefjes.html @@ -9,15 +9,7 @@
-
-

{% translate "Tasks" %}

-

- {% blocktranslate %} - An overview of the tasks for {{ organization }}. Tasks are divided in Boefjes and Normalizers. - Boefjes scan objects and Normalizers dispatch on the output mime-type. - {% endblocktranslate%} -

-
+ {% include "tasks/partials/tasks_overview_header.html" %} {% include "tasks/partials/tab_navigation.html" with view="boefjes_tasks" %} {% if not task_list %} diff --git a/rocky/rocky/templates/tasks/normalizers.html b/rocky/rocky/templates/tasks/normalizers.html index 4f1aba973a7..e92134d6a0b 100644 --- a/rocky/rocky/templates/tasks/normalizers.html +++ b/rocky/rocky/templates/tasks/normalizers.html @@ -10,91 +10,90 @@
-
- {% include "tasks/partials/tab_navigation.html" with view="normalizers_tasks" %} + {% include "tasks/partials/tasks_overview_header.html" %} + {% include "tasks/partials/tab_navigation.html" with view="normalizers_tasks" %} -

{% translate "Normalizers" %}

- {% if not task_list %} -

{% translate "There are no tasks for normalizers" %}

- {% include "tasks/partials/task_filter.html" %} +

{% translate "Normalizers" %}

+ {% if not task_list %} +

{% translate "There are no tasks for normalizers" %}

+ {% include "tasks/partials/task_filter.html" %} - {% else %} -

{% translate "List of tasks for normalizers" %}

- {% include "tasks/partials/task_filter.html" %} + {% else %} +

{% translate "List of tasks for normalizers" %}

+ {% include "tasks/partials/task_filter.html" %} -
- - - - {% if not organization.code %} - - {% endif %} - - - - - - - - - - - {% for task in task_list %} - - {% if not organization %} - - {% endif %} +
+
{% translate "Organization" %}{% translate "Normalizer" %}{% translate "Status" %}{% translate "Created date" %}{% translate "Modified date" %}{% translate "Boefje" %}{% translate "Boefje input OOI" %}
- {{ task.data.raw_data.boefje_meta.organization }} -
+ + + {% if not organization.code %} + + {% endif %} + + + + + + + + + + + {% for task in task_list %} + + {% if not organization %} - - - - + + + + - - - - - + + + + + - - {% endfor %} - -
{% translate "Organization" %}{% translate "Normalizer" %}{% translate "Status" %}{% translate "Created date" %}{% translate "Modified date" %}{% translate "Boefje" %}{% translate "Boefje input OOI" %}
- {{ task.data.normalizer.id }} - -  {{ task.status.value|capfirst }} + {{ task.data.raw_data.boefje_meta.organization }} {{ task.created_at }}{{ task.modified_at }} - {% if task.data.raw_data.boefje_meta.boefje.name %} - {% if task.data.raw_data.boefje_meta.boefje.id == "manual" %} -

{% translate "Manually added" %}

- {% else %} - {{ task.data.raw_data.boefje_meta.boefje.name }} - {% endif %} + {% endif %} + 		+ {{ task.data.normalizer.id }} + +  {{ task.status.value|capfirst }} + {{ task.created_at }}{{ task.modified_at }} + {% if task.data.raw_data.boefje_meta.boefje.name %} + {% if task.data.raw_data.boefje_meta.boefje.id == "manual" %} +

{% translate "Manually added" %}

{% else %} - {{ task.data.raw_data.boefje_meta.boefje.id }} - {% endif %} - 		- {% if task.data.raw_data.boefje_meta.input_ooi %} - {{ task.data.raw_data.boefje_meta.input_ooi }} + {{ task.data.raw_data.boefje_meta.boefje.name }} {% endif %} - - -
- {% include "tasks/partials/task_actions.html" %} + {% else %} + {{ task.data.raw_data.boefje_meta.boefje.id }} + {% endif %} + + {% if task.data.raw_data.boefje_meta.input_ooi %} + {{ task.data.raw_data.boefje_meta.input_ooi }} + {% endif %} + + +
+ {% include "tasks/partials/task_actions.html" %} -
- {% include "partials/list_paginator.html" %} + + + {% endfor %} + + + {% include "partials/list_paginator.html" %} -
- {% endif %} -
+ + {% endif %}
{% include "tasks/partials/stats.html" %} diff --git a/rocky/rocky/templates/tasks/partials/tab_navigation.html b/rocky/rocky/templates/tasks/partials/tab_navigation.html index 4825b8a65d6..4a54bda939a 100644 --- a/rocky/rocky/templates/tasks/partials/tab_navigation.html +++ b/rocky/rocky/templates/tasks/partials/tab_navigation.html @@ -17,6 +17,11 @@ {% translate "Normalizers" %} {% endif %} +
  • + {% if organization %} + {% translate "Reports" %} + {% endif %} +
    • diff --git a/rocky/rocky/templates/tasks/partials/tasks_overview_header.html b/rocky/rocky/templates/tasks/partials/tasks_overview_header.html new file mode 100644 index 00000000000..095448adf84 --- /dev/null +++ b/rocky/rocky/templates/tasks/partials/tasks_overview_header.html @@ -0,0 +1,12 @@ +{% load i18n %} + +
    +

    {% translate "Tasks" %}

    +

    + {% blocktranslate %} + An overview of the tasks for {{ organization }}. Tasks are divided in Boefjes, Normalizers and Reports. + Boefjes scan objects and Normalizers dispatch on the output mime-type. Additionally, there is a Report + tasks. This task aggregates and presents findings from both Boefjes and Normalizers. + {% endblocktranslate%} +

    +
    diff --git a/rocky/rocky/templates/tasks/report_tasks.html b/rocky/rocky/templates/tasks/report_tasks.html new file mode 100644 index 00000000000..803f8e28561 --- /dev/null +++ b/rocky/rocky/templates/tasks/report_tasks.html @@ -0,0 +1,26 @@ +{% extends "layouts/base.html" %} + +{% load i18n %} +{% load static %} +{% load ooi_extra %} +{% load compress %} + +{% block content %} + {% include "header.html" %} + +
    +
    + {% include "tasks/partials/tasks_overview_header.html" %} + {% include "tasks/partials/tab_navigation.html" with view="report_tasks" %} + +

    {% translate "Reports" %}

    + {% if not report_task_list %} +

    {% translate "There are no tasks for reports" %}

    + {% else %} +

    {% translate "List of tasks for reports" %}

    + {% include "tasks/report_tasks_list.html" %} + + {% endif %} +
    +
    +{% endblock content %} diff --git a/rocky/rocky/urls.py b/rocky/rocky/urls.py index a579c3c3410..a101d27f4dd 100644 --- a/rocky/rocky/urls.py +++ b/rocky/rocky/urls.py @@ -45,6 +45,7 @@ AllNormalizersTaskListView, BoefjesTaskListView, NormalizersTaskListView, + ReportsTaskListView, ) from rocky.views.upload_csv import UploadCSV from rocky.views.upload_raw import UploadRaw @@ -135,6 +136,7 @@ path("/tasks/boefjes", BoefjesTaskListView.as_view(), name="boefjes_task_list"), path("/tasks/boefjes/", BoefjeTaskDetailView.as_view(), name="boefje_task_view"), path("/tasks/normalizers", NormalizersTaskListView.as_view(), name="normalizers_task_list"), + path("/tasks/reports", ReportsTaskListView.as_view(), name="reports_task_list"), path( "/tasks/normalizers/", NormalizerTaskJSONView.as_view(), name="normalizer_task_view" ), diff --git a/rocky/rocky/views/tasks.py b/rocky/rocky/views/tasks.py index f6808296e16..be1df45ec56 100644 --- a/rocky/rocky/views/tasks.py +++ b/rocky/rocky/views/tasks.py @@ -52,6 +52,18 @@ class BoefjesTaskListView(TaskListView): template_name = "tasks/boefjes.html" task_type = "boefje" + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + + context["breadcrumbs"] = [ + {"url": reverse("task_list", kwargs={"organization_code": self.organization.code}), "text": _("Tasks")}, + { + "url": reverse("boefjes_task_list", kwargs={"organization_code": self.organization.code}), + "text": _("Boefjes"), + }, + ] + return context + class NormalizersTaskListView(TaskListView): template_name = "tasks/normalizers.html" @@ -59,9 +71,16 @@ class NormalizersTaskListView(TaskListView): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) + context["breadcrumbs"] = [ + {"url": reverse("task_list", kwargs={"organization_code": self.organization.code}), "text": _("Tasks")}, + { + "url": reverse("normalizers_task_list", kwargs={"organization_code": self.organization.code}), + "text": _("Normalizers"), + }, + ] # Search for the corresponding Boefje names and add those to the task_list - task_list = context["task_list"] + task_list = context.get("task_list", []) ids = [ task.data.raw_data.boefje_meta.boefje.id for task in task_list @@ -77,6 +96,25 @@ def get_context_data(self, **kwargs): return context +class ReportsTaskListView(TaskListView): + template_name = "tasks/report_tasks.html" + paginate_by = 25 + context_object_name = "report_task_list" + task_type = "report" + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + + context["breadcrumbs"] = [ + {"url": reverse("task_list", kwargs={"organization_code": self.organization.code}), "text": _("Tasks")}, + { + "url": reverse("reports_task_list", kwargs={"organization_code": self.organization.code}), + "text": _("Reports"), + }, + ] + return context + + class AllTaskListView(SchedulerListView, PageActionsView): paginator_class = RockyPaginator paginate_by = 20 diff --git a/rocky/tests/conftest.py b/rocky/tests/conftest.py index 5c72f84b462..f517231be85 100644 --- a/rocky/tests/conftest.py +++ b/rocky/tests/conftest.py @@ -44,7 +44,7 @@ from octopoes.models.tree import ReferenceTree from octopoes.models.types import OOIType from rocky.health import ServiceHealth -from rocky.scheduler import PaginatedTasksResponse, Task +from rocky.scheduler import PaginatedTasksResponse, ReportTask, Task, TaskStatus LANG_LIST = [code for code, _ in settings.LANGUAGES] @@ -1832,3 +1832,42 @@ def asset_report(name: str, report_type: str, template: str): ) return Paginated(count=1, items=[aggregate_report]) + + +@pytest.fixture +def reports_task_list(): + return PaginatedTasksResponse( + count=2, + next=None, + previous=None, + results=[ + Task( + id=UUID("7f9d5b00-dbab-45f3-93a6-dd44cc20c359"), + scheduler_id="report-_rieven", + schedule_id="86032b20-f7ae-4a48-9093-87ec5a56e939", + priority=1738747928, + status=TaskStatus.FAILED, + type="report", + hash="8f73ee4346118b7814711eba8ebb13d8", + data=ReportTask( + type="report", organisation_id="_rieven", report_recipe_id="3f5c1a46-1969-49b7-b402-4676fb59ca4b" + ), + created_at=datetime(2025, 2, 5, 9, 32, 8, 325523), + modified_at=datetime(2025, 2, 5, 9, 32, 8, 325526), + ), + Task( + id=UUID("9e23611d-36c2-4972-82f0-077bcb1a8941"), + scheduler_id="report-_rieven", + schedule_id="bd821e6e-6680-4215-8557-e049deeb0175", + priority=1738684879, + status=TaskStatus.COMPLETED, + type="report", + hash="5fc17aa4a8ff4874203446a106b4d5bb", + data=ReportTask( + type="report", organisation_id="_rieven", report_recipe_id="451a676d-91f8-4366-ac24-d1a47205181d" + ), + created_at=datetime(2025, 2, 4, 16, 1, 19, 951925), + modified_at=datetime(2025, 2, 4, 16, 1, 19, 951927), + ), + ], + ) diff --git a/rocky/tests/reports/test_report_tasks.py b/rocky/tests/reports/test_report_tasks.py new file mode 100644 index 00000000000..2d69f6d96b5 --- /dev/null +++ b/rocky/tests/reports/test_report_tasks.py @@ -0,0 +1,58 @@ +from pytest_django.asserts import assertContains + +from rocky.scheduler import SchedulerConnectError, SchedulerValidationError +from rocky.views.tasks import ReportsTaskListView +from tests.conftest import setup_request + + +def test_report_task_list(rf, client_member, mock_scheduler, reports_task_list): + """ + Test report task general page . + """ + + mock_scheduler.list_tasks.return_value = reports_task_list + + recipe_ids = [report_task.data.report_recipe_id for report_task in reports_task_list.results] + + response = ReportsTaskListView.as_view()( + setup_request(rf.get("reports_task_list"), client_member.user), + organization_code=client_member.organization.code, + ) + + assert response.status_code == 200 + + assertContains(response, "List of tasks for reports") + assertContains(response, 'Failed', html=True) + assertContains(response, 'Completed', html=True) + assertContains(response, recipe_ids[0]) + assertContains(response, recipe_ids[1]) + + +def test_report_task_list_connect_error(rf, client_member, mock_scheduler): + """ + Test report task general page . + """ + + mock_scheduler.list_tasks.side_effect = SchedulerConnectError + + request = setup_request(rf.get("reports_task_list"), client_member.user) + + response = ReportsTaskListView.as_view()(request, organization_code=client_member.organization.code) + + assert response.status_code == 200 + assert list(request._messages)[0].message == "Could not connect to Scheduler. Service is possibly down." + + +def test_report_task_list_validation_error(rf, client_member, mock_scheduler): + """ + Test report task general page . + """ + + mock_scheduler.list_tasks.side_effect = SchedulerValidationError + + request = setup_request(rf.get("reports_task_list"), client_member.user) + + response = ReportsTaskListView.as_view()(request, organization_code=client_member.organization.code) + + assert response.status_code == 200 + assert list(request._messages)[0].message == "Your request could not be validated." From b67e88de436bfab5102f2336053320c83bdd4f84 Mon Sep 17 00:00:00 2001 From: JP Bruins Slot Date: Thu, 6 Feb 2025 15:09:59 +0100 Subject: [PATCH 2/2] Add one-off jobs for report scheduler (#4045) Co-authored-by: Rieven Co-authored-by: Jan Klopper Co-authored-by: stephanie0x00 <9821756+stephanie0x00@users.noreply.github.com> --- mula/scheduler/schedulers/scheduler.py | 17 +++++---- .../scheduler/schedulers/schedulers/report.py | 34 ----------------- mula/scheduler/storage/stores/schedule.py | 2 +- mula/tests/integration/test_scheduler.py | 37 +++++++++++++++++++ 4 files changed, 48 insertions(+), 42 deletions(-) diff --git a/mula/scheduler/schedulers/scheduler.py b/mula/scheduler/schedulers/scheduler.py index b5ec2f07182..a9115abbc9d 100644 --- a/mula/scheduler/schedulers/scheduler.py +++ b/mula/scheduler/schedulers/scheduler.py @@ -342,21 +342,22 @@ def post_push(self, item: models.Task, create_schedule: bool = True) -> models.T item.schedule_id = schedule_db.id - # Set the cron schedule based on the item, default this is None. - # We do this because we want to explicitly set the cron schedule. When - # a schedule already has a cron expression, this will not be updated - # unless this is specifically overridden in a subclass. + # Determine the cron expression, either from the overridden set_cron() + # or explicitly set. cron_expr = self.set_cron(item) if cron_expr is not None: schedule_db.schedule = cron_expr - # If the schedule has a cron schedule, we calculate the next run - # based on the cron schedule, otherwise we calculate the deadline - # based on the item. + # When a Schedule does not have a schedule (cron expression), we + # calculate the deadline when a Schedules specified a way to calculate + # this. Otherwise we set the deadline to None make sure the Schedule + # will not continue to be processed. if schedule_db.schedule is not None: schedule_db.deadline_at = cron.next_run(schedule_db.schedule) elif self.auto_calculate_deadline: schedule_db.deadline_at = self.calculate_deadline(item) + else: + schedule_db.deadline_at = None self.ctx.datastores.schedule_store.update_schedule(schedule_db) self.ctx.datastores.task_store.update_task(item) @@ -418,6 +419,8 @@ def set_cron(self, task: models.Task) -> str | None: return None def calculate_deadline(self, task: models.Task) -> datetime: + """The default deadline calculation for a task, when the + auto_calculate_deadline attribute is set to True""" # We at least delay a job by the grace period minimum = self.ctx.config.pq_grace_period deadline = datetime.now(timezone.utc) + timedelta(seconds=minimum) diff --git a/mula/scheduler/schedulers/schedulers/report.py b/mula/scheduler/schedulers/schedulers/report.py index 4fa4189c58c..05f4d5d3e43 100644 --- a/mula/scheduler/schedulers/schedulers/report.py +++ b/mula/scheduler/schedulers/schedulers/report.py @@ -87,40 +87,6 @@ def push_tasks_for_rescheduling(self): ) as executor: for schedule in schedules: report_task = ReportTask.model_validate(schedule.data) - - # When the schedule has no schedule (cron expression), but a - # task is already executed for this schedule we should not run - # the task again - if schedule.schedule is None: - try: - _, count = self.ctx.datastores.task_store.get_tasks( - scheduler_id=self.scheduler_id, - task_type=report_task.type, - filters=filters.FilterRequest( - filters=[ - filters.Filter(column="hash", operator="eq", value=report_task.hash), - filters.Filter(column="schedule_id", operator="eq", value=str(schedule.id)), - ] - ), - ) - if count > 0: - self.logger.debug( - "Schedule has no schedule, but task already executed", - schedule_id=schedule.id, - scheduler_id=self.scheduler_id, - organisation_id=self.organisation.id, - ) - continue - except storage.errors.StorageError as exc_db: - self.logger.error( - "Could not get latest task by hash %s", - report_task.hash, - scheduler_id=self.scheduler_id, - organisation_id=self.organisation.id, - exc_info=exc_db, - ) - continue - executor.submit(self.push_report_task, report_task, self.push_tasks_for_rescheduling.__name__) def push_report_task(self, report_task: ReportTask, caller: str = "") -> None: diff --git a/mula/scheduler/storage/stores/schedule.py b/mula/scheduler/storage/stores/schedule.py index a91b680d03b..e8d1881dac2 100644 --- a/mula/scheduler/storage/stores/schedule.py +++ b/mula/scheduler/storage/stores/schedule.py @@ -21,7 +21,7 @@ def get_schedules( self, scheduler_id: str | None = None, schedule_hash: str | None = None, - enabled: bool | None = True, # FIXME: None? + enabled: bool | None = None, min_deadline_at: datetime | None = None, max_deadline_at: datetime | None = None, min_created_at: datetime | None = None, diff --git a/mula/tests/integration/test_scheduler.py b/mula/tests/integration/test_scheduler.py index 7483f5b9880..aecda637a09 100644 --- a/mula/tests/integration/test_scheduler.py +++ b/mula/tests/integration/test_scheduler.py @@ -314,6 +314,43 @@ def test_post_push_schedule_is_not_none(self): datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0) + timedelta(days=1), ) + def test_post_push_schedule_is_none(self): + """When a schedule is not provided, the deadline should be set to None""" + # Arrange + first_item = functions.create_item(scheduler_id=self.scheduler.scheduler_id, priority=1) + + schedule = models.Schedule(scheduler_id=self.scheduler.scheduler_id, hash=first_item.hash, data=first_item.data) + schedule_db = self.mock_ctx.datastores.schedule_store.create_schedule(schedule) + + first_item.schedule_id = schedule_db.id + self.mock_ctx.datastores.task_store.update_task(first_item) + + # Act + self.scheduler.push_item_to_queue(first_item) + + # Assert: + self.assertIsNone(schedule_db.deadline_at) + + def test_post_push_schedule_auto_calculate_deadline(self): + """When a schedule is not provided, and auto_calculate_deadline is True, the deadline should be set""" + # Arrange + self.scheduler.auto_calculate_deadline = True + + first_item = functions.create_item(scheduler_id=self.scheduler.scheduler_id, priority=1) + + schedule = models.Schedule(scheduler_id=self.scheduler.scheduler_id, hash=first_item.hash, data=first_item.data) + schedule_db = self.mock_ctx.datastores.schedule_store.create_schedule(schedule) + + first_item.schedule_id = schedule_db.id + self.mock_ctx.datastores.task_store.update_task(first_item) + + # Act + self.scheduler.push_item_to_queue(first_item) + + # Assert: Check if the deadline_at is set correctly + schedule_db_updated = self.mock_ctx.datastores.schedule_store.get_schedule(first_item.schedule_id) + self.assertIsNotNone(schedule_db_updated.deadline_at) + def test_post_pop(self): """When a task is popped from the queue, it should be removed from the database""" # Arrange