From c6e136be5c9a63d49182840420a5e2158703e4d5 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Mon, 30 Jan 2023 11:49:18 +0000 Subject: [PATCH 01/34] add custom filter to check whether the date is in the past, use it in toc.html --- portality/app.py | 7 ++++++- portality/templates/doaj/toc.html | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/portality/app.py b/portality/app.py index 9e890ed970..9f3a674a55 100644 --- a/portality/app.py +++ b/portality/app.py @@ -17,7 +17,7 @@ from flask import request, abort, render_template, redirect, send_file, url_for, jsonify, send_from_directory from flask_login import login_user, current_user -from datetime import datetime +from datetime import datetime, date import portality.models as models from portality.core import app, es_connection, initialise_index @@ -279,6 +279,11 @@ def form_diff_table_subject_expand(val): return ", ".join(results) +@app.template_filter("is_in_the_past") +def is_in_the_past(dttm): + date = datetime.strptime(dttm, "%Y-%m-%d").date() + return date <= date.today() + ####################################################### diff --git a/portality/templates/doaj/toc.html b/portality/templates/doaj/toc.html index 5c12d2dc58..27b71c7581 100644 --- a/portality/templates/doaj/toc.html +++ b/portality/templates/doaj/toc.html @@ -84,7 +84,7 @@

- {% if bibjson.discontinued_date %} + {% if bibjson.discontinued_date | is_in_the_past %}

Ceased publication on {{ bibjson.discontinued_datestamp.strftime("%d %B %Y") }}

{% endif %} From 8e200bedd13a89a89134d68ebe8add80bd1bc4e6 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Tue, 31 Jan 2023 14:20:42 +0000 Subject: [PATCH 02/34] add background job --- portality/settings.py | 2 + portality/tasks/find_discontinued_soon.py | 113 ++++++++++++++++++ .../templates/email/discontinue_soon.jinja2 | 7 ++ 3 files changed, 122 insertions(+) create mode 100644 portality/tasks/find_discontinued_soon.py create mode 100644 portality/templates/email/discontinue_soon.jinja2 diff --git a/portality/settings.py b/portality/settings.py index 76bd18fbaa..7896d33ac3 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -1354,3 +1354,5 @@ } } +# report journals that discontinue in ... days (eg. 1 = tomorrow) +DISCONTINUED_DATE_DELTA = 1 \ No newline at end of file diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py new file mode 100644 index 0000000000..7ff598d79a --- /dev/null +++ b/portality/tasks/find_discontinued_soon.py @@ -0,0 +1,113 @@ +import datetime +import csv +import json + +from portality.core import app +from portality import models,app_email + +from portality.background import BackgroundTask + + +def _date(): + return (datetime.datetime.today() + datetime.timedelta(days=app.config.get('DISCONTINUED_DATE_DELTA', 1))).strftime( + '%Y-%m-%d') +class DiscontinuedSoonQuery: + @classmethod + def query(cls): + return { + "query": { + "bool": { + "filter": { + "bool" : { + "must": [ + {"term" : {"bibjson.discontinued_date": _date()}} + ] + } + } + } + } + } + +# ~~FindDiscontinuedSoonBackgroundTask:Task~~ + +class FindDiscontinuedSoonBackgroundTask(BackgroundTask): + __action__ = "find_discontinued_soon" + + def run(self): + jdata = [] + + for journal in models.Journal.iterate(q=DiscontinuedSoonQuery.query(), keepalive='5m', wrap=True): + bibjson = journal.bibjson() + owner = journal.owner + account = models.Account.pull(owner) + + jdata.append({"id": journal.id, + "title":bibjson.title, + "eissn": bibjson.get_one_identifier(bibjson.E_ISSN), + "pissn": bibjson.get_one_identifier(bibjson.P_ISSN), + "account_email": account.email if account else "Not Found", + "publisher": bibjson.publisher}) + + try: + # send warning email about the service tag in article metadata detected + to = app.config.get('SCRIPT_TAG_DETECTED_EMAIL_RECIPIENTS') + fro = app.config.get("SYSTEM_EMAIL_FROM", "helpdesk@doaj.org") + subject = app.config.get("SERVICE_NAME", "") + " - script tag detected in application metadata" + es_type = "application" + app_email.send_mail(to=to, + fro=fro, + subject=subject, + template_name="email/discontinue_soon.jinja2", + es_type=es_type, + days=app.config.get('DISCONTINUED_DATE_DELTA',1), + data=json.dumps({"data": jdata}, indent=4, separators=(',', ': '))) + except app_email.EmailException: + app.logger.exception('Error sending email with journals discountinuing soon - ' + jdata) + + def cleanup(self): + """ + Cleanup after a successful OR failed run of the task + :return: + """ + pass + + @classmethod + def prepare(cls, username, **kwargs): + """ + Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob, + or fail with a suitable exception + + :param kwargs: arbitrary keyword arguments pertaining to this task type + :return: a BackgroundJob instance representing this task + """ + + # first prepare a job record + job = background_helper.create_job(username, cls.__action__, + queue_id=huey_helper.queue_id, ) + return job + + @classmethod + def submit(cls, background_job): + """ + Submit the specified BackgroundJob to the background queue + + :param background_job: the BackgroundJob instance + :return: + """ + background_job.save() + request_es_backup.schedule(args=(background_job.id,), delay=1) + +huey_helper = RequestESBackupBackgroundTask.create_huey_helper(main_queue) + +@huey_helper.register_schedule +def scheduled_request_es_backup(): + user = app.config.get("SYSTEM_USERNAME") + job = FindDiscontinuedSoonBackgroundTask.prepare(user) + FindDiscontinuedSoonBackgroundTask.submit(job) + + +@huey_helper.register_execute(is_load_config=False) +def request_es_backup(job_id): + job = models.BackgroundJob.pull(job_id) + task = FindDiscontinuedSoonBackgroundTask(job) + BackgroundApi.execute(task) diff --git a/portality/templates/email/discontinue_soon.jinja2 b/portality/templates/email/discontinue_soon.jinja2 new file mode 100644 index 0000000000..25debb60a8 --- /dev/null +++ b/portality/templates/email/discontinue_soon.jinja2 @@ -0,0 +1,7 @@ +{# +~~FindDiscontinuedSoonBackgroundTask:Email~~ +#} + +Following journals will discontinue in {{ days }} days. + +{{ data }} \ No newline at end of file From d5e1608f69437b57cf2808ea8c2fd66e6c9ed8a6 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Tue, 31 Jan 2023 14:23:05 +0000 Subject: [PATCH 03/34] copy-paste mistakes fixed --- portality/tasks/find_discontinued_soon.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index 7ff598d79a..c691b0c54f 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -95,19 +95,19 @@ def submit(cls, background_job): :return: """ background_job.save() - request_es_backup.schedule(args=(background_job.id,), delay=1) + find_discontinued_soon.schedule(args=(background_job.id,), delay=1) huey_helper = RequestESBackupBackgroundTask.create_huey_helper(main_queue) @huey_helper.register_schedule -def scheduled_request_es_backup(): +def scheduled_find_discontinued_soon(): user = app.config.get("SYSTEM_USERNAME") job = FindDiscontinuedSoonBackgroundTask.prepare(user) FindDiscontinuedSoonBackgroundTask.submit(job) @huey_helper.register_execute(is_load_config=False) -def request_es_backup(job_id): +def find_discontinued_soon(job_id): job = models.BackgroundJob.pull(job_id) task = FindDiscontinuedSoonBackgroundTask(job) BackgroundApi.execute(task) From 8c173c3df942ee1abb118a28afd00b48ba3f0302 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Tue, 31 Jan 2023 14:25:21 +0000 Subject: [PATCH 04/34] change delay to seconds --- portality/tasks/request_es_backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/tasks/request_es_backup.py b/portality/tasks/request_es_backup.py index 57a077a1fd..438d437490 100644 --- a/portality/tasks/request_es_backup.py +++ b/portality/tasks/request_es_backup.py @@ -73,7 +73,7 @@ def submit(cls, background_job): :return: """ background_job.save() - request_es_backup.schedule(args=(background_job.id,), delay=10) + request_es_backup.schedule(args=(background_job.id,), delay=86400) huey_helper = RequestESBackupBackgroundTask.create_huey_helper(main_queue) From 7edc4c20133e93138a3c4421171f2b5048fc1d79 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Tue, 31 Jan 2023 14:26:07 +0000 Subject: [PATCH 05/34] wrong file edited! --- portality/tasks/find_discontinued_soon.py | 2 +- portality/tasks/request_es_backup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index c691b0c54f..c6c1e24a99 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -95,7 +95,7 @@ def submit(cls, background_job): :return: """ background_job.save() - find_discontinued_soon.schedule(args=(background_job.id,), delay=1) + find_discontinued_soon.schedule(args=(background_job.id,), delay=86400) huey_helper = RequestESBackupBackgroundTask.create_huey_helper(main_queue) diff --git a/portality/tasks/request_es_backup.py b/portality/tasks/request_es_backup.py index 438d437490..57a077a1fd 100644 --- a/portality/tasks/request_es_backup.py +++ b/portality/tasks/request_es_backup.py @@ -73,7 +73,7 @@ def submit(cls, background_job): :return: """ background_job.save() - request_es_backup.schedule(args=(background_job.id,), delay=86400) + request_es_backup.schedule(args=(background_job.id,), delay=10) huey_helper = RequestESBackupBackgroundTask.create_huey_helper(main_queue) From c88549b1f028a0d24b32995604dc25ef2deb3176 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Wed, 1 Feb 2023 13:17:21 +0000 Subject: [PATCH 06/34] add UT and fix code --- doajtest/unit/test_task_discontinued_soon.py | 65 ++++++++++++++++++++ portality/settings.py | 1 + portality/tasks/find_discontinued_soon.py | 48 +++++++++------ 3 files changed, 95 insertions(+), 19 deletions(-) create mode 100644 doajtest/unit/test_task_discontinued_soon.py diff --git a/doajtest/unit/test_task_discontinued_soon.py b/doajtest/unit/test_task_discontinued_soon.py new file mode 100644 index 0000000000..77e9c558c5 --- /dev/null +++ b/doajtest/unit/test_task_discontinued_soon.py @@ -0,0 +1,65 @@ +import unittest +import datetime + +from doajtest.helpers import DoajTestCase + +from portality.core import app +from portality import models +from portality.tasks import find_discontinued_soon +from doajtest.fixtures import JournalFixtureFactory + +class TestDiscontinuedSoon(DoajTestCase): + + def _date_tomorrow(self): + return (datetime.datetime.today() + datetime.timedelta(days=1)).strftime('%Y-%m-%d') + + def _date_in_2_days(self): + return (datetime.datetime.today() + datetime.timedelta(days=2)).strftime('%Y-%m-%d') + + def test_discontinued_soon_found(self): + + journal_discontinued_tommorow_1 = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True)) + journal_discontinued_tommorow_1.set_id("1") + jbib = journal_discontinued_tommorow_1.bibjson() + jbib.title = "Discontinued Tomorrow 1" + jbib.discontinued_date = self._date_tomorrow() + journal_discontinued_tommorow_1.save(blocking=True) + + journal_discontinued_tommorow_2 = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True)) + journal_discontinued_tommorow_2.set_id("2") + jbib = journal_discontinued_tommorow_2.bibjson() + jbib.title = "Discontinued Tomorrow 2" + jbib.discontinued_date = self._date_tomorrow() + journal_discontinued_tommorow_2.save(blocking=True) + + journal_discontinued_in_2_days = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True)) + journal_discontinued_in_2_days.set_id("3") + jbib = journal_discontinued_in_2_days.bibjson() + jbib.title = "Discontinued In 2 days" + jbib.discontinued_date = self._date_in_2_days() + journal_discontinued_in_2_days.save(blocking=True) + + job = find_discontinued_soon.FindDiscontinuedSoonBackgroundTask.prepare("system") + task = find_discontinued_soon.FindDiscontinuedSoonBackgroundTask(job) + task.run() + + assert len(job.audit) == 3 + assert job.audit[0]["message"] == "Journal discontinuing soon found: 1" + assert job.audit[1]["message"] == "Journal discontinuing soon found: 2" + assert job.audit[2]["message"] == "Email with journals discontinuing soon sent" + + def test_discontinued_soon_not_found(self): + + journal_discontinued_in_2_days = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True)) + journal_discontinued_in_2_days.set_id("3") + jbib = journal_discontinued_in_2_days.bibjson() + jbib.title = "Discontinued In 2 days" + jbib.discontinued_date = self._date_in_2_days() + journal_discontinued_in_2_days.save(blocking=True) + + job = find_discontinued_soon.FindDiscontinuedSoonBackgroundTask.prepare("system") + task = find_discontinued_soon.FindDiscontinuedSoonBackgroundTask(job) + task.run() + + assert len(job.audit) == 1 + assert job.audit[0]["message"] == "No journals discontinuing soon found" diff --git a/portality/settings.py b/portality/settings.py index 7896d33ac3..64a734a0d0 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -417,6 +417,7 @@ "anon_export": {"month": "*", "day": "10", "day_of_week": "*", "hour": "6", "minute": "30"}, "old_data_cleanup": {"month": "*", "day": "12", "day_of_week": "*", "hour": "6", "minute": "30"}, "monitor_bgjobs": {"month": "*", "day": "*/6", "day_of_week": "*", "hour": "10", "minute": "0"}, + "find_discontinued_soon": {"month": "*", "day": "*", "day_of_week": "*", "hour": "7", "minute": "0"}, } HUEY_TASKS = { diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index c6c1e24a99..3212559636 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -5,7 +5,10 @@ from portality.core import app from portality import models,app_email -from portality.background import BackgroundTask +from portality.tasks.redis_huey import main_queue + +from portality.background import BackgroundTask, BackgroundSummary +from portality.tasks.helpers import background_helper def _date(): @@ -34,6 +37,7 @@ class FindDiscontinuedSoonBackgroundTask(BackgroundTask): __action__ = "find_discontinued_soon" def run(self): + job = self.background_job jdata = [] for journal in models.Journal.iterate(q=DiscontinuedSoonQuery.query(), keepalive='5m', wrap=True): @@ -47,22 +51,28 @@ def run(self): "pissn": bibjson.get_one_identifier(bibjson.P_ISSN), "account_email": account.email if account else "Not Found", "publisher": bibjson.publisher}) - - try: - # send warning email about the service tag in article metadata detected - to = app.config.get('SCRIPT_TAG_DETECTED_EMAIL_RECIPIENTS') - fro = app.config.get("SYSTEM_EMAIL_FROM", "helpdesk@doaj.org") - subject = app.config.get("SERVICE_NAME", "") + " - script tag detected in application metadata" - es_type = "application" - app_email.send_mail(to=to, - fro=fro, - subject=subject, - template_name="email/discontinue_soon.jinja2", - es_type=es_type, - days=app.config.get('DISCONTINUED_DATE_DELTA',1), - data=json.dumps({"data": jdata}, indent=4, separators=(',', ': '))) - except app_email.EmailException: - app.logger.exception('Error sending email with journals discountinuing soon - ' + jdata) + job.add_audit_message("Journal discontinuing soon found: " + journal.id) + + if len(jdata): + try: + # send warning email about the service tag in article metadata detected + to = app.config.get('SCRIPT_TAG_DETECTED_EMAIL_RECIPIENTS') + fro = app.config.get("SYSTEM_EMAIL_FROM", "helpdesk@doaj.org") + subject = app.config.get("SERVICE_NAME", "") + " - script tag detected in application metadata" + es_type = "application" + app_email.send_mail(to=to, + fro=fro, + subject=subject, + template_name="email/discontinue_soon.jinja2", + es_type=es_type, + days=app.config.get('DISCONTINUED_DATE_DELTA',1), + data=json.dumps({"data": jdata}, indent=4, separators=(',', ': '))) + except app_email.EmailException: + app.logger.exception('Error sending email with journals discountinuing soon - ' + jdata) + + job.add_audit_message("Email with journals discontinuing soon sent") + else: + job.add_audit_message("No journals discontinuing soon found") def cleanup(self): """ @@ -95,9 +105,9 @@ def submit(cls, background_job): :return: """ background_job.save() - find_discontinued_soon.schedule(args=(background_job.id,), delay=86400) + find_discontinued_soon.schedule(args=(background_job.id,), delay=10) -huey_helper = RequestESBackupBackgroundTask.create_huey_helper(main_queue) +huey_helper = FindDiscontinuedSoonBackgroundTask.create_huey_helper(main_queue) @huey_helper.register_schedule def scheduled_find_discontinued_soon(): From 1c7e0e10927d276fd8676622bfba50a6827e5ba1 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Wed, 1 Feb 2023 13:49:19 +0000 Subject: [PATCH 07/34] refactoring --- doajtest/unit/test_task_discontinued_soon.py | 9 +-- portality/settings.py | 1 + portality/tasks/find_discontinued_soon.py | 58 +++++++++++--------- portality/ui/messages.py | 6 ++ 4 files changed, 45 insertions(+), 29 deletions(-) diff --git a/doajtest/unit/test_task_discontinued_soon.py b/doajtest/unit/test_task_discontinued_soon.py index 77e9c558c5..00c3183dce 100644 --- a/doajtest/unit/test_task_discontinued_soon.py +++ b/doajtest/unit/test_task_discontinued_soon.py @@ -6,6 +6,7 @@ from portality.core import app from portality import models from portality.tasks import find_discontinued_soon +from portality.ui.messages import Messages from doajtest.fixtures import JournalFixtureFactory class TestDiscontinuedSoon(DoajTestCase): @@ -44,9 +45,9 @@ def test_discontinued_soon_found(self): task.run() assert len(job.audit) == 3 - assert job.audit[0]["message"] == "Journal discontinuing soon found: 1" - assert job.audit[1]["message"] == "Journal discontinuing soon found: 2" - assert job.audit[2]["message"] == "Email with journals discontinuing soon sent" + assert job.audit[0]["message"] == Messages.DISCONTINUED_JOURNAL_FOUND_LOG.format(id="1") + assert job.audit[1]["message"] == Messages.DISCONTINUED_JOURNAL_FOUND_LOG.format(id="2") + assert job.audit[2]["message"] == Messages.DISCONTINUED_JOURNALS_FOUND_EMAIL_SENT_LOG def test_discontinued_soon_not_found(self): @@ -62,4 +63,4 @@ def test_discontinued_soon_not_found(self): task.run() assert len(job.audit) == 1 - assert job.audit[0]["message"] == "No journals discontinuing soon found" + assert job.audit[0]["message"] == Messages.NO_DISCONTINUED_JOURNALS_FOUND_LOG diff --git a/portality/settings.py b/portality/settings.py index 64a734a0d0..ab044207d6 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -341,6 +341,7 @@ MANAGING_EDITOR_EMAIL = "managing-editors@doaj.org" CONTACT_FORM_ADDRESS = "feedback+contactform@doaj.org" SCRIPT_TAG_DETECTED_EMAIL_RECIPIENTS = ["helpdesk@doaj.org"] +DISCONTINUED_JOURNALS_FOUND_RECEIPIENTS = ["dom@doaj.org"] SYSTEM_EMAIL_FROM = 'helpdesk@doaj.org' CC_ALL_EMAILS_TO = SYSTEM_EMAIL_FROM # DOAJ may get a dedicated inbox in the future diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index 3212559636..f322cd1e74 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -9,6 +9,7 @@ from portality.background import BackgroundTask, BackgroundSummary from portality.tasks.helpers import background_helper +from portality.ui.messages import Messages def _date(): @@ -36,8 +37,7 @@ def query(cls): class FindDiscontinuedSoonBackgroundTask(BackgroundTask): __action__ = "find_discontinued_soon" - def run(self): - job = self.background_job + def find_journals_discontinuing_soon(self, job): jdata = [] for journal in models.Journal.iterate(q=DiscontinuedSoonQuery.query(), keepalive='5m', wrap=True): @@ -50,29 +50,37 @@ def run(self): "eissn": bibjson.get_one_identifier(bibjson.E_ISSN), "pissn": bibjson.get_one_identifier(bibjson.P_ISSN), "account_email": account.email if account else "Not Found", - "publisher": bibjson.publisher}) - job.add_audit_message("Journal discontinuing soon found: " + journal.id) - - if len(jdata): - try: - # send warning email about the service tag in article metadata detected - to = app.config.get('SCRIPT_TAG_DETECTED_EMAIL_RECIPIENTS') - fro = app.config.get("SYSTEM_EMAIL_FROM", "helpdesk@doaj.org") - subject = app.config.get("SERVICE_NAME", "") + " - script tag detected in application metadata" - es_type = "application" - app_email.send_mail(to=to, - fro=fro, - subject=subject, - template_name="email/discontinue_soon.jinja2", - es_type=es_type, - days=app.config.get('DISCONTINUED_DATE_DELTA',1), - data=json.dumps({"data": jdata}, indent=4, separators=(',', ': '))) - except app_email.EmailException: - app.logger.exception('Error sending email with journals discountinuing soon - ' + jdata) - - job.add_audit_message("Email with journals discontinuing soon sent") + "publisher": bibjson.publisher, + "discontinued date": bibjson.discontinued_date}) + job.add_audit_message(Messages.DISCONTINUED_JOURNAL_FOUND_LOG.format(id=journal.id)) + + return jdata + + + def send_email(self, data, job): + try: + # send warning email about the service tag in article metadata detected + to = app.config.get('DISCONTINUED_JOURNALS_FOUND_RECEIPIENTS') + fro = app.config.get("SYSTEM_EMAIL_FROM", "helpdesk@doaj.org") + subject = app.config.get("SERVICE_NAME", "") + " - journals discontinuing soon found" + app_email.send_mail(to=to, + fro=fro, + subject=subject, + template_name="email/discontinue_soon.jinja2", + days=app.config.get('DISCONTINUED_DATE_DELTA',1), + data=json.dumps({"data": data}, indent=4, separators=(',', ': '))) + except app_email.EmailException: + app.logger.exception(Messages.DISCONTINUED_JOURNALS_FOUND_EMAIL_ERROR_LOG) + + job.add_audit_message(Messages.DISCONTINUED_JOURNALS_FOUND_EMAIL_SENT_LOG) + + def run(self): + job = self.background_job + journals = self.find_journals_discontinuing_soon(job=job) + if len(journals): + self.send_email(job=job, data=journals) else: - job.add_audit_message("No journals discontinuing soon found") + job.add_audit_message(Messages.NO_DISCONTINUED_JOURNALS_FOUND_LOG) def cleanup(self): """ @@ -120,4 +128,4 @@ def scheduled_find_discontinued_soon(): def find_discontinued_soon(job_id): job = models.BackgroundJob.pull(job_id) task = FindDiscontinuedSoonBackgroundTask(job) - BackgroundApi.execute(task) + BackgroundApi.execute(task) \ No newline at end of file diff --git a/portality/ui/messages.py b/portality/ui/messages.py index 82667e36e9..1c5af3a6da 100644 --- a/portality/ui/messages.py +++ b/portality/ui/messages.py @@ -109,6 +109,12 @@ class Messages(object): NOTIFY__DEFAULT_SHORT_NOTIFICATION = "You have a new notification" + DISCONTINUED_JOURNAL_FOUND_LOG = "Journal discontinuing soon found: {id}" + DISCONTINUED_JOURNALS_FOUND_EMAIL_SENT_LOG = "Email with journals discontinuing soon sent" + DISCONTINUED_JOURNALS_FOUND_EMAIL_ERROR_LOG = "Error sending email with journals discountinuing soon." + NO_DISCONTINUED_JOURNALS_FOUND_LOG = "No journals discontinuing soon found" + + @classmethod def flash(cls, tup): if isinstance(tup, tuple): From 6357e7f581696da8fc2082187ca2b46a7a536d6b Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Wed, 1 Feb 2023 13:53:17 +0000 Subject: [PATCH 08/34] add FeatureMap annotations --- portality/tasks/find_discontinued_soon.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index f322cd1e74..eec4b8a20f 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -41,6 +41,7 @@ def find_journals_discontinuing_soon(self, job): jdata = [] for journal in models.Journal.iterate(q=DiscontinuedSoonQuery.query(), keepalive='5m', wrap=True): + # ~~->Journal:Model~~ bibjson = journal.bibjson() owner = journal.owner account = models.Account.pull(owner) @@ -58,6 +59,7 @@ def find_journals_discontinuing_soon(self, job): def send_email(self, data, job): + # ~~->Email:ExternalService~~ try: # send warning email about the service tag in article metadata detected to = app.config.get('DISCONTINUED_JOURNALS_FOUND_RECEIPIENTS') From 91ad6eb3583fdd0774133b5b9ca80318ec97fa94 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Wed, 1 Feb 2023 14:10:39 +0000 Subject: [PATCH 09/34] add functional test --- doajtest/testbook/public_site/ToC.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 doajtest/testbook/public_site/ToC.yml diff --git a/doajtest/testbook/public_site/ToC.yml b/doajtest/testbook/public_site/ToC.yml new file mode 100644 index 0000000000..b481085cf9 --- /dev/null +++ b/doajtest/testbook/public_site/ToC.yml @@ -0,0 +1,15 @@ +suite: Public Site +testset: ToC +tests: +- title: Test Correctly Displayed Discontinued Date + context: + role: anonymous + steps: + - step: To prepare to do this test make sure there are 3 journals publically available in DOAJ + one with discontinued date in the past + one with discontinued date in the future + one with discontinued date today + - step: Search for every journal from the list above + results: + - On the ToC of the journal with discontinued date in the past or today - the discontinued date is displayed + - On the ToC of the journal with discontinued date in the future - the discontinued date is not displayed From a037fd54eb453ddda40d4aaf707fa90365607ff1 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Tue, 7 Feb 2023 14:09:13 +0000 Subject: [PATCH 10/34] implement notification --- .../journal_discontinuing_soon_notify.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 portality/events/consumers/journal_discontinuing_soon_notify.py diff --git a/portality/events/consumers/journal_discontinuing_soon_notify.py b/portality/events/consumers/journal_discontinuing_soon_notify.py new file mode 100644 index 0000000000..dbfc7cd236 --- /dev/null +++ b/portality/events/consumers/journal_discontinuing_soon_notify.py @@ -0,0 +1,45 @@ +# ~~JournalDiscontinuingSoonNotify:Consumer~~ +from portality.util import url_for + +from portality.events.consumer import EventConsumer +from portality import constants +from portality import models +from portality.bll import DOAJ +from portality.bll import exceptions + + +class JournalDiscontinuingSoon(EventConsumer): + ID = "journal:assed:discontinuing_soon:notify" + + @classmethod + def consumes(cls, event): + return event.id == constants.EVENT_JOURNAL_DISCONTINUING_SOON and \ + event.context.get("journal") is not None + + @classmethod + def consume(cls, event): + j_source = event.context.get("journal") + + try: + journal = models.Journal(**j_source) + except Exception as e: + raise exceptions.NoSuchObjectException("Unable to construct Journal from supplied source - data structure validation error, {x}".format(x=e)) + + if not journal.editor: + raise exceptions.NoSuchPropertyException("Journal {x} does not have property `editor`".format(x=journal.id)) + + # ~~-> Notifications:Service ~~ + svc = DOAJ.notificationsService() + + notification = models.Notification() + notification.who = journal.editor + notification.created_by = cls.ID + notification.classification = constants.NOTIFICATION_CLASSIFICATION_ASSIGN + notification.long = svc.long_notification(cls.ID).format( + journal_name=journal.bibjson().title, + group_name=journal.editor_group + ) + notification.short = svc.short_notification(cls.ID) + notification.action = url_for("editor.journal_page", journal_id=journal.id) + + svc.notify(notification) From b2557c160799e2e06100851138fa6c351dbe916b Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Tue, 7 Feb 2023 14:10:25 +0000 Subject: [PATCH 11/34] implement notification --- cms/data/notifications.yml | 8 +++ portality/bll/services/events.py | 4 +- portality/constants.py | 2 + .../journal_discontinuing_soon_notify.py | 53 ++++++++++-------- portality/tasks/find_discontinued_soon.py | 55 +++++++++++-------- 5 files changed, 75 insertions(+), 47 deletions(-) diff --git a/cms/data/notifications.yml b/cms/data/notifications.yml index 6aca840e49..a33ae53b5c 100644 --- a/cms/data/notifications.yml +++ b/cms/data/notifications.yml @@ -142,3 +142,11 @@ update_request:publisher:rejected:notify: short: Your update request was rejected +journal:assed:discontinuing_soon:notify: + long: | + Following journals will discontinue in {days} days. + + {data} + short: + Journals discontinuing soon found + diff --git a/portality/bll/services/events.py b/portality/bll/services/events.py index 2b27b85beb..3c5e96473c 100644 --- a/portality/bll/services/events.py +++ b/portality/bll/services/events.py @@ -21,6 +21,7 @@ from portality.events.consumers.journal_editor_group_assigned_notify import JournalEditorGroupAssignedNotify from portality.events.consumers.application_publisher_inprogress_notify import ApplicationPublisherInprogressNotify from portality.events.consumers.update_request_publisher_rejected_notify import UpdateRequestPublisherRejectedNotify +from portality.events.consumers.journal_discontinuing_soon_notify import JournalDiscontinuingSoonNotify class EventsService(object): @@ -44,7 +45,8 @@ class EventsService(object): JournalEditorGroupAssignedNotify, UpdateRequestPublisherAcceptedNotify, UpdateRequestPublisherAssignedNotify, - UpdateRequestPublisherRejectedNotify + UpdateRequestPublisherRejectedNotify, + JournalDiscontinuingSoonNotify ] def __init__(self): diff --git a/portality/constants.py b/portality/constants.py index e9ff3e86f0..9224eda606 100644 --- a/portality/constants.py +++ b/portality/constants.py @@ -53,7 +53,9 @@ EVENT_APPLICATION_EDITOR_GROUP_ASSIGNED = "application:editor_group:assigned" EVENT_JOURNAL_ASSED_ASSIGNED = "journal:assed:assigned" EVENT_JOURNAL_EDITOR_GROUP_ASSIGNED = "journal:editor_group:assigned" +EVENT_JOURNAL_DISCONTINUING_SOON = "journal:discontinuing_soon" +NOTIFICATION_CLASSIFICATION_STATUS = "status" NOTIFICATION_CLASSIFICATION_STATUS_CHANGE = "status_change" NOTIFICATION_CLASSIFICATION_ASSIGN = "assign" NOTIFICATION_CLASSIFICATION_CREATE = "create" diff --git a/portality/events/consumers/journal_discontinuing_soon_notify.py b/portality/events/consumers/journal_discontinuing_soon_notify.py index dbfc7cd236..d2c8e10778 100644 --- a/portality/events/consumers/journal_discontinuing_soon_notify.py +++ b/portality/events/consumers/journal_discontinuing_soon_notify.py @@ -2,44 +2,49 @@ from portality.util import url_for from portality.events.consumer import EventConsumer +from portality.core import app from portality import constants from portality import models from portality.bll import DOAJ from portality.bll import exceptions -class JournalDiscontinuingSoon(EventConsumer): +class JournalDiscontinuingSoonNotify(EventConsumer): ID = "journal:assed:discontinuing_soon:notify" @classmethod def consumes(cls, event): return event.id == constants.EVENT_JOURNAL_DISCONTINUING_SOON and \ - event.context.get("journal") is not None + event.context.get("job") is not None @classmethod def consume(cls, event): - j_source = event.context.get("journal") + source = event.context.get("job") try: - journal = models.Journal(**j_source) + job = models.BackgroundJob(**source) except Exception as e: - raise exceptions.NoSuchObjectException("Unable to construct Journal from supplied source - data structure validation error, {x}".format(x=e)) - - if not journal.editor: - raise exceptions.NoSuchPropertyException("Journal {x} does not have property `editor`".format(x=journal.id)) - - # ~~-> Notifications:Service ~~ - svc = DOAJ.notificationsService() - - notification = models.Notification() - notification.who = journal.editor - notification.created_by = cls.ID - notification.classification = constants.NOTIFICATION_CLASSIFICATION_ASSIGN - notification.long = svc.long_notification(cls.ID).format( - journal_name=journal.bibjson().title, - group_name=journal.editor_group - ) - notification.short = svc.short_notification(cls.ID) - notification.action = url_for("editor.journal_page", journal_id=journal.id) - - svc.notify(notification) + raise exceptions.NoSuchObjectException( + "Unable to construct a BackgroundJob object from the data supplied {x}".format(x=e)) + + if job.user is None: + return + + acc = models.Account.pull(job.user) + if acc is None or not acc.has_role("admin"): + return + + # ~~-> Notifications:Service ~~ + svc = DOAJ.notificationsService() + + notification = models.Notification() + notification.who = acc.id + notification.created_by = cls.ID + notification.classification = constants.NOTIFICATION_CLASSIFICATION_ASSIGN + notification.long = svc.long_notification(cls.ID).format( + days=app.config.get('DISCONTINUED_DATE_DELTA',1), + data=event.context.get("data") + ) + notification.short = svc.short_notification(cls.ID) + + svc.notify(notification) diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index eec4b8a20f..fa3e4e09a8 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -3,6 +3,7 @@ import json from portality.core import app +from portality.bll import DOAJ from portality import models,app_email from portality.tasks.redis_huey import main_queue @@ -10,6 +11,7 @@ from portality.background import BackgroundTask, BackgroundSummary from portality.tasks.helpers import background_helper from portality.ui.messages import Messages +from portality import constants def _date(): @@ -37,7 +39,7 @@ def query(cls): class FindDiscontinuedSoonBackgroundTask(BackgroundTask): __action__ = "find_discontinued_soon" - def find_journals_discontinuing_soon(self, job): + def find_journals_discontinuing_soon(self): jdata = [] for journal in models.Journal.iterate(q=DiscontinuedSoonQuery.query(), keepalive='5m', wrap=True): @@ -57,30 +59,19 @@ def find_journals_discontinuing_soon(self, job): return jdata - - def send_email(self, data, job): - # ~~->Email:ExternalService~~ - try: - # send warning email about the service tag in article metadata detected - to = app.config.get('DISCONTINUED_JOURNALS_FOUND_RECEIPIENTS') - fro = app.config.get("SYSTEM_EMAIL_FROM", "helpdesk@doaj.org") - subject = app.config.get("SERVICE_NAME", "") + " - journals discontinuing soon found" - app_email.send_mail(to=to, - fro=fro, - subject=subject, - template_name="email/discontinue_soon.jinja2", - days=app.config.get('DISCONTINUED_DATE_DELTA',1), - data=json.dumps({"data": data}, indent=4, separators=(',', ': '))) - except app_email.EmailException: - app.logger.exception(Messages.DISCONTINUED_JOURNALS_FOUND_EMAIL_ERROR_LOG) - - job.add_audit_message(Messages.DISCONTINUED_JOURNALS_FOUND_EMAIL_SENT_LOG) - def run(self): job = self.background_job journals = self.find_journals_discontinuing_soon(job=job) + journals = find_journals_discontinuing_soon() if len(journals): - self.send_email(job=job, data=journals) + DOAJ.eventsService().trigger(models.Event( + constants.EVENT_JOURNAL_DISCONTINUING_SOON, + "system", + { + "context": "job", + "data": jdata, + "job": job + })) else: job.add_audit_message(Messages.NO_DISCONTINUED_JOURNALS_FOUND_LOG) @@ -130,4 +121,24 @@ def scheduled_find_discontinued_soon(): def find_discontinued_soon(job_id): job = models.BackgroundJob.pull(job_id) task = FindDiscontinuedSoonBackgroundTask(job) - BackgroundApi.execute(task) \ No newline at end of file + BackgroundApi.execute(task) + +def find_journals_discontinuing_soon(): + jdata = [] + + for journal in models.Journal.iterate(q=DiscontinuedSoonQuery.query(), keepalive='5m', wrap=True): + # ~~->Journal:Model~~ + bibjson = journal.bibjson() + owner = journal.owner + account = models.Account.pull(owner) + + jdata.append({"id": journal.id, + "title":bibjson.title, + "eissn": bibjson.get_one_identifier(bibjson.E_ISSN), + "pissn": bibjson.get_one_identifier(bibjson.P_ISSN), + "account_email": account.email if account else "Not Found", + "publisher": bibjson.publisher, + "discontinued date": bibjson.discontinued_date}) + print(Messages.DISCONTINUED_JOURNAL_FOUND_LOG.format(id=journal.id)) + + return jdata \ No newline at end of file From 255bce208d9db383be96ae607467392f9d36179c Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Tue, 7 Feb 2023 14:32:08 +0000 Subject: [PATCH 12/34] add notif action and remove test code --- .../consumers/journal_discontinuing_soon_notify.py | 12 ++++++++++-- portality/tasks/find_discontinued_soon.py | 5 ++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/portality/events/consumers/journal_discontinuing_soon_notify.py b/portality/events/consumers/journal_discontinuing_soon_notify.py index d2c8e10778..216a857df5 100644 --- a/portality/events/consumers/journal_discontinuing_soon_notify.py +++ b/portality/events/consumers/journal_discontinuing_soon_notify.py @@ -1,4 +1,5 @@ # ~~JournalDiscontinuingSoonNotify:Consumer~~ +import json from portality.util import url_for from portality.events.consumer import EventConsumer @@ -15,7 +16,8 @@ class JournalDiscontinuingSoonNotify(EventConsumer): @classmethod def consumes(cls, event): return event.id == constants.EVENT_JOURNAL_DISCONTINUING_SOON and \ - event.context.get("job") is not None + event.context.get("job") is not None and \ + event.context.get("data") is not None @classmethod def consume(cls, event): @@ -34,6 +36,11 @@ def consume(cls, event): if acc is None or not acc.has_role("admin"): return + + journals = [] + for j in data: + journals.append(j["id"]) + # ~~-> Notifications:Service ~~ svc = DOAJ.notificationsService() @@ -43,8 +50,9 @@ def consume(cls, event): notification.classification = constants.NOTIFICATION_CLASSIFICATION_ASSIGN notification.long = svc.long_notification(cls.ID).format( days=app.config.get('DISCONTINUED_DATE_DELTA',1), - data=event.context.get("data") + data=json.dumps({"data": data}, indent=4, separators=(',', ': ')) ) notification.short = svc.short_notification(cls.ID) + notification.action = url_for("dashboard.notifications") svc.notify(notification) diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index fa3e4e09a8..2102d4723d 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -62,14 +62,13 @@ def find_journals_discontinuing_soon(self): def run(self): job = self.background_job journals = self.find_journals_discontinuing_soon(job=job) - journals = find_journals_discontinuing_soon() if len(journals): DOAJ.eventsService().trigger(models.Event( constants.EVENT_JOURNAL_DISCONTINUING_SOON, "system", { "context": "job", - "data": jdata, + "data": journals, "job": job })) else: @@ -139,6 +138,6 @@ def find_journals_discontinuing_soon(): "account_email": account.email if account else "Not Found", "publisher": bibjson.publisher, "discontinued date": bibjson.discontinued_date}) - print(Messages.DISCONTINUED_JOURNAL_FOUND_LOG.format(id=journal.id)) + job.add_audit_message(Messages.DISCONTINUED_JOURNAL_FOUND_LOG.format(id=journal.id)) return jdata \ No newline at end of file From e8edabb43c2c4e8d5383248443f2ef593a649e30 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Wed, 8 Feb 2023 11:48:17 +0000 Subject: [PATCH 13/34] change notification classification --- portality/events/consumers/journal_discontinuing_soon_notify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/events/consumers/journal_discontinuing_soon_notify.py b/portality/events/consumers/journal_discontinuing_soon_notify.py index 216a857df5..ed00f6151e 100644 --- a/portality/events/consumers/journal_discontinuing_soon_notify.py +++ b/portality/events/consumers/journal_discontinuing_soon_notify.py @@ -47,7 +47,7 @@ def consume(cls, event): notification = models.Notification() notification.who = acc.id notification.created_by = cls.ID - notification.classification = constants.NOTIFICATION_CLASSIFICATION_ASSIGN + notification.classification = constants.NOTIFICATION_CLASSIFICATION_STATUS notification.long = svc.long_notification(cls.ID).format( days=app.config.get('DISCONTINUED_DATE_DELTA',1), data=json.dumps({"data": data}, indent=4, separators=(',', ': ')) From f23b55bc009c5013e47874a7065920f6fcc679f3 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Wed, 8 Feb 2023 12:59:16 +0000 Subject: [PATCH 14/34] revert accidental deletion --- .../event_consumers/test_journal_discontinuing_soon_notify.py | 0 portality/events/consumers/journal_discontinuing_soon_notify.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py diff --git a/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py b/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/portality/events/consumers/journal_discontinuing_soon_notify.py b/portality/events/consumers/journal_discontinuing_soon_notify.py index ed00f6151e..f61c035014 100644 --- a/portality/events/consumers/journal_discontinuing_soon_notify.py +++ b/portality/events/consumers/journal_discontinuing_soon_notify.py @@ -21,7 +21,7 @@ def consumes(cls, event): @classmethod def consume(cls, event): - + data = event.context.get("data") source = event.context.get("job") try: job = models.BackgroundJob(**source) From 8cbc1465ab025f97d1c89bf04430734dd092026c Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Wed, 8 Feb 2023 13:02:56 +0000 Subject: [PATCH 15/34] new even notif unit tests --- .../test_journal_discontinuing_soon_notify.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py b/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py index e69de29bb2..ab62fa433f 100644 --- a/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py +++ b/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py @@ -0,0 +1,68 @@ +from portality import models +from portality import constants +from portality.bll import exceptions +from doajtest.helpers import DoajTestCase +from doajtest.fixtures import JournalFixtureFactory +from portality.events.consumers.journal_discontinuing_soon_notify import JournalDiscontinuingSoonNotify +from doajtest.fixtures import BackgroundFixtureFactory +import time + + +class TestJournalDiscontinuingSoonNotify(DoajTestCase): + def setUp(self): + super(TestJournalDiscontinuingSoonNotify, self).setUp() + + def tearDown(self): + super(TestJournalDiscontinuingSoonNotify, self).tearDown() + + def test_consumes(self): + event = models.Event(constants.EVENT_JOURNAL_DISCONTINUING_SOON, context={"job" : {}}) + assert not JournalDiscontinuingSoonNotify.consumes(event) + + event = models.Event("test:event", context={"data" : {"1234"}}) + assert not JournalDiscontinuingSoonNotify.consumes(event) + + event = models.Event("test:event", context={"data": {}}) + assert not JournalDiscontinuingSoonNotify.consumes(event) + + event = models.Event(constants.EVENT_JOURNAL_DISCONTINUING_SOON) + assert not JournalDiscontinuingSoonNotify.consumes(event) + + event = models.Event(constants.EVENT_JOURNAL_DISCONTINUING_SOON, context = {"data": {"1234"}, "job": {"1234"}}) + assert JournalDiscontinuingSoonNotify.consumes(event) + + def test_consume_success(self): + self._make_and_push_test_context("/") + + source = BackgroundFixtureFactory.example() + bj = models.BackgroundJob(**source) + # bj.save(blocking=True) + + acc = models.Account() + acc.set_id('testuser') + acc.set_email("test@example.com") + acc.add_role('admin') + acc.save(blocking=True) + + + + event = models.Event(constants.BACKGROUND_JOB_FINISHED, context={"job" : bj.data, "data" : JournalFixtureFactory.make_many_journal_sources(2)}) + JournalDiscontinuingSoonNotify.consume(event) + + time.sleep(2) + ns = models.Notification.all() + assert len(ns) == 1 + + n = ns[0] + assert n.who == acc.id + assert n.created_by == JournalDiscontinuingSoonNotify.ID + assert n.classification == constants.NOTIFICATION_CLASSIFICATION_STATUS + assert n.long is not None + assert n.short is not None + assert n.action is not None + assert not n.is_seen() + + def test_consume_fail(self): + event = models.Event(constants.NOTIFICATION_CLASSIFICATION_STATUS, context={"job": "abcd"}) + with self.assertRaises(exceptions.NoSuchObjectException): + JournalDiscontinuingSoonNotify.consume(event) From f72e7218501792b7466778c3262d8935644dafe5 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Wed, 8 Feb 2023 13:15:31 +0000 Subject: [PATCH 16/34] make sure only journals in doaj are found and fix unit tests --- doajtest/unit/test_task_discontinued_soon.py | 15 ++++++++++++--- portality/tasks/find_discontinued_soon.py | 5 +++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/doajtest/unit/test_task_discontinued_soon.py b/doajtest/unit/test_task_discontinued_soon.py index 00c3183dce..f6a4be9483 100644 --- a/doajtest/unit/test_task_discontinued_soon.py +++ b/doajtest/unit/test_task_discontinued_soon.py @@ -19,6 +19,7 @@ def _date_in_2_days(self): def test_discontinued_soon_found(self): + # Both these should be found journal_discontinued_tommorow_1 = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True)) journal_discontinued_tommorow_1.set_id("1") jbib = journal_discontinued_tommorow_1.bibjson() @@ -33,6 +34,7 @@ def test_discontinued_soon_found(self): jbib.discontinued_date = self._date_tomorrow() journal_discontinued_tommorow_2.save(blocking=True) + # that shouldn't be found journal_discontinued_in_2_days = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True)) journal_discontinued_in_2_days.set_id("3") jbib = journal_discontinued_in_2_days.bibjson() @@ -44,20 +46,27 @@ def test_discontinued_soon_found(self): task = find_discontinued_soon.FindDiscontinuedSoonBackgroundTask(job) task.run() - assert len(job.audit) == 3 + assert len(job.audit) == 2 assert job.audit[0]["message"] == Messages.DISCONTINUED_JOURNAL_FOUND_LOG.format(id="1") assert job.audit[1]["message"] == Messages.DISCONTINUED_JOURNAL_FOUND_LOG.format(id="2") - assert job.audit[2]["message"] == Messages.DISCONTINUED_JOURNALS_FOUND_EMAIL_SENT_LOG def test_discontinued_soon_not_found(self): + # None of these should be found - this one discontinues in 2 days journal_discontinued_in_2_days = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True)) - journal_discontinued_in_2_days.set_id("3") + journal_discontinued_in_2_days.set_id("1") jbib = journal_discontinued_in_2_days.bibjson() jbib.title = "Discontinued In 2 days" jbib.discontinued_date = self._date_in_2_days() journal_discontinued_in_2_days.save(blocking=True) + # this one is not in doaj + journal_not_in_doaj = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=False)) + journal_not_in_doaj.set_id("2") + jbib = journal_not_in_doaj.bibjson() + jbib.discontinued_date = self._date_tomorrow() + journal_not_in_doaj.save(blocking=True) + job = find_discontinued_soon.FindDiscontinuedSoonBackgroundTask.prepare("system") task = find_discontinued_soon.FindDiscontinuedSoonBackgroundTask(job) task.run() diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index 2102d4723d..6762a5b315 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -26,7 +26,8 @@ def query(cls): "filter": { "bool" : { "must": [ - {"term" : {"bibjson.discontinued_date": _date()}} + {"term" : {"bibjson.discontinued_date": _date()}}, + {"term" : {"admin.in_doaj":True}} ] } } @@ -39,7 +40,7 @@ def query(cls): class FindDiscontinuedSoonBackgroundTask(BackgroundTask): __action__ = "find_discontinued_soon" - def find_journals_discontinuing_soon(self): + def find_journals_discontinuing_soon(self, job): jdata = [] for journal in models.Journal.iterate(q=DiscontinuedSoonQuery.query(), keepalive='5m', wrap=True): From b394331a448007d320d4d6e057b4c0be5c316514 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Wed, 8 Feb 2023 16:50:34 +0000 Subject: [PATCH 17/34] add link to notification --- .../test_journal_discontinuing_soon_notify.py | 2 +- .../journal_discontinuing_soon_notify.py | 17 ++++++++++---- portality/models/v2/journal.py | 23 ++++++++++++++++++- portality/tasks/find_discontinued_soon.py | 3 ++- 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py b/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py index ab62fa433f..e6e3b397e7 100644 --- a/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py +++ b/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py @@ -28,7 +28,7 @@ def test_consumes(self): event = models.Event(constants.EVENT_JOURNAL_DISCONTINUING_SOON) assert not JournalDiscontinuingSoonNotify.consumes(event) - event = models.Event(constants.EVENT_JOURNAL_DISCONTINUING_SOON, context = {"data": {"1234"}, "job": {"1234"}}) + event = models.Event(constants.EVENT_JOURNAL_DISCONTINUING_SOON, context = {"data": {"1234"}, "job": {"1234"}, "discontinue_date": "2002-22-02"}) assert JournalDiscontinuingSoonNotify.consumes(event) def test_consume_success(self): diff --git a/portality/events/consumers/journal_discontinuing_soon_notify.py b/portality/events/consumers/journal_discontinuing_soon_notify.py index f61c035014..c7945bafbe 100644 --- a/portality/events/consumers/journal_discontinuing_soon_notify.py +++ b/portality/events/consumers/journal_discontinuing_soon_notify.py @@ -1,14 +1,18 @@ # ~~JournalDiscontinuingSoonNotify:Consumer~~ import json -from portality.util import url_for +import urllib.parse +from portality.util import url_for from portality.events.consumer import EventConsumer from portality.core import app from portality import constants from portality import models -from portality.bll import DOAJ -from portality.bll import exceptions +from portality.bll import DOAJ, exceptions +from portality.lib import edges +from portality import dao + +from portality.models.v2.journal import JournalDiscontinuedDateQuery class JournalDiscontinuingSoonNotify(EventConsumer): ID = "journal:assed:discontinuing_soon:notify" @@ -17,12 +21,14 @@ class JournalDiscontinuingSoonNotify(EventConsumer): def consumes(cls, event): return event.id == constants.EVENT_JOURNAL_DISCONTINUING_SOON and \ event.context.get("job") is not None and \ - event.context.get("data") is not None + event.context.get("data") is not None and \ + event.context.get("discontinue_date") is not None @classmethod def consume(cls, event): data = event.context.get("data") source = event.context.get("job") + discontinued_date = event.context.get("discontinue_date") try: job = models.BackgroundJob(**source) except Exception as e: @@ -53,6 +59,7 @@ def consume(cls, event): data=json.dumps({"data": data}, indent=4, separators=(',', ': ')) ) notification.short = svc.short_notification(cls.ID) - notification.action = url_for("dashboard.notifications") + q = JournalDiscontinuedDateQuery(discontinued_date=discontinued_date).query() + notification.action = url_for('admin.index') + '?ref=toc&source=' + dao.Facetview2.url_encode_query(q) svc.notify(notification) diff --git a/portality/models/v2/journal.py b/portality/models/v2/journal.py index 03ef069b70..fb9246be1f 100644 --- a/portality/models/v2/journal.py +++ b/portality/models/v2/journal.py @@ -1121,4 +1121,25 @@ def query(self): "sort" : [ {"created_date" : {"order" : "desc"}} ] - } \ No newline at end of file + } + +class JournalDiscontinuedDateQuery(object): + base_query = { + "track_total_hits": True, + "query": { + "bool": { + "must": [ + {"terms": {"bibjson.discontinued_date": ""}} + ] + } + }, + "size": 10000 + } + + def __init__(self, discontinued_date): + self.discontinued_date = discontinued_date + + def query(self): + q = deepcopy(self.base_query) + q["query"]["bool"]["must"][0]["terms"]["bibjson.discontinued_date"] = [self.discontinued_date] + return q \ No newline at end of file diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index 6762a5b315..a0bc15f261 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -70,7 +70,8 @@ def run(self): { "context": "job", "data": journals, - "job": job + "job": job, + "discontinue_date": _date() })) else: job.add_audit_message(Messages.NO_DISCONTINUED_JOURNALS_FOUND_LOG) From 555e1a6231b368d1703980222c5838c5d1759353 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Wed, 15 Feb 2023 16:27:42 +0000 Subject: [PATCH 18/34] send separate notifications to appropriate managing editors --- cms/data/notifications.yml | 7 +- .../journal_discontinuing_soon_notify.py | 37 +++++----- portality/settings.py | 2 +- portality/tasks/find_discontinued_soon.py | 70 ++++++++----------- 4 files changed, 52 insertions(+), 64 deletions(-) diff --git a/cms/data/notifications.yml b/cms/data/notifications.yml index a33ae53b5c..d06d287d0d 100644 --- a/cms/data/notifications.yml +++ b/cms/data/notifications.yml @@ -144,9 +144,6 @@ update_request:publisher:rejected:notify: journal:assed:discontinuing_soon:notify: long: | - Following journals will discontinue in {days} days. - - {data} + Journal "{title}" (id: {id}) will discontinue in {days} days. short: - Journals discontinuing soon found - + Journals discontinuing soon found \ No newline at end of file diff --git a/portality/events/consumers/journal_discontinuing_soon_notify.py b/portality/events/consumers/journal_discontinuing_soon_notify.py index c7945bafbe..09e3f1cb70 100644 --- a/portality/events/consumers/journal_discontinuing_soon_notify.py +++ b/portality/events/consumers/journal_discontinuing_soon_notify.py @@ -20,46 +20,45 @@ class JournalDiscontinuingSoonNotify(EventConsumer): @classmethod def consumes(cls, event): return event.id == constants.EVENT_JOURNAL_DISCONTINUING_SOON and \ - event.context.get("job") is not None and \ - event.context.get("data") is not None and \ + event.context.get("journal") is not None and \ event.context.get("discontinue_date") is not None @classmethod def consume(cls, event): - data = event.context.get("data") - source = event.context.get("job") + journal_id = event.context.get("journal") discontinued_date = event.context.get("discontinue_date") - try: - job = models.BackgroundJob(**source) - except Exception as e: - raise exceptions.NoSuchObjectException( - "Unable to construct a BackgroundJob object from the data supplied {x}".format(x=e)) - if job.user is None: + journal = models.Journal.pull(journal_id) + if journal is None: return - acc = models.Account.pull(job.user) - if acc is None or not acc.has_role("admin"): + app_id = journal.latest_related_application_id() + if app_id is None: return + else: + application = models.Application.pull(app_id) + if not application.editor_group: + return - journals = [] - for j in data: - journals.append(j["id"]) + eg = models.EditorGroup.pull_by_key("name", application.editor_group) + managing_editor = eg.maned + if not managing_editor: + return # ~~-> Notifications:Service ~~ svc = DOAJ.notificationsService() notification = models.Notification() - notification.who = acc.id + notification.who = managing_editor notification.created_by = cls.ID notification.classification = constants.NOTIFICATION_CLASSIFICATION_STATUS notification.long = svc.long_notification(cls.ID).format( days=app.config.get('DISCONTINUED_DATE_DELTA',1), - data=json.dumps({"data": data}, indent=4, separators=(',', ': ')) + title=journal.bibjson().title, + id=journal.id ) notification.short = svc.short_notification(cls.ID) - q = JournalDiscontinuedDateQuery(discontinued_date=discontinued_date).query() - notification.action = url_for('admin.index') + '?ref=toc&source=' + dao.Facetview2.url_encode_query(q) + notification.action = url_for("admin.journal_page", journal_id=journal.id) svc.notify(notification) diff --git a/portality/settings.py b/portality/settings.py index ab044207d6..6711a70bbd 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -1357,4 +1357,4 @@ } # report journals that discontinue in ... days (eg. 1 = tomorrow) -DISCONTINUED_DATE_DELTA = 1 \ No newline at end of file +DISCONTINUED_DATE_DELTA = 0 \ No newline at end of file diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index a0bc15f261..0562f09297 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -45,17 +45,7 @@ def find_journals_discontinuing_soon(self, job): for journal in models.Journal.iterate(q=DiscontinuedSoonQuery.query(), keepalive='5m', wrap=True): # ~~->Journal:Model~~ - bibjson = journal.bibjson() - owner = journal.owner - account = models.Account.pull(owner) - - jdata.append({"id": journal.id, - "title":bibjson.title, - "eissn": bibjson.get_one_identifier(bibjson.E_ISSN), - "pissn": bibjson.get_one_identifier(bibjson.P_ISSN), - "account_email": account.email if account else "Not Found", - "publisher": bibjson.publisher, - "discontinued date": bibjson.discontinued_date}) + jdata.append(journal.id) job.add_audit_message(Messages.DISCONTINUED_JOURNAL_FOUND_LOG.format(id=journal.id)) return jdata @@ -64,15 +54,14 @@ def run(self): job = self.background_job journals = self.find_journals_discontinuing_soon(job=job) if len(journals): - DOAJ.eventsService().trigger(models.Event( - constants.EVENT_JOURNAL_DISCONTINUING_SOON, - "system", - { - "context": "job", - "data": journals, - "job": job, - "discontinue_date": _date() - })) + for j in journals: + DOAJ.eventsService().trigger(models.Event( + constants.EVENT_JOURNAL_DISCONTINUING_SOON, + "system", + { + "journal": j, + "discontinue_date": _date() + })) else: job.add_audit_message(Messages.NO_DISCONTINUED_JOURNALS_FOUND_LOG) @@ -124,22 +113,25 @@ def find_discontinued_soon(job_id): task = FindDiscontinuedSoonBackgroundTask(job) BackgroundApi.execute(task) -def find_journals_discontinuing_soon(): - jdata = [] - - for journal in models.Journal.iterate(q=DiscontinuedSoonQuery.query(), keepalive='5m', wrap=True): - # ~~->Journal:Model~~ - bibjson = journal.bibjson() - owner = journal.owner - account = models.Account.pull(owner) - - jdata.append({"id": journal.id, - "title":bibjson.title, - "eissn": bibjson.get_one_identifier(bibjson.E_ISSN), - "pissn": bibjson.get_one_identifier(bibjson.P_ISSN), - "account_email": account.email if account else "Not Found", - "publisher": bibjson.publisher, - "discontinued date": bibjson.discontinued_date}) - job.add_audit_message(Messages.DISCONTINUED_JOURNAL_FOUND_LOG.format(id=journal.id)) - - return jdata \ No newline at end of file + +# Test code - please do not clean up until after the tests +# def find_journals_discontinuing_soon(): +# jdata = [] +# +# for journal in models.Journal.iterate(q=DiscontinuedSoonQuery.query(), keepalive='5m', wrap=True): +# # ~~->Journal:Model~~ +# jdata.append(journal.id) +# +# return jdata +# +# if __name__ == "__main__": +# journals = find_journals_discontinuing_soon() +# if len(journals): +# for j in journals: +# DOAJ.eventsService().trigger(models.Event( +# constants.EVENT_JOURNAL_DISCONTINUING_SOON, +# "system", +# { +# "journal": j, +# "discontinue_date": _date() +# })) \ No newline at end of file From b9e984fad203b726438fe84802adc60ba14b2fbf Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Wed, 15 Feb 2023 16:35:07 +0000 Subject: [PATCH 19/34] check for current application --- .../consumers/journal_discontinuing_soon_notify.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/portality/events/consumers/journal_discontinuing_soon_notify.py b/portality/events/consumers/journal_discontinuing_soon_notify.py index 09e3f1cb70..5502ebddae 100644 --- a/portality/events/consumers/journal_discontinuing_soon_notify.py +++ b/portality/events/consumers/journal_discontinuing_soon_notify.py @@ -32,11 +32,13 @@ def consume(cls, event): if journal is None: return - app_id = journal.latest_related_application_id() + app_id = journal.current_application if app_id is None: - return - else: - application = models.Application.pull(app_id) + app_id = journal.latest_related_application_id() + if app_id is None: + return + + application = models.Application.pull(app_id) if not application.editor_group: return From ae807e82231ff5b3efa6f7f1aecedde375c15e11 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Wed, 15 Feb 2023 16:42:50 +0000 Subject: [PATCH 20/34] remove unnecessary setting --- portality/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/portality/settings.py b/portality/settings.py index 6711a70bbd..878619ea91 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -341,7 +341,6 @@ MANAGING_EDITOR_EMAIL = "managing-editors@doaj.org" CONTACT_FORM_ADDRESS = "feedback+contactform@doaj.org" SCRIPT_TAG_DETECTED_EMAIL_RECIPIENTS = ["helpdesk@doaj.org"] -DISCONTINUED_JOURNALS_FOUND_RECEIPIENTS = ["dom@doaj.org"] SYSTEM_EMAIL_FROM = 'helpdesk@doaj.org' CC_ALL_EMAILS_TO = SYSTEM_EMAIL_FROM # DOAJ may get a dedicated inbox in the future From 24214f7c8ded9c5aa48e1d36bb7549e2a7a9b1d8 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Fri, 17 Feb 2023 12:34:57 +0000 Subject: [PATCH 21/34] fix imports and tests --- .../test_journal_discontinuing_soon_notify.py | 40 ++++++++++++++----- portality/app.py | 2 +- portality/tasks/find_discontinued_soon.py | 2 +- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py b/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py index e6e3b397e7..f4f70f2f78 100644 --- a/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py +++ b/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py @@ -2,22 +2,43 @@ from portality import constants from portality.bll import exceptions from doajtest.helpers import DoajTestCase -from doajtest.fixtures import JournalFixtureFactory +from doajtest.fixtures import JournalFixtureFactory, ApplicationFixtureFactory from portality.events.consumers.journal_discontinuing_soon_notify import JournalDiscontinuingSoonNotify from doajtest.fixtures import BackgroundFixtureFactory import time +# Mock required to make application lookup work +@classmethod +def pull_application(cls, id): + app = models.Application(**ApplicationFixtureFactory.make_application_source()) + return app + +@classmethod +def pull_by_key(cls, key, value): + ed = models.EditorGroup() + acc = models.Account() + acc.set_id('testuser') + acc.set_email("test@example.com") + acc.save(blocking=True) + ed.set_maned(acc.id) + ed.save(blocking=True) + + return ed class TestJournalDiscontinuingSoonNotify(DoajTestCase): def setUp(self): super(TestJournalDiscontinuingSoonNotify, self).setUp() + self.pull_application = models.Application.pull + models.Application.pull = pull_application + self.pull_by_key = models.EditorGroup.pull_by_key + models.EditorGroup.pull_by_key = pull_by_key def tearDown(self): super(TestJournalDiscontinuingSoonNotify, self).tearDown() + models.Application.pull = self.pull_application + models.EditorGroup.pull_by_key = self.pull_by_key def test_consumes(self): - event = models.Event(constants.EVENT_JOURNAL_DISCONTINUING_SOON, context={"job" : {}}) - assert not JournalDiscontinuingSoonNotify.consumes(event) event = models.Event("test:event", context={"data" : {"1234"}}) assert not JournalDiscontinuingSoonNotify.consumes(event) @@ -28,7 +49,7 @@ def test_consumes(self): event = models.Event(constants.EVENT_JOURNAL_DISCONTINUING_SOON) assert not JournalDiscontinuingSoonNotify.consumes(event) - event = models.Event(constants.EVENT_JOURNAL_DISCONTINUING_SOON, context = {"data": {"1234"}, "job": {"1234"}, "discontinue_date": "2002-22-02"}) + event = models.Event(constants.EVENT_JOURNAL_DISCONTINUING_SOON, context = {"journal": {"1234"}, "discontinue_date": "2002-22-02"}) assert JournalDiscontinuingSoonNotify.consumes(event) def test_consume_success(self): @@ -44,9 +65,11 @@ def test_consume_success(self): acc.add_role('admin') acc.save(blocking=True) + source = JournalFixtureFactory.make_journal_source() + journal = models.Journal(**source) + journal.save(blocking=True) - - event = models.Event(constants.BACKGROUND_JOB_FINISHED, context={"job" : bj.data, "data" : JournalFixtureFactory.make_many_journal_sources(2)}) + event = models.Event(constants.BACKGROUND_JOB_FINISHED, context={"job" : bj.data, "journal" : journal.id}) JournalDiscontinuingSoonNotify.consume(event) time.sleep(2) @@ -61,8 +84,3 @@ def test_consume_success(self): assert n.short is not None assert n.action is not None assert not n.is_seen() - - def test_consume_fail(self): - event = models.Event(constants.NOTIFICATION_CLASSIFICATION_STATUS, context={"job": "abcd"}) - with self.assertRaises(exceptions.NoSuchObjectException): - JournalDiscontinuingSoonNotify.consume(event) diff --git a/portality/app.py b/portality/app.py index 9f3a674a55..a11a928959 100644 --- a/portality/app.py +++ b/portality/app.py @@ -17,7 +17,7 @@ from flask import request, abort, render_template, redirect, send_file, url_for, jsonify, send_from_directory from flask_login import login_user, current_user -from datetime import datetime, date +from datetime import datetime import portality.models as models from portality.core import app, es_connection, initialise_index diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index 0562f09297..2214a6e560 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -8,7 +8,7 @@ from portality.tasks.redis_huey import main_queue -from portality.background import BackgroundTask, BackgroundSummary +from portality.background import BackgroundTask, BackgroundSummary, BackgroundApi from portality.tasks.helpers import background_helper from portality.ui.messages import Messages from portality import constants From f54af1c0f2e17efd8cdd7a8081a6d147ded17a59 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Fri, 17 Feb 2023 12:40:48 +0000 Subject: [PATCH 22/34] fix task test --- doajtest/unit/test_task_discontinued_soon.py | 52 ++++++++++---------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/doajtest/unit/test_task_discontinued_soon.py b/doajtest/unit/test_task_discontinued_soon.py index f6a4be9483..3c9e4bcd1d 100644 --- a/doajtest/unit/test_task_discontinued_soon.py +++ b/doajtest/unit/test_task_discontinued_soon.py @@ -9,38 +9,40 @@ from portality.ui.messages import Messages from doajtest.fixtures import JournalFixtureFactory +DELTA = app.config.get('DISCONTINUED_DATE_DELTA',1) + class TestDiscontinuedSoon(DoajTestCase): - def _date_tomorrow(self): - return (datetime.datetime.today() + datetime.timedelta(days=1)).strftime('%Y-%m-%d') + def _date_to_found(self): + return (datetime.datetime.today() + datetime.timedelta(days=DELTA)).strftime('%Y-%m-%d') - def _date_in_2_days(self): - return (datetime.datetime.today() + datetime.timedelta(days=2)).strftime('%Y-%m-%d') + def _date_too_late(self): + return (datetime.datetime.today() + datetime.timedelta(days=DELTA+1)).strftime('%Y-%m-%d') def test_discontinued_soon_found(self): # Both these should be found - journal_discontinued_tommorow_1 = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True)) - journal_discontinued_tommorow_1.set_id("1") - jbib = journal_discontinued_tommorow_1.bibjson() + journal_discontinued_to_found_1 = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True)) + journal_discontinued_to_found_1.set_id("1") + jbib = journal_discontinued_to_found_1.bibjson() jbib.title = "Discontinued Tomorrow 1" - jbib.discontinued_date = self._date_tomorrow() - journal_discontinued_tommorow_1.save(blocking=True) + jbib.discontinued_date = self._date_to_found() + journal_discontinued_to_found_1.save(blocking=True) - journal_discontinued_tommorow_2 = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True)) - journal_discontinued_tommorow_2.set_id("2") - jbib = journal_discontinued_tommorow_2.bibjson() + journal_discontinued_to_found_2 = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True)) + journal_discontinued_to_found_2.set_id("2") + jbib = journal_discontinued_to_found_2.bibjson() jbib.title = "Discontinued Tomorrow 2" - jbib.discontinued_date = self._date_tomorrow() - journal_discontinued_tommorow_2.save(blocking=True) + jbib.discontinued_date = self._date_to_found() + journal_discontinued_to_found_2.save(blocking=True) # that shouldn't be found - journal_discontinued_in_2_days = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True)) - journal_discontinued_in_2_days.set_id("3") - jbib = journal_discontinued_in_2_days.bibjson() + journal_discontinued_too_late = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True)) + journal_discontinued_too_late.set_id("3") + jbib = journal_discontinued_too_late.bibjson() jbib.title = "Discontinued In 2 days" - jbib.discontinued_date = self._date_in_2_days() - journal_discontinued_in_2_days.save(blocking=True) + jbib.discontinued_date = self._date_too_late() + journal_discontinued_too_late.save(blocking=True) job = find_discontinued_soon.FindDiscontinuedSoonBackgroundTask.prepare("system") task = find_discontinued_soon.FindDiscontinuedSoonBackgroundTask(job) @@ -53,18 +55,18 @@ def test_discontinued_soon_found(self): def test_discontinued_soon_not_found(self): # None of these should be found - this one discontinues in 2 days - journal_discontinued_in_2_days = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True)) - journal_discontinued_in_2_days.set_id("1") - jbib = journal_discontinued_in_2_days.bibjson() + journal_discontinued_too_late = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True)) + journal_discontinued_too_late.set_id("1") + jbib = journal_discontinued_too_late.bibjson() jbib.title = "Discontinued In 2 days" - jbib.discontinued_date = self._date_in_2_days() - journal_discontinued_in_2_days.save(blocking=True) + jbib.discontinued_date = self._date_too_late() + journal_discontinued_too_late.save(blocking=True) # this one is not in doaj journal_not_in_doaj = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=False)) journal_not_in_doaj.set_id("2") jbib = journal_not_in_doaj.bibjson() - jbib.discontinued_date = self._date_tomorrow() + jbib.discontinued_date = self._date_to_found() journal_not_in_doaj.save(blocking=True) job = find_discontinued_soon.FindDiscontinuedSoonBackgroundTask.prepare("system") From 82b206a972ea41fb26fa8bb56f633e0e0fb3c008 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Tue, 7 Mar 2023 14:16:49 +0000 Subject: [PATCH 23/34] fix the filter --- portality/templates/doaj/toc.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/templates/doaj/toc.html b/portality/templates/doaj/toc.html index 27b71c7581..de5c99a52a 100644 --- a/portality/templates/doaj/toc.html +++ b/portality/templates/doaj/toc.html @@ -84,7 +84,7 @@

- {% if bibjson.discontinued_date | is_in_the_past %} + {% if bibjson.discontinued_date is not none and bibjson.discontinued_date | is_in_the_past %}

Ceased publication on {{ bibjson.discontinued_datestamp.strftime("%d %B %Y") }}

{% endif %} From ffd151dc0b45c737064493e7f9a08640538c5a0d Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Mon, 20 Mar 2023 12:47:14 +0000 Subject: [PATCH 24/34] remove unused method --- portality/models/v2/journal.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/portality/models/v2/journal.py b/portality/models/v2/journal.py index fb9246be1f..edd2d5b397 100644 --- a/portality/models/v2/journal.py +++ b/portality/models/v2/journal.py @@ -1122,24 +1122,3 @@ def query(self): {"created_date" : {"order" : "desc"}} ] } - -class JournalDiscontinuedDateQuery(object): - base_query = { - "track_total_hits": True, - "query": { - "bool": { - "must": [ - {"terms": {"bibjson.discontinued_date": ""}} - ] - } - }, - "size": 10000 - } - - def __init__(self, discontinued_date): - self.discontinued_date = discontinued_date - - def query(self): - q = deepcopy(self.base_query) - q["query"]["bool"]["must"][0]["terms"]["bibjson.discontinued_date"] = [self.discontinued_date] - return q \ No newline at end of file From b35475182fa88ecf95e58928e1c388238f61a39e Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Mon, 20 Mar 2023 12:48:59 +0000 Subject: [PATCH 25/34] default discontinued date delta to 0 --- portality/events/consumers/journal_discontinuing_soon_notify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/events/consumers/journal_discontinuing_soon_notify.py b/portality/events/consumers/journal_discontinuing_soon_notify.py index 5502ebddae..79f1349a24 100644 --- a/portality/events/consumers/journal_discontinuing_soon_notify.py +++ b/portality/events/consumers/journal_discontinuing_soon_notify.py @@ -56,7 +56,7 @@ def consume(cls, event): notification.created_by = cls.ID notification.classification = constants.NOTIFICATION_CLASSIFICATION_STATUS notification.long = svc.long_notification(cls.ID).format( - days=app.config.get('DISCONTINUED_DATE_DELTA',1), + days=app.config.get('DISCONTINUED_DATE_DELTA',0), title=journal.bibjson().title, id=journal.id ) From ccf94f1c2528275cf660627d455f0dbd556d6d8a Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Mon, 20 Mar 2023 13:03:14 +0000 Subject: [PATCH 26/34] add functionality to dates lib --- portality/lib/dates.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/portality/lib/dates.py b/portality/lib/dates.py index 7c81618cf8..c061f5af7d 100644 --- a/portality/lib/dates.py +++ b/portality/lib/dates.py @@ -71,9 +71,18 @@ def before_now(seconds: int): return before(datetime.utcnow(), seconds) -def after(timestamp, seconds): +def seconds_after(timestamp, seconds): return timestamp + timedelta(seconds=seconds) +def seconds_after_now(timestamp, seconds): + return after(datetime.utcnow(), seconds) + +def days_after(timestamp, days): + return timestamp + timedelta(days=days) + +def days_after_now(timestamp, days): + return days_after(datetime.utcnow(), days) + def eta(since, sofar, total): now = datetime.utcnow() @@ -116,3 +125,13 @@ def day_ranges(fro, to): def human_date(stamp, string_format="%d %B %Y"): return reformat(stamp, out_format=string_format) + +def is_before(mydate, comparison=None): + if comparison is None: + comparison = datetime.utcnow() + if isinstance(mydate, str): + mydate = parse(mydate) + if isinstance(comparison, str): + comparison = parse(comparison) + return mydate < comparison + From d30fbd2fd1db435ea77842cfa57bf3777abb73f6 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Mon, 20 Mar 2023 13:03:29 +0000 Subject: [PATCH 27/34] refactor DiscontinuedSoonQuery --- portality/tasks/find_discontinued_soon.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index 2214a6e560..9e0e4b82db 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -4,6 +4,7 @@ from portality.core import app from portality.bll import DOAJ +from portality.lib import dates from portality import models,app_email from portality.tasks.redis_huey import main_queue @@ -18,6 +19,9 @@ def _date(): return (datetime.datetime.today() + datetime.timedelta(days=app.config.get('DISCONTINUED_DATE_DELTA', 1))).strftime( '%Y-%m-%d') class DiscontinuedSoonQuery: + def __init__(self, time_delta=None): + self._delta = time_delta if time_delta is not None else app.config.get('DISCONTINUED_DATE_DELTA', 1); + self._date = days_after_now(days=time_delta) @classmethod def query(cls): return { @@ -26,7 +30,7 @@ def query(cls): "filter": { "bool" : { "must": [ - {"term" : {"bibjson.discontinued_date": _date()}}, + {"term" : {"bibjson.discontinued_date": self._date}}, {"term" : {"admin.in_doaj":True}} ] } From ef0510a06e3472b1183756584831ba8009eb3316 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Mon, 20 Mar 2023 13:11:00 +0000 Subject: [PATCH 28/34] use dates lib in jinja filter --- portality/app.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/portality/app.py b/portality/app.py index a11a928959..218fd4e793 100644 --- a/portality/app.py +++ b/portality/app.py @@ -281,8 +281,7 @@ def form_diff_table_subject_expand(val): @app.template_filter("is_in_the_past") def is_in_the_past(dttm): - date = datetime.strptime(dttm, "%Y-%m-%d").date() - return date <= date.today() + return dates.is_before(dates.today()) ####################################################### From a3d869491755c62708f15b9b153bb4cb3d756a08 Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Mon, 20 Mar 2023 13:26:46 +0000 Subject: [PATCH 29/34] fix errors --- .../consumers/journal_discontinuing_soon_notify.py | 3 --- portality/lib/dates.py | 4 ++-- portality/tasks/find_discontinued_soon.py | 14 +++++++------- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/portality/events/consumers/journal_discontinuing_soon_notify.py b/portality/events/consumers/journal_discontinuing_soon_notify.py index 79f1349a24..7f41dd8ba7 100644 --- a/portality/events/consumers/journal_discontinuing_soon_notify.py +++ b/portality/events/consumers/journal_discontinuing_soon_notify.py @@ -11,9 +11,6 @@ from portality.lib import edges from portality import dao - -from portality.models.v2.journal import JournalDiscontinuedDateQuery - class JournalDiscontinuingSoonNotify(EventConsumer): ID = "journal:assed:discontinuing_soon:notify" diff --git a/portality/lib/dates.py b/portality/lib/dates.py index c061f5af7d..7d394f0559 100644 --- a/portality/lib/dates.py +++ b/portality/lib/dates.py @@ -74,13 +74,13 @@ def before_now(seconds: int): def seconds_after(timestamp, seconds): return timestamp + timedelta(seconds=seconds) -def seconds_after_now(timestamp, seconds): +def seconds_after_now(seconds: int): return after(datetime.utcnow(), seconds) def days_after(timestamp, days): return timestamp + timedelta(days=days) -def days_after_now(timestamp, days): +def days_after_now(days: int): return days_after(datetime.utcnow(), days) diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index 9e0e4b82db..5111a4af51 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -20,17 +20,16 @@ def _date(): '%Y-%m-%d') class DiscontinuedSoonQuery: def __init__(self, time_delta=None): - self._delta = time_delta if time_delta is not None else app.config.get('DISCONTINUED_DATE_DELTA', 1); - self._date = days_after_now(days=time_delta) - @classmethod - def query(cls): + self._delta = time_delta if time_delta is not None else app.config.get('DISCONTINUED_DATE_DELTA', 0); + self._date = dates.days_after_now(days=self._delta) + def query(self): return { "query": { "bool": { "filter": { "bool" : { "must": [ - {"term" : {"bibjson.discontinued_date": self._date}}, + {"term" : {"bibjson.discontinued_date": dates.format(self._date, format="%Y-%m-%d")}}, {"term" : {"admin.in_doaj":True}} ] } @@ -47,7 +46,7 @@ class FindDiscontinuedSoonBackgroundTask(BackgroundTask): def find_journals_discontinuing_soon(self, job): jdata = [] - for journal in models.Journal.iterate(q=DiscontinuedSoonQuery.query(), keepalive='5m', wrap=True): + for journal in models.Journal.iterate(q=DiscontinuedSoonQuery().query(), keepalive='5m', wrap=True): # ~~->Journal:Model~~ jdata.append(journal.id) job.add_audit_message(Messages.DISCONTINUED_JOURNAL_FOUND_LOG.format(id=journal.id)) @@ -122,7 +121,8 @@ def find_discontinued_soon(job_id): # def find_journals_discontinuing_soon(): # jdata = [] # -# for journal in models.Journal.iterate(q=DiscontinuedSoonQuery.query(), keepalive='5m', wrap=True): +# q = DiscontinuedSoonQuery() +# for journal in models.Journal.iterate(q=q.query(), keepalive='5m', wrap=True): # # ~~->Journal:Model~~ # jdata.append(journal.id) # From 427ad2bb8b229422857740ee52f82bb640a3c23c Mon Sep 17 00:00:00 2001 From: Aga Domanska Date: Mon, 27 Mar 2023 12:00:03 +0100 Subject: [PATCH 30/34] new full new notification type - alert --- portality/constants.py | 2 +- .../static/js/edges/notifications.edge.js | 5 ++ portality/tasks/find_discontinued_soon.py | 52 +++++++++---------- 3 files changed, 32 insertions(+), 27 deletions(-) diff --git a/portality/constants.py b/portality/constants.py index 2600c65173..cfd7346711 100644 --- a/portality/constants.py +++ b/portality/constants.py @@ -55,7 +55,7 @@ EVENT_JOURNAL_EDITOR_GROUP_ASSIGNED = "journal:editor_group:assigned" EVENT_JOURNAL_DISCONTINUING_SOON = "journal:discontinuing_soon" -NOTIFICATION_CLASSIFICATION_STATUS = "status" +NOTIFICATION_CLASSIFICATION_STATUS = "alert" NOTIFICATION_CLASSIFICATION_STATUS_CHANGE = "status_change" NOTIFICATION_CLASSIFICATION_ASSIGN = "assign" NOTIFICATION_CLASSIFICATION_CREATE = "create" diff --git a/portality/static/js/edges/notifications.edge.js b/portality/static/js/edges/notifications.edge.js index 65788540ad..4b8c00e083 100644 --- a/portality/static/js/edges/notifications.edge.js +++ b/portality/static/js/edges/notifications.edge.js @@ -10,6 +10,10 @@ $.extend(true, doaj, { seen_url: "/dashboard/notifications/{notification_id}/seen", icons: { + alert: ` + + + `, finished: ` @@ -34,6 +38,7 @@ $.extend(true, doaj, { }, classifications: { + alert: "Requires attention", finished: "Task has completed", status_change: "Application status change", assign: "Assigned to user" diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index 5111a4af51..16786480d2 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -14,10 +14,6 @@ from portality.ui.messages import Messages from portality import constants - -def _date(): - return (datetime.datetime.today() + datetime.timedelta(days=app.config.get('DISCONTINUED_DATE_DELTA', 1))).strftime( - '%Y-%m-%d') class DiscontinuedSoonQuery: def __init__(self, time_delta=None): self._delta = time_delta if time_delta is not None else app.config.get('DISCONTINUED_DATE_DELTA', 0); @@ -43,6 +39,10 @@ def query(self): class FindDiscontinuedSoonBackgroundTask(BackgroundTask): __action__ = "find_discontinued_soon" + def __init__(self, time_delta=None): + self._delta = time_delta if time_delta is not None else app.config.get('DISCONTINUED_DATE_DELTA', 0); + self._date = dates.days_after_now(days=self._delta) + def find_journals_discontinuing_soon(self, job): jdata = [] @@ -63,7 +63,7 @@ def run(self): "system", { "journal": j, - "discontinue_date": _date() + "discontinue_date": self._date() })) else: job.add_audit_message(Messages.NO_DISCONTINUED_JOURNALS_FOUND_LOG) @@ -118,24 +118,24 @@ def find_discontinued_soon(job_id): # Test code - please do not clean up until after the tests -# def find_journals_discontinuing_soon(): -# jdata = [] -# -# q = DiscontinuedSoonQuery() -# for journal in models.Journal.iterate(q=q.query(), keepalive='5m', wrap=True): -# # ~~->Journal:Model~~ -# jdata.append(journal.id) -# -# return jdata -# -# if __name__ == "__main__": -# journals = find_journals_discontinuing_soon() -# if len(journals): -# for j in journals: -# DOAJ.eventsService().trigger(models.Event( -# constants.EVENT_JOURNAL_DISCONTINUING_SOON, -# "system", -# { -# "journal": j, -# "discontinue_date": _date() -# })) \ No newline at end of file +def find_journals_discontinuing_soon(): + jdata = [] + + q = DiscontinuedSoonQuery() + for journal in models.Journal.iterate(q=q.query(), keepalive='5m', wrap=True): + # ~~->Journal:Model~~ + jdata.append(journal.id) + + return jdata + +if __name__ == "__main__": + journals = find_journals_discontinuing_soon() + if len(journals): + for j in journals: + DOAJ.eventsService().trigger(models.Event( + constants.EVENT_JOURNAL_DISCONTINUING_SOON, + "system", + { + "journal": j, + "discontinue_date": dates.days_after_now(days=0) + })) \ No newline at end of file From 890f2263acba70f9512986c17ee761b160249622 Mon Sep 17 00:00:00 2001 From: Aga Date: Fri, 14 Apr 2023 10:05:18 +0100 Subject: [PATCH 31/34] fix is_before usage --- portality/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/app.py b/portality/app.py index 218fd4e793..1083899d85 100644 --- a/portality/app.py +++ b/portality/app.py @@ -281,7 +281,7 @@ def form_diff_table_subject_expand(val): @app.template_filter("is_in_the_past") def is_in_the_past(dttm): - return dates.is_before(dates.today()) + return dates.is_before(dttm, dates.today()) ####################################################### From 6139624309485ee1f391e943b3f5c0a66eb756dc Mon Sep 17 00:00:00 2001 From: Aga Date: Mon, 17 Apr 2023 13:28:48 +0100 Subject: [PATCH 32/34] clean up --- portality/tasks/find_discontinued_soon.py | 26 +---------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/portality/tasks/find_discontinued_soon.py b/portality/tasks/find_discontinued_soon.py index 16786480d2..d9c281abb2 100644 --- a/portality/tasks/find_discontinued_soon.py +++ b/portality/tasks/find_discontinued_soon.py @@ -114,28 +114,4 @@ def scheduled_find_discontinued_soon(): def find_discontinued_soon(job_id): job = models.BackgroundJob.pull(job_id) task = FindDiscontinuedSoonBackgroundTask(job) - BackgroundApi.execute(task) - - -# Test code - please do not clean up until after the tests -def find_journals_discontinuing_soon(): - jdata = [] - - q = DiscontinuedSoonQuery() - for journal in models.Journal.iterate(q=q.query(), keepalive='5m', wrap=True): - # ~~->Journal:Model~~ - jdata.append(journal.id) - - return jdata - -if __name__ == "__main__": - journals = find_journals_discontinuing_soon() - if len(journals): - for j in journals: - DOAJ.eventsService().trigger(models.Event( - constants.EVENT_JOURNAL_DISCONTINUING_SOON, - "system", - { - "journal": j, - "discontinue_date": dates.days_after_now(days=0) - })) \ No newline at end of file + BackgroundApi.execute(task) \ No newline at end of file From 73416774d544d29013f7796fb8b8b1e81687f8fd Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Tue, 18 Apr 2023 12:48:16 +0100 Subject: [PATCH 33/34] Update the cron schedule for discontinued task --- portality/settings.py | 2 +- test.cfg | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/portality/settings.py b/portality/settings.py index 645ae5da18..30c9ddc85f 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -434,7 +434,7 @@ "anon_export": {"month": "*", "day": "10", "day_of_week": "*", "hour": "6", "minute": "30"}, "old_data_cleanup": {"month": "*", "day": "12", "day_of_week": "*", "hour": "6", "minute": "30"}, "monitor_bgjobs": {"month": "*", "day": "*/6", "day_of_week": "*", "hour": "10", "minute": "0"}, - "find_discontinued_soon": {"month": "*", "day": "*", "day_of_week": "*", "hour": "7", "minute": "0"}, + "find_discontinued_soon": {"month": "*", "day": "*", "day_of_week": "*", "hour": "0", "minute": "3"} } HUEY_TASKS = { diff --git a/test.cfg b/test.cfg index 37eea9ca92..907b59bb7b 100644 --- a/test.cfg +++ b/test.cfg @@ -55,7 +55,8 @@ HUEY_SCHEDULE = { "harvest": {"month": "*", "day": "*", "day_of_week": "*", "hour": "5", "minute": "30"}, "anon_export": CRON_NEVER, "old_data_cleanup": {"month": "*", "day": "*", "day_of_week": "3", "hour": "12", "minute": "0"}, - "monitor_bgjobs": {"month": "*", "day": "*/6", "day_of_week": "*", "hour": "10", "minute": "0"} + "monitor_bgjobs": {"month": "*", "day": "*/6", "day_of_week": "*", "hour": "10", "minute": "0"}, + "find_discontinued_soon": {"month": "*", "day": "*", "day_of_week": "*", "hour": "0", "minute": "3"} } # ======================= From ea294dc4e1419cad8c692c12c60645cf5be48de4 Mon Sep 17 00:00:00 2001 From: Steve Eardley Date: Tue, 2 May 2023 17:52:57 +0100 Subject: [PATCH 34/34] Add missing import for new background job --- portality/tasks/consumer_long_running.py | 2 +- portality/tasks/consumer_main_queue.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/portality/tasks/consumer_long_running.py b/portality/tasks/consumer_long_running.py index bf0aa61b61..ff8763e148 100644 --- a/portality/tasks/consumer_long_running.py +++ b/portality/tasks/consumer_long_running.py @@ -16,4 +16,4 @@ from portality.tasks.harvester import scheduled_harvest # noqa from portality.tasks.anon_export import scheduled_anon_export, anon_export # noqa from portality.tasks.old_data_cleanup import scheduled_old_data_cleanup, execute_old_data_cleanup # noqa -from portality.tasks.monitor_bgjobs import scheduled_monitor_bgjobs, execute_monitor_bgjobs +from portality.tasks.monitor_bgjobs import scheduled_monitor_bgjobs, execute_monitor_bgjobs # noqa diff --git a/portality/tasks/consumer_main_queue.py b/portality/tasks/consumer_main_queue.py index 2eda75d8da..ea3774d560 100644 --- a/portality/tasks/consumer_main_queue.py +++ b/portality/tasks/consumer_main_queue.py @@ -26,3 +26,4 @@ from portality.tasks.async_workflow_notifications import async_workflow_notifications # noqa from portality.tasks.check_latest_es_backup import scheduled_check_latest_es_backup, check_latest_es_backup # noqa from portality.tasks.request_es_backup import scheduled_request_es_backup, request_es_backup # noqa +from portality.tasks.find_discontinued_soon import scheduled_find_discontinued_soon, find_discontinued_soon # noqa