Feature/3959 bg monitoring fixed

deploy/dev/huey_events.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+# ~~LongRunning:Queue->Huey:Technology~~
+huey_consumer.py portality.tasks.consumer_events_queue.events_queue >> ~/huey_events_queue.log 2>&1

deploy/dev/huey_scheduled_long.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+# ~~LongRunning:Queue->Huey:Technology~~
+huey_consumer.py portality.tasks.consumer_scheduled_long_queue.scheduled_long_queue >> ~/huey_scheduled_long_queue.log 2>&1

deploy/dev/huey_scheduled_short.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+# ~~LongRunning:Queue->Huey:Technology~~
+huey_consumer.py portality.tasks.consumer_scheduled_short_queue.scheduled_short_queue >> ~/huey_scheduled_short_queue.log 2>&1

deploy/supervisor/production-background/huey-events.conf

@@ -0,0 +1,9 @@
+[program:huey-events]
+command=/home/cloo/doaj/bin/python /home/cloo/doaj/bin/huey_consumer.py -v portality.tasks.consumer_events_queue.events_queue
+environment= DOAJENV=production
+user=cloo
+directory=/home/cloo/doaj/src/doaj
+stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log
+stderr_logfile=/var/log/supervisor/%(program_name)s-error.log
+autostart=true
+autorestart=true

deploy/supervisor/production-background/huey-scheduled-long.conf

@@ -0,0 +1,9 @@
+[program:huey-scheduled-long]
+command=/home/cloo/doaj/bin/python /home/cloo/doaj/bin/huey_consumer.py -v portality.tasks.consumer_scheduled_long_queue.scheduled_long_queue
+environment= DOAJENV=production
+user=cloo
+directory=/home/cloo/doaj/src/doaj
+stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log
+stderr_logfile=/var/log/supervisor/%(program_name)s-error.log
+autostart=true
+autorestart=true

deploy/supervisor/production-background/huey-scheduled-short.conf

@@ -0,0 +1,9 @@
+[program:huey-scheduled-short]
+command=/home/cloo/doaj/bin/python /home/cloo/doaj/bin/huey_consumer.py -v portality.tasks.consumer_scheduled_short_queue.scheduled_short_queue
+environment= DOAJENV=production
+user=cloo
+directory=/home/cloo/doaj/src/doaj
+stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log
+stderr_logfile=/var/log/supervisor/%(program_name)s-error.log
+autostart=true
+autorestart=true

deploy/supervisor/test/huey-events.conf

@@ -0,0 +1,9 @@
+[program:huey-events]
+command=/home/cloo/doaj/bin/python /home/cloo/doaj/bin/huey_consumer.py -v portality.tasks.consumer_events_queue.events_queue
+environment= DOAJENV=test
+user=cloo
+directory=/home/cloo/doaj/src/doaj
+stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log
+stderr_logfile=/var/log/supervisor/%(program_name)s-error.log
+autostart=true
+autorestart=true

deploy/supervisor/test/huey-scheduled-long.conf

@@ -0,0 +1,9 @@
+[program:huey-scheduled-long]
+command=/home/cloo/doaj/bin/python /home/cloo/doaj/bin/huey_consumer.py -v portality.tasks.consumer_scheduled_long_queue.scheduled_long_queue
+environment= DOAJENV=test
+user=cloo
+directory=/home/cloo/doaj/src/doaj
+stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log
+stderr_logfile=/var/log/supervisor/%(program_name)s-error.log
+autostart=true
+autorestart=true

deploy/supervisor/test/huey-scheduled-short.conf

@@ -0,0 +1,9 @@
+[program:huey-scheduled-short]
+command=/home/cloo/doaj/bin/python /home/cloo/doaj/bin/huey_consumer.py -v portality.tasks.consumer_scheduled_short_queue.scheduled_short_queue
+environment= DOAJENV=test
+user=cloo
+directory=/home/cloo/doaj/src/doaj
+stdout_logfile=/var/log/supervisor/%(program_name)s-stdout.log
+stderr_logfile=/var/log/supervisor/%(program_name)s-error.log
+autostart=true
+autorestart=true

doajtest/fixtures/background.py

@@ -52,12 +52,14 @@ def cleanup(self):
 
 
 def save_mock_bgjob(action=None, status=None, created_before_sec=0, is_save=True,
-                    queue_id=None):
+                    blocking=True, queue_id=None):
     bgjob = BackgroundJob()
 
-    if action:
+    if not action:
         from portality.tasks.journal_csv import JournalCSVBackgroundTask
         bgjob.action = JournalCSVBackgroundTask.__action__
+    else:
+        bgjob.action = action
 
     if status:
         bgjob._set_with_struct("status", status)
@@ -69,6 +71,6 @@ def save_mock_bgjob(action=None, status=None, created_before_sec=0, is_save=True
         bgjob.queue_id = queue_id
 
     if is_save:
-        bgjob.save(blocking=True)
+        bgjob.save(blocking=blocking)
 
     return bgjob

doajtest/helpers.py

@@ -19,7 +19,7 @@
 from portality.lib import paths, dates
 from portality.lib.dates import FMT_DATE_STD
 from portality.lib.thread_utils import wait_until
-from portality.tasks.redis_huey import main_queue, long_running
+from portality.tasks.redis_huey import events_queue, scheduled_short_queue, scheduled_long_queue
 from portality.util import url_for
 
 
@@ -163,10 +163,13 @@ def setUpClass(cls) -> None:
         # always_eager has been replaced by immediate
         # for huey version > 2
         # https://huey.readthedocs.io/en/latest/guide.html
-        main_queue.always_eager = True
-        long_running.always_eager = True
-        main_queue.immediate = True
-        long_running.immediate = True
+        events_queue.always_eager = True
+        scheduled_short_queue.always_eager = True
+        scheduled_long_queue.always_eager = True
+
+        events_queue.immediate = True
+        scheduled_short_queue.immediate = True
+        scheduled_long_queue.immediate = True
 
         dao.DomainObject.save = dao_proxy(dao.DomainObject.save, type="instance")
         dao.DomainObject.delete = dao_proxy(dao.DomainObject.delete, type="instance")

doajtest/matrices/background_task_status/background_task_status.matrix.csv

@@ -0,0 +1,36 @@
+test_id,in_queue,oldest_queued,error_count,error_age,lrs_success_or_error,queued,errors,lrs
+1,0,young,0,out_of_period,complete,stable,stable,stable
+2,0,young,0,out_of_period,error,stable,stable,unstable
+3,0,young,0,out_of_period,empty,stable,stable,unstable
+4,0,young,1,in_period,complete,stable,unstable,stable
+5,0,young,1,in_period,error,stable,unstable,unstable
+6,0,young,1,out_of_period,complete,stable,stable,stable
+7,0,young,1,out_of_period,error,stable,stable,unstable
+8,1,old,0,out_of_period,complete,unstable,stable,stable
+9,1,old,0,out_of_period,error,unstable,stable,unstable
+10,1,old,0,out_of_period,empty,unstable,stable,unstable
+11,1,old,1,in_period,complete,unstable,unstable,stable
+12,1,old,1,in_period,error,unstable,unstable,unstable
+13,1,old,1,out_of_period,complete,unstable,stable,stable
+14,1,old,1,out_of_period,error,unstable,stable,unstable
+15,1,young,0,out_of_period,complete,stable,stable,stable
+16,1,young,0,out_of_period,error,stable,stable,unstable
+17,1,young,0,out_of_period,empty,stable,stable,unstable
+18,1,young,1,in_period,complete,stable,unstable,stable
+19,1,young,1,in_period,error,stable,unstable,unstable
+20,1,young,1,out_of_period,complete,stable,stable,stable
+21,1,young,1,out_of_period,error,stable,stable,unstable
+22,2,old,0,out_of_period,complete,unstable,stable,stable
+23,2,old,0,out_of_period,error,unstable,stable,unstable
+24,2,old,0,out_of_period,empty,unstable,stable,unstable
+25,2,old,1,in_period,complete,unstable,unstable,stable
+26,2,old,1,in_period,error,unstable,unstable,unstable
+27,2,old,1,out_of_period,complete,unstable,stable,stable
+28,2,old,1,out_of_period,error,unstable,stable,unstable
+29,2,young,0,out_of_period,complete,unstable,stable,stable
+30,2,young,0,out_of_period,error,unstable,stable,unstable
+31,2,young,0,out_of_period,empty,unstable,stable,unstable
+32,2,young,1,in_period,complete,unstable,unstable,stable
+33,2,young,1,in_period,error,unstable,unstable,unstable
+34,2,young,1,out_of_period,complete,unstable,stable,stable
+35,2,young,1,out_of_period,error,unstable,stable,unstable

doajtest/matrices/background_task_status/background_task_status.settings.csv

@@ -0,0 +1,17 @@
+field,test_id,in_queue,oldest_queued,error_count,error_age,lrs_success_or_error,queued,errors,lrs
+type,index,generated,generated,generated,generated,generated,conditional,conditional,conditional
+default,,,,,,,stable,stable,stable
+,,,,,,,,,
+values,,0,old,0,in_period,complete,stable,stable,stable
+values,,1,young,1,out_of_period,error,unstable,unstable,unstable
+values,,2,,,,empty,,,
+,,,,,,,,,
+constraint in_queue,,0,young,,,,,,
+constraint error_count,,,,0,out_of_period,,,,
+constraint error_count,,,,1,,!empty,,,
+,,,,,,,,,
+conditional queued,,2,,,,,unstable,,
+conditional queued,,,old,,,,unstable,,
+conditional errors,,,,1,in_period,,,unstable,
+conditional lrs,,,,,,error,,,unstable
+conditional lrs,,,,,,empty,,,unstable

doajtest/matrices/background_task_status/background_task_status.settings.json

@@ -0,0 +1,150 @@
+{
+  "parameters": [
+    {
+      "name": "test_id",
+      "type": "index"
+    },
+    {
+      "name": "in_queue",
+      "type": "generated",
+      "values": {
+        "0": {
+          "constraints": {
+            "oldest_queued": {
+              "or": [
+                "young"
+              ]
+            }
+          }
+        },
+        "1": {},
+        "2": {}
+      }
+    },
+    {
+      "name": "oldest_queued",
+      "type": "generated",
+      "values": {
+        "old": {},
+        "young": {}
+      }
+    },
+    {
+      "name": "error_count",
+      "type": "generated",
+      "values": {
+        "0": {
+          "constraints": {
+            "error_age": {
+              "or": [
+                "out_of_period"
+              ]
+            }
+          }
+        },
+        "1": {
+          "constraints": {
+            "lrs_success_or_error": {
+              "nor": [
+                "empty"
+              ]
+            }
+          }
+        }
+      }
+    },
+    {
+      "name": "error_age",
+      "type": "generated",
+      "values": {
+        "in_period": {},
+        "out_of_period": {}
+      }
+    },
+    {
+      "name": "lrs_success_or_error",
+      "type": "generated",
+      "values": {
+        "complete": {},
+        "error": {},
+        "empty": {}
+      }
+    },
+    {
+      "name": "queued",
+      "type": "conditional",
+      "default": "stable",
+      "values": {
+        "stable": {},
+        "unstable": {
+          "conditions": [
+            {
+              "in_queue": {
+                "or": [
+                  "2"
+                ]
+              }
+            },
+            {
+              "oldest_queued": {
+                "or": [
+                  "old"
+                ]
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "name": "errors",
+      "type": "conditional",
+      "default": "stable",
+      "values": {
+        "stable": {},
+        "unstable": {
+          "conditions": [
+            {
+              "error_count": {
+                "or": [
+                  "1"
+                ]
+              },
+              "error_age": {
+                "or": [
+                  "in_period"
+                ]
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "name": "lrs",
+      "type": "conditional",
+      "default": "stable",
+      "values": {
+        "stable": {},
+        "unstable": {
+          "conditions": [
+            {
+              "lrs_success_or_error": {
+                "or": [
+                  "error"
+                ]
+              }
+            },
+            {
+              "lrs_success_or_error": {
+                "or": [
+                  "empty"
+                ]
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

doajtest/unit/test_background_helper.py

@@ -7,15 +7,16 @@
 from portality.background import BackgroundTask
 from portality.core import app
 from portality.tasks.helpers import background_helper
-from portality.tasks.redis_huey import long_running, main_queue
+from portality.tasks.redis_huey import events_queue, scheduled_short_queue, scheduled_long_queue
 
 
 class TestBackgroundHelper(TestCase):
 
     def test_get_queue_id_by_task_queue(self):
         cases = [
-            (long_running, constants.BGJOB_QUEUE_ID_LONG),
-            (main_queue, constants.BGJOB_QUEUE_ID_MAIN),
+            (events_queue, constants.BGJOB_QUEUE_ID_EVENTS),
+            (scheduled_long_queue, constants.BGJOB_QUEUE_ID_SCHEDULED_LONG),
+            (scheduled_short_queue, constants.BGJOB_QUEUE_ID_SCHEDULED_SHORT),
             (None, constants.BGJOB_QUEUE_ID_UNKNOWN),
         ]
 
@@ -57,7 +58,7 @@ def tearDownClass(cls) -> None:
         helpers.patch_config(app, cls.org_config)
 
     def test_register_schedule(self):
-        helper = background_helper.RedisHueyTaskHelper(main_queue, fixture_bgtask_class(self.task_name_a))
+        helper = background_helper.RedisHueyTaskHelper(scheduled_short_queue, fixture_bgtask_class(self.task_name_a))
 
         @helper.register_schedule
         def _fn():
@@ -66,15 +67,15 @@ def _fn():
         assert isinstance(_fn, huey.api.TaskWrapper)
 
     def test_register_schedule__schedule_not_found(self):
-        helper = background_helper.RedisHueyTaskHelper(main_queue,
+        helper = background_helper.RedisHueyTaskHelper(scheduled_short_queue,
                                                        fixture_bgtask_class(self.task_name_schedule_not_exist))
         with self.assertRaises(RuntimeError):
             @helper.register_schedule
             def _fn():
                 print('fake fn')
 
     def test_register_execute(self):
-        helper = background_helper.RedisHueyTaskHelper(main_queue, fixture_bgtask_class(self.task_name_b))
+        helper = background_helper.RedisHueyTaskHelper(scheduled_short_queue, fixture_bgtask_class(self.task_name_b))
 
         @helper.register_execute(is_load_config=True)
         def _fn():
@@ -84,7 +85,7 @@ def _fn():
         assert _fn.retries == self.expected_retries
 
     def test_register_execute__config_not_found(self):
-        helper = background_helper.RedisHueyTaskHelper(main_queue,
+        helper = background_helper.RedisHueyTaskHelper(events_queue,
                                                        fixture_bgtask_class(self.task_name_schedule_not_exist))
 
         with self.assertRaises(RuntimeError):
@@ -93,7 +94,7 @@ def _fn():
                 print('fake fn')
 
     def test_register_execute__without_load_config(self):
-        helper = background_helper.RedisHueyTaskHelper(main_queue,
+        helper = background_helper.RedisHueyTaskHelper(events_queue,
                                                        fixture_bgtask_class(self.task_name_schedule_not_exist))
 
         @helper.register_execute(is_load_config=False)