From 892c54f6b5ccc309cb54a37b8fe3cca659ef93d3 Mon Sep 17 00:00:00 2001 From: Alexander Haase Date: Mon, 7 Oct 2024 00:40:30 +0200 Subject: [PATCH 01/12] Fix check for existing jobs If a job is to be enqueued once and no specific scheduled time is specified, any scheduled time of existing jobs will be valid. Only if a specific scheduled time is specified for 'enqueue_once()' can it be evaluated. --- netbox/netbox/jobs.py | 2 +- netbox/netbox/tests/test_jobs.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/netbox/netbox/jobs.py b/netbox/netbox/jobs.py index 087c2489687..b357c714473 100644 --- a/netbox/netbox/jobs.py +++ b/netbox/netbox/jobs.py @@ -127,7 +127,7 @@ class scheduled for `instance`, the existing job will be updated if necessary. T if job: # If the job parameters haven't changed, don't schedule a new job and keep the current schedule. Otherwise, # delete the existing job and schedule a new job instead. - if (schedule_at and job.scheduled == schedule_at) and (job.interval == interval): + if (not schedule_at or job.scheduled == schedule_at) and (job.interval == interval): return job job.delete() diff --git a/netbox/netbox/tests/test_jobs.py b/netbox/netbox/tests/test_jobs.py index cb3024038ae..fec9ba83772 100644 --- a/netbox/netbox/tests/test_jobs.py +++ b/netbox/netbox/tests/test_jobs.py @@ -90,6 +90,15 @@ def test_enqueue_once_twice_same(self): self.assertEqual(job1, job2) self.assertEqual(TestJobRunner.get_jobs(instance).count(), 1) + def test_enqueue_once_twice_same_no_schedule_at(self): + instance = Job() + schedule_at = self.get_schedule_at() + job1 = TestJobRunner.enqueue_once(instance, schedule_at=schedule_at) + job2 = TestJobRunner.enqueue_once(instance) + + self.assertEqual(job1, job2) + self.assertEqual(TestJobRunner.get_jobs(instance).count(), 1) + def test_enqueue_once_twice_different_schedule_at(self): instance = Job() job1 = TestJobRunner.enqueue_once(instance, schedule_at=self.get_schedule_at()) From 1f103eddaccbbac0e05d001edafdfa21c5607d13 Mon Sep 17 00:00:00 2001 From: Alexander Haase Date: Mon, 7 Oct 2024 14:15:39 +0200 Subject: [PATCH 02/12] Allow system jobs to be registered A new registry key allows background system jobs to be registered and automatically scheduled when rqworker starts. --- netbox/core/management/commands/rqworker.py | 13 +++++++++++++ netbox/netbox/registry.py | 1 + netbox/netbox/utils.py | 12 ++++++++++++ 3 files changed, 26 insertions(+) diff --git a/netbox/core/management/commands/rqworker.py b/netbox/core/management/commands/rqworker.py index e1fb6fd1118..0e785be4685 100644 --- a/netbox/core/management/commands/rqworker.py +++ b/netbox/core/management/commands/rqworker.py @@ -1,7 +1,10 @@ import logging +from django.core.management.base import CommandError from django_rq.management.commands.rqworker import Command as _Command +from netbox.registry import registry + DEFAULT_QUEUES = ('high', 'default', 'low') @@ -14,6 +17,16 @@ class Command(_Command): of only the 'default' queue). """ def handle(self, *args, **options): + # Setup system jobs. + for job in registry['system_jobs'].values(): + if getattr(job.Meta, 'enabled', True): + try: + logger.debug(f"Scheduling system job {job.name}") + job.enqueue_once(interval=getattr(job.Meta, 'interval')) + + except AttributeError as e: + raise CommandError(f"Job {job.name} is missing required attribute in Meta: {e.name}") + # Run the worker with scheduler functionality options['with_scheduler'] = True diff --git a/netbox/netbox/registry.py b/netbox/netbox/registry.py index 0920cbccf96..48d7921f259 100644 --- a/netbox/netbox/registry.py +++ b/netbox/netbox/registry.py @@ -30,6 +30,7 @@ def __delitem__(self, key): 'models': collections.defaultdict(set), 'plugins': dict(), 'search': dict(), + 'system_jobs': dict(), 'tables': collections.defaultdict(dict), 'views': collections.defaultdict(dict), 'widgets': dict(), diff --git a/netbox/netbox/utils.py b/netbox/netbox/utils.py index f27d1b5f7fc..60067c55343 100644 --- a/netbox/netbox/utils.py +++ b/netbox/netbox/utils.py @@ -3,6 +3,7 @@ __all__ = ( 'get_data_backend_choices', 'register_data_backend', + 'register_system_job', ) @@ -24,3 +25,14 @@ def _wrapper(cls): return cls return _wrapper + + +def register_system_job(): + """ + Decorator for registering a `JobRunner` class as system background job. + """ + def _wrapper(cls): + registry['system_jobs'][cls.name] = cls + return cls + + return _wrapper From ab3ac3c12be9ce3463cd67055cec91cd0f272a1f Mon Sep 17 00:00:00 2001 From: Alexander Haase Date: Mon, 7 Oct 2024 15:10:03 +0200 Subject: [PATCH 03/12] Test scheduling of system jobs --- netbox/netbox/tests/test_jobs.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/netbox/netbox/tests/test_jobs.py b/netbox/netbox/tests/test_jobs.py index fec9ba83772..0e230d46772 100644 --- a/netbox/netbox/tests/test_jobs.py +++ b/netbox/netbox/tests/test_jobs.py @@ -136,3 +136,30 @@ def test_enqueue_once_after_enqueue(self): self.assertNotEqual(job1, job2) self.assertRaises(Job.DoesNotExist, job1.refresh_from_db) self.assertEqual(TestJobRunner.get_jobs(instance).count(), 1) + + +class SystemJobTest(JobRunnerTestCase): + """ + Test that system jobs can be scheduled. + + General functionality already tested by `JobRunnerTest` and `EnqueueTest`. + """ + + def test_scheduling(self): + # Can job be enqueued? + job = TestJobRunner.enqueue(schedule_at=self.get_schedule_at()) + self.assertIsInstance(job, Job) + self.assertEqual(TestJobRunner.get_jobs().count(), 1) + + # Can job be deleted again? + job.delete() + self.assertRaises(Job.DoesNotExist, job.refresh_from_db) + self.assertEqual(TestJobRunner.get_jobs().count(), 0) + + def test_enqueue_once(self): + schedule_at = self.get_schedule_at() + job1 = TestJobRunner.enqueue_once(schedule_at=schedule_at) + job2 = TestJobRunner.enqueue_once(schedule_at=schedule_at) + + self.assertEqual(job1, job2) + self.assertEqual(TestJobRunner.get_jobs().count(), 1) From 39e5c0d24c700d251ecf5a49c2bcd068d09472b6 Mon Sep 17 00:00:00 2001 From: Alexander Haase Date: Mon, 7 Oct 2024 18:10:02 +0200 Subject: [PATCH 04/12] Fix plugins scheduled job documentation The documentation reflected a non-production state of the JobRunner framework left over from development. Now a more practical example demonstrates the usage. --- docs/plugins/development/background-jobs.md | 27 +++++++++------------ 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/docs/plugins/development/background-jobs.md b/docs/plugins/development/background-jobs.md index 873390a5886..fb9d3511d4f 100644 --- a/docs/plugins/development/background-jobs.md +++ b/docs/plugins/development/background-jobs.md @@ -46,25 +46,20 @@ As described above, jobs can be scheduled for immediate execution or at any late #### Example -```python title="jobs.py" -from netbox.jobs import JobRunner +```python title="models.py" +from django.db import models +from netbox.models import NetBoxModel +from .jobs import MyTestJob +class MyModel(NetBoxModel): + foo = models.CharField() -class MyHousekeepingJob(JobRunner): - class Meta: - name = "Housekeeping" + def save(self, *args, **kwargs): + MyTestJob.enqueue_once(instance=self, interval=60) + return super().save(*args, **kwargs) - def run(self, *args, **kwargs): - # your logic goes here -``` - -```python title="__init__.py" -from netbox.plugins import PluginConfig - -class MyPluginConfig(PluginConfig): - def ready(self): - from .jobs import MyHousekeepingJob - MyHousekeepingJob.setup(interval=60) + def sync(self): + MyTestJob.enqueue(instance=self) ``` ## Task queues From d7b03ee47e734a2bd61d5055494af83e4c9f8652 Mon Sep 17 00:00:00 2001 From: Alexander Haase Date: Mon, 7 Oct 2024 18:37:18 +0200 Subject: [PATCH 05/12] Allow plugins to register system jobs --- docs/plugins/development/background-jobs.md | 37 ++++++++++++++++++++- docs/plugins/development/data-backends.md | 2 +- netbox/netbox/plugins/__init__.py | 9 ++++- netbox/netbox/tests/dummy_plugin/jobs.py | 14 ++++++++ netbox/netbox/tests/test_plugins.py | 8 +++++ 5 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 netbox/netbox/tests/dummy_plugin/jobs.py diff --git a/docs/plugins/development/background-jobs.md b/docs/plugins/development/background-jobs.md index fb9d3511d4f..e950c26e6ce 100644 --- a/docs/plugins/development/background-jobs.md +++ b/docs/plugins/development/background-jobs.md @@ -31,12 +31,20 @@ You can schedule the background job from within your code (e.g. from a model's ` ### Attributes -`JobRunner` attributes are defined under a class named `Meta` within the job. These are optional, but encouraged. +`JobRunner` attributes are defined under a class named `Meta` within the job. These are optional (unless specified otherwise), but encouraged. #### `name` This is the human-friendly names of your background job. If omitted, the class name will be used. +#### `enabled` + +When the `JobRunner` is defined as [system job](#system-jobs), this attribute controls whether a job will be scheduled. By default, this attribute is `True`. + +#### `interval` *(required for system jobs)* + +When the `JobRunner` is defined as [system job](#system-jobs), this attribute controls the interval of the scheduled job. + ### Scheduled Jobs As described above, jobs can be scheduled for immediate execution or at any later time using the `enqueue()` method. However, for management purposes, the `enqueue_once()` method allows a job to be scheduled exactly once avoiding duplicates. If a job is already scheduled for a particular instance, a second one won't be scheduled, respecting thread safety. An example use case would be to schedule a periodic task that is bound to an instance in general, but not to any event of that instance (such as updates). The parameters of the `enqueue_once()` method are identical to those of `enqueue()`. @@ -62,6 +70,33 @@ class MyModel(NetBoxModel): MyTestJob.enqueue(instance=self) ``` + +### System Jobs + +Some plugins may implement background jobs that are decoupled from any object and the request-response cycle. Typical use cases would be housekeeping tasks or synchronization jobs. These can be created using *system jobs*. The `JobRunner` class has everything included to provide this type of job as well. Just add the appropriate metadata to let NetBox schedule all background jobs automatically. + +!!! info + All system jobs are automatically scheduled just before the `./manage.py rqworker` command is started and the job queue is processed. The schedules are also checked at each restart of this process. + +#### Example + +```python title="jobs.py" +from netbox.jobs import JobRunner +from .models import MyModel + +class MyHousekeepingJob(JobRunner): + class Meta: + name = "My Housekeeping Job" + interval = 60 # every 60 minutes + + def run(self, *args, **kwargs): + MyModel.objects.filter(foo='bar').delete() + +system_jobs = ( + MyHousekeepingJob, +) +``` + ## Task queues Three task queues of differing priority are defined by default: diff --git a/docs/plugins/development/data-backends.md b/docs/plugins/development/data-backends.md index 8b7226a4139..0c6d44d9516 100644 --- a/docs/plugins/development/data-backends.md +++ b/docs/plugins/development/data-backends.md @@ -18,6 +18,6 @@ backends = [MyDataBackend] ``` !!! tip - The path to the list of search indexes can be modified by setting `data_backends` in the PluginConfig instance. + The path to the list of data backends can be modified by setting `data_backends` in the PluginConfig instance. ::: netbox.data_backends.DataBackend diff --git a/netbox/netbox/plugins/__init__.py b/netbox/netbox/plugins/__init__.py index e2f0f22fce4..76bb9d26bcc 100644 --- a/netbox/netbox/plugins/__init__.py +++ b/netbox/netbox/plugins/__init__.py @@ -8,7 +8,7 @@ from netbox.registry import registry from netbox.search import register_search -from netbox.utils import register_data_backend +from netbox.utils import register_data_backend, register_system_job from .navigation import * from .registration import * from .templates import * @@ -26,6 +26,7 @@ DEFAULT_RESOURCE_PATHS = { 'search_indexes': 'search.indexes', 'data_backends': 'data_backends.backends', + 'system_jobs': 'jobs.system_jobs', 'graphql_schema': 'graphql.schema', 'menu': 'navigation.menu', 'menu_items': 'navigation.menu_items', @@ -73,6 +74,7 @@ class PluginConfig(AppConfig): # Optional plugin resources search_indexes = None data_backends = None + system_jobs = None graphql_schema = None menu = None menu_items = None @@ -111,6 +113,11 @@ def ready(self): for backend in data_backends: register_data_backend()(backend) + # Register system jobs (if defined) + system_jobs = self._load_resource('system_jobs') or [] + for job in system_jobs: + register_system_job()(job) + # Register template content (if defined) if template_extensions := self._load_resource('template_extensions'): register_template_extensions(template_extensions) diff --git a/netbox/netbox/tests/dummy_plugin/jobs.py b/netbox/netbox/tests/dummy_plugin/jobs.py new file mode 100644 index 00000000000..fbb08c1863d --- /dev/null +++ b/netbox/netbox/tests/dummy_plugin/jobs.py @@ -0,0 +1,14 @@ +from netbox.jobs import JobRunner + + +class DummySystemJob(JobRunner): + class Meta: + interval = 60 + + def run(self, *args, **kwargs): + pass + + +system_jobs = ( + DummySystemJob, +) diff --git a/netbox/netbox/tests/test_plugins.py b/netbox/netbox/tests/test_plugins.py index ba44378c5ce..0fe8549a85a 100644 --- a/netbox/netbox/tests/test_plugins.py +++ b/netbox/netbox/tests/test_plugins.py @@ -7,6 +7,7 @@ from netbox.tests.dummy_plugin import config as dummy_config from netbox.tests.dummy_plugin.data_backends import DummyBackend +from netbox.tests.dummy_plugin.jobs import DummySystemJob from netbox.plugins.navigation import PluginMenu from netbox.plugins.utils import get_plugin_config from netbox.graphql.schema import Query @@ -130,6 +131,13 @@ def test_data_backends(self): self.assertIn('dummy', registry['data_backends']) self.assertIs(registry['data_backends']['dummy'], DummyBackend) + def test_system_jobs(self): + """ + Check registered system jobs. + """ + self.assertIn(DummySystemJob.name, registry['system_jobs']) + self.assertIs(registry['system_jobs'][DummySystemJob.name], DummySystemJob) + def test_queues(self): """ Check that plugin queues are registered with the accurate name. From 3e10d9a1da45963401962f45a21e142e888cc00f Mon Sep 17 00:00:00 2001 From: Alexander Haase Date: Mon, 7 Oct 2024 18:42:26 +0200 Subject: [PATCH 06/12] Rename system job metadata To clarify which meta-attributes belong to system jobs, each of them is now prefixed with 'system_'. --- docs/plugins/development/background-jobs.md | 6 +++--- netbox/core/management/commands/rqworker.py | 4 ++-- netbox/netbox/tests/dummy_plugin/jobs.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/plugins/development/background-jobs.md b/docs/plugins/development/background-jobs.md index e950c26e6ce..4e5cd180343 100644 --- a/docs/plugins/development/background-jobs.md +++ b/docs/plugins/development/background-jobs.md @@ -37,11 +37,11 @@ You can schedule the background job from within your code (e.g. from a model's ` This is the human-friendly names of your background job. If omitted, the class name will be used. -#### `enabled` +#### `system_enabled` When the `JobRunner` is defined as [system job](#system-jobs), this attribute controls whether a job will be scheduled. By default, this attribute is `True`. -#### `interval` *(required for system jobs)* +#### `system_interval` *(required for system jobs)* When the `JobRunner` is defined as [system job](#system-jobs), this attribute controls the interval of the scheduled job. @@ -87,7 +87,7 @@ from .models import MyModel class MyHousekeepingJob(JobRunner): class Meta: name = "My Housekeeping Job" - interval = 60 # every 60 minutes + system_interval = 60 # every 60 minutes def run(self, *args, **kwargs): MyModel.objects.filter(foo='bar').delete() diff --git a/netbox/core/management/commands/rqworker.py b/netbox/core/management/commands/rqworker.py index 0e785be4685..7815c7d569f 100644 --- a/netbox/core/management/commands/rqworker.py +++ b/netbox/core/management/commands/rqworker.py @@ -19,10 +19,10 @@ class Command(_Command): def handle(self, *args, **options): # Setup system jobs. for job in registry['system_jobs'].values(): - if getattr(job.Meta, 'enabled', True): + if getattr(job.Meta, 'system_enabled', True): try: logger.debug(f"Scheduling system job {job.name}") - job.enqueue_once(interval=getattr(job.Meta, 'interval')) + job.enqueue_once(interval=getattr(job.Meta, 'system_interval')) except AttributeError as e: raise CommandError(f"Job {job.name} is missing required attribute in Meta: {e.name}") diff --git a/netbox/netbox/tests/dummy_plugin/jobs.py b/netbox/netbox/tests/dummy_plugin/jobs.py index fbb08c1863d..f0934aa9663 100644 --- a/netbox/netbox/tests/dummy_plugin/jobs.py +++ b/netbox/netbox/tests/dummy_plugin/jobs.py @@ -3,7 +3,7 @@ class DummySystemJob(JobRunner): class Meta: - interval = 60 + system_interval = 60 def run(self, *args, **kwargs): pass From 8a8c3287dfb611230a5779c202772ae9ebbd81eb Mon Sep 17 00:00:00 2001 From: Alexander Haase Date: Wed, 9 Oct 2024 22:14:49 +0200 Subject: [PATCH 07/12] Add predefined job interval choices --- docs/plugins/development/background-jobs.md | 9 +++++++-- netbox/core/choices.py | 14 ++++++++++++++ netbox/netbox/tests/dummy_plugin/jobs.py | 3 ++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/plugins/development/background-jobs.md b/docs/plugins/development/background-jobs.md index 4e5cd180343..c1a313704be 100644 --- a/docs/plugins/development/background-jobs.md +++ b/docs/plugins/development/background-jobs.md @@ -29,6 +29,9 @@ class MyTestJob(JobRunner): You can schedule the background job from within your code (e.g. from a model's `save()` method or a view) by calling `MyTestJob.enqueue()`. This method passes through all arguments to `Job.enqueue()`. However, no `name` argument must be passed, as the background job name will be used instead. +!!! tip + A set of predefined intervals can be used from `core.choices.JobIntervalChoices`. + ### Attributes `JobRunner` attributes are defined under a class named `Meta` within the job. These are optional (unless specified otherwise), but encouraged. @@ -56,6 +59,7 @@ As described above, jobs can be scheduled for immediate execution or at any late ```python title="models.py" from django.db import models +from core.choices import JobIntervalChoices from netbox.models import NetBoxModel from .jobs import MyTestJob @@ -63,7 +67,7 @@ class MyModel(NetBoxModel): foo = models.CharField() def save(self, *args, **kwargs): - MyTestJob.enqueue_once(instance=self, interval=60) + MyTestJob.enqueue_once(instance=self, interval=JobIntervalChoices.INTERVAL_HOURLY) return super().save(*args, **kwargs) def sync(self): @@ -81,13 +85,14 @@ Some plugins may implement background jobs that are decoupled from any object an #### Example ```python title="jobs.py" +from core.choices import JobIntervalChoices from netbox.jobs import JobRunner from .models import MyModel class MyHousekeepingJob(JobRunner): class Meta: name = "My Housekeeping Job" - system_interval = 60 # every 60 minutes + system_interval = JobIntervalChoices.INTERVAL_HOURLY # or integer for n minutes def run(self, *args, **kwargs): MyModel.objects.filter(foo='bar').delete() diff --git a/netbox/core/choices.py b/netbox/core/choices.py index 01a072ce1cf..442acc26bac 100644 --- a/netbox/core/choices.py +++ b/netbox/core/choices.py @@ -72,6 +72,20 @@ class JobStatusChoices(ChoiceSet): ) +class JobIntervalChoices(ChoiceSet): + INTERVAL_MINUTELY = 1 + INTERVAL_HOURLY = 60 + INTERVAL_DAILY = 60 * 24 + INTERVAL_WEEKLY = 60 * 24 * 7 + + CHOICES = ( + (INTERVAL_MINUTELY, _('Minutely')), + (INTERVAL_HOURLY, _('Hourly')), + (INTERVAL_DAILY, _('Daily')), + (INTERVAL_WEEKLY, _('Weekly')), + ) + + # # ObjectChanges # diff --git a/netbox/netbox/tests/dummy_plugin/jobs.py b/netbox/netbox/tests/dummy_plugin/jobs.py index f0934aa9663..bf6560fd6ed 100644 --- a/netbox/netbox/tests/dummy_plugin/jobs.py +++ b/netbox/netbox/tests/dummy_plugin/jobs.py @@ -1,9 +1,10 @@ +from core.choices import JobIntervalChoices from netbox.jobs import JobRunner class DummySystemJob(JobRunner): class Meta: - system_interval = 60 + system_interval = JobIntervalChoices.INTERVAL_HOURLY def run(self, *args, **kwargs): pass From d964b5579ed170739136fbc41cd4d42245e2db2c Mon Sep 17 00:00:00 2001 From: Alexander Haase Date: Sun, 27 Oct 2024 11:17:27 +0100 Subject: [PATCH 08/12] Remove 'system_enabled' JobRunner attribute Previously, the 'system_enabled' attribute was used to control whether a job should run or not. However, this can also be accomplished by evaluating the job's interval. --- docs/plugins/development/background-jobs.md | 6 +----- netbox/core/management/commands/rqworker.py | 12 ++++-------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/docs/plugins/development/background-jobs.md b/docs/plugins/development/background-jobs.md index c1a313704be..3921661c0d5 100644 --- a/docs/plugins/development/background-jobs.md +++ b/docs/plugins/development/background-jobs.md @@ -40,13 +40,9 @@ You can schedule the background job from within your code (e.g. from a model's ` This is the human-friendly names of your background job. If omitted, the class name will be used. -#### `system_enabled` - -When the `JobRunner` is defined as [system job](#system-jobs), this attribute controls whether a job will be scheduled. By default, this attribute is `True`. - #### `system_interval` *(required for system jobs)* -When the `JobRunner` is defined as [system job](#system-jobs), this attribute controls the interval of the scheduled job. +When the `JobRunner` is defined as [system job](#system-jobs), this attribute controls the interval of the scheduled job. If the interval evaluates to `False` (i.e. set to `0` or `None`), the job won't be scheduled. ### Scheduled Jobs diff --git a/netbox/core/management/commands/rqworker.py b/netbox/core/management/commands/rqworker.py index 7815c7d569f..969f4490823 100644 --- a/netbox/core/management/commands/rqworker.py +++ b/netbox/core/management/commands/rqworker.py @@ -1,6 +1,5 @@ import logging -from django.core.management.base import CommandError from django_rq.management.commands.rqworker import Command as _Command from netbox.registry import registry @@ -19,13 +18,10 @@ class Command(_Command): def handle(self, *args, **options): # Setup system jobs. for job in registry['system_jobs'].values(): - if getattr(job.Meta, 'system_enabled', True): - try: - logger.debug(f"Scheduling system job {job.name}") - job.enqueue_once(interval=getattr(job.Meta, 'system_interval')) - - except AttributeError as e: - raise CommandError(f"Job {job.name} is missing required attribute in Meta: {e.name}") + interval = getattr(job.Meta, 'system_interval', 0) + if interval: + logger.debug(f"Scheduling system job {job.name}") + job.enqueue_once(interval=interval) # Run the worker with scheduler functionality options['with_scheduler'] = True From 2bd45d334312d2a6dee114ebd9ace282c7b02719 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 30 Oct 2024 13:32:47 -0400 Subject: [PATCH 09/12] Fix test --- netbox/netbox/tests/test_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/tests/test_jobs.py b/netbox/netbox/tests/test_jobs.py index fef5110fb95..e3e24a235ae 100644 --- a/netbox/netbox/tests/test_jobs.py +++ b/netbox/netbox/tests/test_jobs.py @@ -91,7 +91,7 @@ def test_enqueue_once_twice_same(self): self.assertEqual(TestJobRunner.get_jobs(instance).count(), 1) def test_enqueue_once_twice_same_no_schedule_at(self): - instance = Job() + instance = DataSource() schedule_at = self.get_schedule_at() job1 = TestJobRunner.enqueue_once(instance, schedule_at=schedule_at) job2 = TestJobRunner.enqueue_once(instance) From 566201507d0e59ca92598766a89cd7c9b0b16ec4 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 30 Oct 2024 13:40:30 -0400 Subject: [PATCH 10/12] Use a decorator to register system jobs --- netbox/netbox/jobs.py | 13 +++++++++++++ netbox/netbox/plugins/__init__.py | 9 +-------- netbox/netbox/tests/dummy_plugin/__init__.py | 5 +++++ netbox/netbox/tests/dummy_plugin/jobs.py | 3 ++- netbox/netbox/utils.py | 12 ------------ 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/netbox/netbox/jobs.py b/netbox/netbox/jobs.py index a2a0c35acc0..600ba32a653 100644 --- a/netbox/netbox/jobs.py +++ b/netbox/netbox/jobs.py @@ -9,12 +9,25 @@ from core.choices import JobStatusChoices from core.models import Job, ObjectType from netbox.constants import ADVISORY_LOCK_KEYS +from netbox.registry import registry __all__ = ( 'JobRunner', + 'system_job', ) +def system_job(): + """ + Decorator for registering a `JobRunner` class as system background job. + """ + def _wrapper(cls): + registry['system_jobs'][cls.name] = cls + return cls + + return _wrapper + + class JobRunner(ABC): """ Background Job helper class. diff --git a/netbox/netbox/plugins/__init__.py b/netbox/netbox/plugins/__init__.py index 42e2e26bda2..69881a25146 100644 --- a/netbox/netbox/plugins/__init__.py +++ b/netbox/netbox/plugins/__init__.py @@ -8,7 +8,7 @@ from netbox.registry import registry from netbox.search import register_search -from netbox.utils import register_data_backend, register_system_job +from netbox.utils import register_data_backend from .navigation import * from .registration import * from .templates import * @@ -26,7 +26,6 @@ DEFAULT_RESOURCE_PATHS = { 'search_indexes': 'search.indexes', 'data_backends': 'data_backends.backends', - 'system_jobs': 'jobs.system_jobs', 'graphql_schema': 'graphql.schema', 'menu': 'navigation.menu', 'menu_items': 'navigation.menu_items', @@ -74,7 +73,6 @@ class PluginConfig(AppConfig): # Optional plugin resources search_indexes = None data_backends = None - system_jobs = None graphql_schema = None menu = None menu_items = None @@ -114,11 +112,6 @@ def ready(self): for backend in data_backends: register_data_backend()(backend) - # Register system jobs (if defined) - system_jobs = self._load_resource('system_jobs') or [] - for job in system_jobs: - register_system_job()(job) - # Register template content (if defined) if template_extensions := self._load_resource('template_extensions'): register_template_extensions(template_extensions) diff --git a/netbox/netbox/tests/dummy_plugin/__init__.py b/netbox/netbox/tests/dummy_plugin/__init__.py index 6ab62d6387e..2ca7c290c65 100644 --- a/netbox/netbox/tests/dummy_plugin/__init__.py +++ b/netbox/netbox/tests/dummy_plugin/__init__.py @@ -21,5 +21,10 @@ class DummyPluginConfig(PluginConfig): 'netbox.tests.dummy_plugin.events.process_events_queue' ] + def ready(self): + super().ready() + + from . import jobs # noqa: F401 + config = DummyPluginConfig diff --git a/netbox/netbox/tests/dummy_plugin/jobs.py b/netbox/netbox/tests/dummy_plugin/jobs.py index bf6560fd6ed..92dfd549a01 100644 --- a/netbox/netbox/tests/dummy_plugin/jobs.py +++ b/netbox/netbox/tests/dummy_plugin/jobs.py @@ -1,7 +1,8 @@ from core.choices import JobIntervalChoices -from netbox.jobs import JobRunner +from netbox.jobs import JobRunner, system_job +@system_job() class DummySystemJob(JobRunner): class Meta: system_interval = JobIntervalChoices.INTERVAL_HOURLY diff --git a/netbox/netbox/utils.py b/netbox/netbox/utils.py index 60067c55343..f27d1b5f7fc 100644 --- a/netbox/netbox/utils.py +++ b/netbox/netbox/utils.py @@ -3,7 +3,6 @@ __all__ = ( 'get_data_backend_choices', 'register_data_backend', - 'register_system_job', ) @@ -25,14 +24,3 @@ def _wrapper(cls): return cls return _wrapper - - -def register_system_job(): - """ - Decorator for registering a `JobRunner` class as system background job. - """ - def _wrapper(cls): - registry['system_jobs'][cls.name] = cls - return cls - - return _wrapper From 6b192c9ae851fba587b43aed845f0a46b8230dfc Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 30 Oct 2024 14:34:04 -0400 Subject: [PATCH 11/12] Specify interval when registering system job --- netbox/core/management/commands/rqworker.py | 12 +++++++----- netbox/netbox/jobs.py | 10 ++++++++-- netbox/netbox/tests/dummy_plugin/jobs.py | 9 +-------- netbox/netbox/tests/test_plugins.py | 5 +++-- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/netbox/core/management/commands/rqworker.py b/netbox/core/management/commands/rqworker.py index 969f4490823..b2879c3d8e6 100644 --- a/netbox/core/management/commands/rqworker.py +++ b/netbox/core/management/commands/rqworker.py @@ -17,11 +17,13 @@ class Command(_Command): """ def handle(self, *args, **options): # Setup system jobs. - for job in registry['system_jobs'].values(): - interval = getattr(job.Meta, 'system_interval', 0) - if interval: - logger.debug(f"Scheduling system job {job.name}") - job.enqueue_once(interval=interval) + for job, kwargs in registry['system_jobs'].items(): + try: + interval = kwargs['interval'] + except KeyError: + raise TypeError("System job must specify an interval (in minutes).") + logger.debug(f"Scheduling system job {job.name} (interval={interval})") + job.enqueue_once(**kwargs) # Run the worker with scheduler functionality options['with_scheduler'] = True diff --git a/netbox/netbox/jobs.py b/netbox/netbox/jobs.py index 600ba32a653..965ebc9e980 100644 --- a/netbox/netbox/jobs.py +++ b/netbox/netbox/jobs.py @@ -2,6 +2,7 @@ from abc import ABC, abstractmethod from datetime import timedelta +from django.core.exceptions import ImproperlyConfigured from django.utils.functional import classproperty from django_pglocks import advisory_lock from rq.timeouts import JobTimeoutException @@ -17,12 +18,17 @@ ) -def system_job(): +def system_job(interval): """ Decorator for registering a `JobRunner` class as system background job. """ + if type(interval) is not int: + raise ImproperlyConfigured("System job interval must be an integer (minutes).") + def _wrapper(cls): - registry['system_jobs'][cls.name] = cls + registry['system_jobs'][cls] = { + 'interval': interval + } return cls return _wrapper diff --git a/netbox/netbox/tests/dummy_plugin/jobs.py b/netbox/netbox/tests/dummy_plugin/jobs.py index 92dfd549a01..3b9dc7a5fa6 100644 --- a/netbox/netbox/tests/dummy_plugin/jobs.py +++ b/netbox/netbox/tests/dummy_plugin/jobs.py @@ -2,15 +2,8 @@ from netbox.jobs import JobRunner, system_job -@system_job() +@system_job(interval=JobIntervalChoices.INTERVAL_HOURLY) class DummySystemJob(JobRunner): - class Meta: - system_interval = JobIntervalChoices.INTERVAL_HOURLY def run(self, *args, **kwargs): pass - - -system_jobs = ( - DummySystemJob, -) diff --git a/netbox/netbox/tests/test_plugins.py b/netbox/netbox/tests/test_plugins.py index 753e5c08a71..db82d0a7598 100644 --- a/netbox/netbox/tests/test_plugins.py +++ b/netbox/netbox/tests/test_plugins.py @@ -5,6 +5,7 @@ from django.test import Client, TestCase, override_settings from django.urls import reverse +from core.choices import JobIntervalChoices from netbox.tests.dummy_plugin import config as dummy_config from netbox.tests.dummy_plugin.data_backends import DummyBackend from netbox.tests.dummy_plugin.jobs import DummySystemJob @@ -135,8 +136,8 @@ def test_system_jobs(self): """ Check registered system jobs. """ - self.assertIn(DummySystemJob.name, registry['system_jobs']) - self.assertIs(registry['system_jobs'][DummySystemJob.name], DummySystemJob) + self.assertIn(DummySystemJob, registry['system_jobs']) + self.assertEqual(registry['system_jobs'][DummySystemJob]['interval'], JobIntervalChoices.INTERVAL_HOURLY) def test_queues(self): """ From 89a976f5b8b1e5f37afb0d794bad91e68a2c0ad4 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 30 Oct 2024 14:56:58 -0400 Subject: [PATCH 12/12] Update documentation --- docs/plugins/development/background-jobs.md | 22 ++++++++++----------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/docs/plugins/development/background-jobs.md b/docs/plugins/development/background-jobs.md index 3921661c0d5..d51981b9e0d 100644 --- a/docs/plugins/development/background-jobs.md +++ b/docs/plugins/development/background-jobs.md @@ -30,20 +30,16 @@ class MyTestJob(JobRunner): You can schedule the background job from within your code (e.g. from a model's `save()` method or a view) by calling `MyTestJob.enqueue()`. This method passes through all arguments to `Job.enqueue()`. However, no `name` argument must be passed, as the background job name will be used instead. !!! tip - A set of predefined intervals can be used from `core.choices.JobIntervalChoices`. + A set of predefined intervals is available at `core.choices.JobIntervalChoices` for convenience. ### Attributes -`JobRunner` attributes are defined under a class named `Meta` within the job. These are optional (unless specified otherwise), but encouraged. +`JobRunner` attributes are defined under a class named `Meta` within the job. These are optional, but encouraged. #### `name` This is the human-friendly names of your background job. If omitted, the class name will be used. -#### `system_interval` *(required for system jobs)* - -When the `JobRunner` is defined as [system job](#system-jobs), this attribute controls the interval of the scheduled job. If the interval evaluates to `False` (i.e. set to `0` or `None`), the job won't be scheduled. - ### Scheduled Jobs As described above, jobs can be scheduled for immediate execution or at any later time using the `enqueue()` method. However, for management purposes, the `enqueue_once()` method allows a job to be scheduled exactly once avoiding duplicates. If a job is already scheduled for a particular instance, a second one won't be scheduled, respecting thread safety. An example use case would be to schedule a periodic task that is bound to an instance in general, but not to any event of that instance (such as updates). The parameters of the `enqueue_once()` method are identical to those of `enqueue()`. @@ -73,22 +69,21 @@ class MyModel(NetBoxModel): ### System Jobs -Some plugins may implement background jobs that are decoupled from any object and the request-response cycle. Typical use cases would be housekeeping tasks or synchronization jobs. These can be created using *system jobs*. The `JobRunner` class has everything included to provide this type of job as well. Just add the appropriate metadata to let NetBox schedule all background jobs automatically. - -!!! info - All system jobs are automatically scheduled just before the `./manage.py rqworker` command is started and the job queue is processed. The schedules are also checked at each restart of this process. +Some plugins may implement background jobs that are decoupled from the request/response cycle. Typical use cases would be housekeeping tasks or synchronization jobs. These can be registered as _system jobs_ using the `system_job()` decorator. The job interval must be passed as an integer (in minutes) when registering a system job. System jobs are scheduled automatically when the RQ worker (`manage.py rqworker`) is run. #### Example ```python title="jobs.py" from core.choices import JobIntervalChoices -from netbox.jobs import JobRunner +from netbox.jobs import JobRunner, system_job from .models import MyModel +# Specify a predefined choice or an integer indicating +# the number of minutes between job executions +@system_job(interval=JobIntervalChoices.INTERVAL_HOURLY) class MyHousekeepingJob(JobRunner): class Meta: name = "My Housekeeping Job" - system_interval = JobIntervalChoices.INTERVAL_HOURLY # or integer for n minutes def run(self, *args, **kwargs): MyModel.objects.filter(foo='bar').delete() @@ -98,6 +93,9 @@ system_jobs = ( ) ``` +!!! note + Ensure that any system jobs are imported on initialization. Otherwise, they won't be registered. This can be achieved by extending the PluginConfig's `ready()` method. + ## Task queues Three task queues of differing priority are defined by default: