Add tags to jobs and a command line arg to select them

This allows the user to selectively run jobs based on a tag for use
cases where you might want to run urlwatch on different schedules for
different jobs and other use cases.

Signed-off-by: James Hewitt <[email protected]>

CHANGELOG.md

@@ -6,6 +6,10 @@ The format mostly follows [Keep a Changelog](http://keepachangelog.com/en/1.0.0/
 
 ## UNRELEASED
 
+### Added
+
+- Added tags to jobs and the ability to select them at the command line (#789 by jamstah)
+
 ### Changed
 
 - Remove EOL'd Python 3.7 (new minimum requirement is Python 3.8), add Python 3.12 testing

docs/source/jobs.rst

@@ -168,6 +168,7 @@ Optional keys for all job types
 -------------------------------
 
 - ``name``: Human-readable name/label of the job
+- ``tags``: Array of tags
 - ``filter``: :doc:`filters` (if any) to apply to the output (can be tested with ``--test-filter``)
 - ``max_tries``: Number of times to retry fetching the resource
 - ``diff_tool``: Command to a custom tool for generating diff text

docs/source/manpage.rst

@@ -30,6 +30,10 @@ optional arguments:
    -h, --help
           show this help message and exit
 
+   --tags
+          comma separated tags of job(s) to run.
+          If none specified, then all jobs will be run.
+
    --version
           show program's version number and exit
 

lib/urlwatch/config.py

@@ -68,6 +68,7 @@ def parse_args(self, cmdline_args):
         parser = argparse.ArgumentParser(description=urlwatch.__doc__,
                                          formatter_class=argparse.RawDescriptionHelpFormatter)
         parser.add_argument('joblist', metavar='JOB', type=int, nargs="*", help='index of job(s) to run, as numbered according to the --list command. If none specified, then all jobs will be run.')
+        parser.add_argument('--tags', metavar='TAG,TAG...', type=lambda x: x.split(","), help='comma separated tags of job(s) to run. If none specified, then all jobs will be run.')
         parser.add_argument('--version', action='version', version='%(prog)s {}'.format(urlwatch.__version__))
         parser.add_argument('-v', '--verbose', action='store_true', help='show debug output')
         group = parser.add_argument_group('files and directories')

lib/urlwatch/jobs.py

@@ -196,7 +196,10 @@ def ignore_error(self, exception):
 
 class Job(JobBase):
     __required__ = ()
-    __optional__ = ('name', 'filter', 'max_tries', 'diff_tool', 'compared_versions', 'diff_filter', 'treat_new_as_changed', 'user_visible_url')
+    __optional__ = ('name', 'tags', 'filter', 'max_tries', 'diff_tool', 'compared_versions', 'diff_filter', 'treat_new_as_changed', 'user_visible_url')
+
+    def matches_tags(self, tags):
+        return set(self.tags) & set(tags)
 
     # determine if hyperlink "a" tag is used in HtmlReporter
     def location_is_url(self):

lib/urlwatch/tests/data/jobs-with-tags.yaml

@@ -0,0 +1,17 @@
+---
+name: UTC
+command: date -u
+tags:
+  - arg
+  - utc
+---
+name: RFC
+command: date -R
+tags:
+  - arg
+  - rfc
+---
+name: Local
+command: date
+tags:
+  - local

lib/urlwatch/tests/test_handler.py

@@ -122,6 +122,57 @@ def test_run_watcher():
             cache_storage.close()
 
 
+def prepare_tags_test():
+    urls = os.path.join(here, 'data', 'jobs-with-tags.yaml')
+    config = os.path.join(here, 'data', 'urlwatch.yaml')
+    cache = os.path.join(here, 'data', 'cache.db')
+    hooks = ''
+
+    config_storage = YamlConfigStorage(config)
+    urls_storage = UrlsYaml(urls)
+    cache_storage = CacheMiniDBStorage(cache)
+
+    urlwatch_config = ConfigForTest(config, urls, cache, hooks, True)
+    urlwatcher = Urlwatch(urlwatch_config, config_storage, cache_storage, urls_storage)
+
+    return urlwatcher, cache_storage
+
+
+def test_tags_none():
+    with teardown_func():
+        urlwatcher, cache_storage = prepare_tags_test()
+        try:
+            urlwatcher.run_jobs()
+
+            assert len(urlwatcher.report.job_states) == 3
+        finally:
+            cache_storage.close()
+
+
+def test_tags_single():
+    with teardown_func():
+        urlwatcher, cache_storage = prepare_tags_test()
+        try:
+            urlwatcher.urlwatch_config.tags = ["arg"]
+            urlwatcher.run_jobs()
+
+            assert len(urlwatcher.report.job_states) == 2
+        finally:
+            cache_storage.close()
+
+
+def test_tags_multiple():
+    with teardown_func():
+        urlwatcher, cache_storage = prepare_tags_test()
+        try:
+            urlwatcher.urlwatch_config.tags = ["utc", "local"]
+            urlwatcher.run_jobs()
+
+            assert len(urlwatcher.report.job_states) == 2
+        finally:
+            cache_storage.close()
+
+
 def test_unserialize_shell_job_without_kind():
     job = JobBase.unserialize({
         'name': 'hoho',

lib/urlwatch/worker.py

@@ -55,7 +55,11 @@ def run_jobs(urlwatcher):
         raise ValueError(f'All job indices must be between 1 and {len(urlwatcher.jobs)}: {urlwatcher.urlwatch_config.joblist}')
     cache_storage = urlwatcher.cache_storage
     jobs = [job.with_defaults(urlwatcher.config_storage.config)
-            for (idx, job) in enumerate(urlwatcher.jobs) if ((idx + 1) in urlwatcher.urlwatch_config.joblist or (not urlwatcher.urlwatch_config.joblist))]
+            for (idx, job) in enumerate(urlwatcher.jobs)
+            if (not urlwatcher.urlwatch_config.joblist and not urlwatcher.urlwatch_config.tags)
+            or (idx + 1) in urlwatcher.urlwatch_config.joblist
+            or job.matches_tags(urlwatcher.urlwatch_config.tags)
+            ]
     report = urlwatcher.report
 
     logger.debug('Processing %d jobs (out of %d)', len(jobs), len(urlwatcher.jobs))