Add 'expand' subcommand to list jobs and templates in a test plan (Ne…

…w) (#1065)

* Add 'expand' subcommand to list jobs and templates in a test plan

The new subcommand "expands" a given test plan by going through all its
sections and returning the pure (non-instantiated) jobs and templates
being used.

It is different from the "list-boostrapped" subcommand since it does not
perform any bootstrap.

By default, it outputs the jobs/templates id as text, but it can be
called with the `--format json` option to output JSON data with all the
known information for said jobs and templates.

This is to be used by external tools, for instance to create a document
listing information (summary, description...) about executed jobs and
templates.

Fix CHECKBOX-1263

* Rename plainbox.impl.secure.qualifiers.select_jobs() to select_units()

The select_jobs() function can actually be called with not only a list
of jobs, but also templates.

* Rename IJobQualifier to IUnitQualifier

Qualifiers can be used on jobs as well as templates.

* Add unit tests

* Sort the unit list before returning it

Sort the unit list by id/template-id before printing it out. In
addition, sort the keys inside each JSON record for easier comparison.

checkbox-ng/checkbox_ng/launcher/checkbox_cli.py

@@ -37,6 +37,7 @@
     StartProvider,
     Submit,
     ListBootstrapped,
+    Expand,
     TestPlanExport,
     Show,
 )
@@ -72,6 +73,7 @@ def main():
         "submit": Submit,
         "show": Show,
         "list-bootstrapped": ListBootstrapped,
+        "expand": Expand,
         "merge-reports": MergeReports,
         "merge-submissions": MergeSubmissions,
         "tp-export": TestPlanExport,

checkbox-ng/checkbox_ng/launcher/subcommands.py

@@ -19,6 +19,7 @@
 Definition of sub-command classes for checkbox-cli
 """
 from argparse import ArgumentTypeError
+from argparse import RawDescriptionHelpFormatter
 from argparse import SUPPRESS
 from collections import defaultdict
 from string import Formatter
@@ -45,12 +46,14 @@
 from plainbox.impl.result import MemoryJobResult
 from plainbox.impl.runner import slugify
 from plainbox.impl.secure.sudo_broker import sudo_password_provider
+from plainbox.impl.secure.qualifiers import select_units
 from plainbox.impl.session.assistant import SA_RESTARTABLE
 from plainbox.impl.session.restart import detect_restart_strategy
 from plainbox.impl.session.storage import WellKnownDirsHelper
 from plainbox.impl.transport import TransportError
 from plainbox.impl.transport import get_all_transports
 from plainbox.impl.transport import SECURE_ID_PATTERN
+from plainbox.impl.unit.testplan import TestPlanUnitSupport
 
 from checkbox_ng.config import load_configs
 from checkbox_ng.launcher.stages import MainLoopStage, ReportsStage
@@ -1267,6 +1270,92 @@ def __missing__(self, key):
         print_objs(ctx.args.GROUP, ctx.sa, ctx.args.attrs)
 
 
+class Expand:
+    def __init__(self):
+        self.override_list = []
+
+    @property
+    def sa(self):
+        return self.ctx.sa
+
+    def register_arguments(self, parser):
+        parser.formatter_class = RawDescriptionHelpFormatter
+        parser.description = (
+            "Expand a given test plan: display all the jobs and templates "
+            "that are defined in this test plan and that would be executed "
+            "if ran. This is useful to visualize the full list of jobs and "
+            "templates for complex test plans that consist of many nested "
+            "parts with different 'include' and 'exclude' sections.\n\n"
+            "NOTE: the elements listed here are not sorted by execution "
+            "order. To see the execution order, please use the "
+            "'list-bootstrapped' command instead."
+        )
+        parser.add_argument("TEST_PLAN", help=_("test-plan id to expand"))
+        parser.add_argument(
+            "-f",
+            "--format",
+            type=str,
+            default="text",
+            help=_("output format: 'text' or 'json' (default: %(default)s)"),
+        )
+
+    def invoked(self, ctx):
+        self.ctx = ctx
+        session_title = "checkbox-expand-{}".format(ctx.args.TEST_PLAN)
+        self.sa.start_new_session(session_title)
+        tps = self.sa.get_test_plans()
+        if ctx.args.TEST_PLAN not in tps:
+            raise SystemExit("Test plan not found")
+        self.sa.select_test_plan(ctx.args.TEST_PLAN)
+        all_jobs_and_templates = [
+            unit
+            for unit in self.sa._context.state.unit_list
+            if unit.unit in ["job", "template"]
+        ]
+        tp = self.sa._context._test_plan_list[0]
+        tp_us = TestPlanUnitSupport(tp)
+        self.override_list = tp_us.override_list
+        jobs_and_templates_list = select_units(
+            all_jobs_and_templates,
+            [tp.get_mandatory_qualifier()] + [tp.get_qualifier()],
+        )
+
+        obj_list = []
+        for unit in jobs_and_templates_list:
+            obj = unit._raw_data.copy()
+            obj["unit"] = unit.unit
+            obj["id"] = unit.id  # To get the fully qualified id
+            obj[
+                "certification-status"
+            ] = self.get_effective_certification_status(unit)
+            if unit.template_id:
+                obj["template-id"] = unit.template_id
+            obj_list.append(obj)
+        obj_list.sort(key=lambda x: x.get("template-id", x["id"]))
+        if ctx.args.format == "json":
+            print(json.dumps(obj_list, sort_keys=True))
+        else:
+            for obj in obj_list:
+                if obj["unit"] == "template":
+                    print("Template '{}'".format(obj["template-id"]))
+                else:
+                    print("Job '{}'".format(obj["id"]))
+
+    def get_effective_certification_status(self, unit):
+        if unit.unit == "template":
+            unit_id = unit.template_id
+        else:
+            unit_id = unit.id
+        for regex, override_field_list in self.override_list:
+            if re.match(regex, unit_id):
+                for field, value in override_field_list:
+                    if field == "certification_status":
+                        return value
+        if hasattr(unit, "certification_status"):
+            return unit.certification_status
+        return "unspecified"
+
+
 class ListBootstrapped:
     @property
     def sa(self):

checkbox-ng/checkbox_ng/launcher/test_subcommands.py

@@ -24,7 +24,12 @@
 from unittest.mock import patch, Mock, MagicMock
 
 from io import StringIO
+
+from plainbox.impl.unit.job import JobDefinition
+from plainbox.impl.unit.template import TemplateUnit
+
 from checkbox_ng.launcher.subcommands import (
+    Expand,
     Launcher,
     ListBootstrapped,
     IncompatibleJobError,
@@ -682,6 +687,91 @@ def test_invoke_print_output_customized_format(self, stdout):
         self.assertEqual(stdout.getvalue(), expected_out)
 
 
+class TestExpand(TestCase):
+    def setUp(self):
+        self.launcher = Expand()
+        self.ctx = Mock()
+        self.ctx.args = Mock(TEST_PLAN="", format="")
+        self.ctx.sa = Mock(
+            start_new_session=Mock(),
+            get_test_plans=Mock(return_value=["test-plan1", "test-plan2"]),
+            select_test_plan=Mock(),
+            # get_resumable_sessions=Mock(return_value=[]),
+            _context=Mock(
+                state=Mock(
+                    unit_list=[]
+                ),
+                _test_plan_list=[Mock()],
+            ),
+        )
+
+    def test_register_arguments(self):
+        parser_mock = Mock()
+        self.launcher.register_arguments(parser_mock)
+        self.assertTrue(parser_mock.add_argument.called)
+
+    def test_invoke__test_plan_not_found(self):
+        self.ctx.args.TEST_PLAN = "test-plan3"
+
+        with self.assertRaisesRegex(SystemExit, "Test plan not found"):
+            self.launcher.invoked(self.ctx)
+
+    @patch("sys.stdout", new_callable=StringIO)
+    @patch("checkbox_ng.launcher.subcommands.TestPlanUnitSupport")
+    @patch("checkbox_ng.launcher.subcommands.select_units")
+    def test_invoke__text(self, mock_select_units, mock_tpus, stdout):
+        template1 = TemplateUnit({
+            "template-id": "test-template",
+            "id": "test-{res}",
+            "template-summary": "Test Template Summary",
+        })
+        job1 = JobDefinition({
+            "id": "job1",
+        })
+        mock_select_units.return_value = [job1, template1]
+        self.ctx.args.TEST_PLAN = "test-plan1"
+        self.launcher.invoked(self.ctx)
+        self.assertIn("Template 'test-template'", stdout.getvalue())
+
+    @patch("sys.stdout", new_callable=StringIO)
+    @patch("checkbox_ng.launcher.subcommands.TestPlanUnitSupport")
+    @patch("checkbox_ng.launcher.subcommands.select_units")
+    def test_invoke__json(self, mock_select_units, mock_tpus, stdout):
+        template1 = TemplateUnit({
+            "template-id": "test-template",
+            "id": "test-{res}",
+            "template-summary": "Test Template Summary",
+        })
+        job1 = JobDefinition({
+            "id": "job1",
+        })
+        mock_select_units.return_value = [job1, template1]
+        self.ctx.args.TEST_PLAN = "test-plan1"
+        self.ctx.args.format = "json"
+        self.launcher.invoked(self.ctx)
+        self.assertIn('"template-id": "test-template"', stdout.getvalue())
+
+    def test_get_effective_certificate_status(self):
+        job1 = JobDefinition({
+            "id": "job1",
+        })
+        template1 = TemplateUnit({
+            "template-id": "template1",
+            "id": "job-{res}",
+        })
+        self.launcher.override_list = [
+            ("^job1$", [("certification_status", "blocker"),]),
+        ]
+        self.assertEqual(
+            self.launcher.get_effective_certification_status(job1),
+            "blocker"
+        )
+        self.assertEqual(
+            self.launcher.get_effective_certification_status(template1),
+            "unspecified"
+        )
+
+
 class TestUtilsFunctions(TestCase):
     @patch("checkbox_ng.launcher.subcommands.Colorizer", new=MagicMock())
     @patch("builtins.print")

checkbox-ng/plainbox/abc.py

@@ -287,12 +287,12 @@ def is_hollow(self):
         """
 
 
-class IJobQualifier(metaclass=ABCMeta):
+class IUnitQualifier(metaclass=ABCMeta):
     """
-    An opaque qualifier for a job definition.
+    An opaque qualifier for a unit (job or template).
 
-    This is an abstraction for matching jobs definitions to names, patterns and
-    other means of selecting jobs.
+    This is an abstraction for matching jobs and templates to names, patterns
+    and other means of selecting jobs and templates.
 
     There are two ways to use a qualifier object. The naive, direct, old API
     can simply check if a qualifier designates a particular job (if it selects
@@ -305,15 +305,15 @@ class IJobQualifier(metaclass=ABCMeta):
     expressiveness can be preserved.
 
     :attr VOTE_EXCLUDE:
-        (0) vote indicating that a job should *not* be included for
+        (0) vote indicating that a unit should *not* be included for
         selection. It overwrites any other votes.
 
     :attr VOTE_INCLUDE:
-        (1) vote indicating that a job should be included for selection. It is
+        (1) vote indicating that a unit should be included for selection. It is
         overridden by VOTE_EXCLUDE.
 
     :attr VOTE_IGNORE:
-        (2) vote indicating that a job should neither be included nor excluded
+        (2) vote indicating that a unit should neither be included nor excluded
         for selection. This is a neutral value overridden by all other votes.
     """
 
@@ -326,13 +326,13 @@ class IJobQualifier(metaclass=ABCMeta):
     VOTE_IGNORE = 2
 
     @abstractmethod
-    def get_vote(self, job):
+    def get_vote(self, unit):
         """
         Get one of the :attr:`VOTE_IGNORE`, :attr:`VOTE_INCLUDE`,
         :attr:`VOTE_EXCLUDE` votes that this qualifier associated with the
-        specified job.
+        specified unit.
 
-        :param job:
+        :param unit:
             A IJobDefinition instance that is to be visited
         :returns:
             one of the ``VOTE_xxx`` constants
@@ -346,7 +346,7 @@ def get_primitive_qualifiers(self):
         Return a list of primitives that constitute this qualifier.
 
         :returns:
-            A list of IJobQualifier objects that each is the smallest,
+            A list of IUnitQualifier objects that each is the smallest,
             indivisible entity.
 
         When each vote cast by those qualifiers is applied sequentially to

checkbox-ng/plainbox/impl/applogic.py

@@ -32,20 +32,20 @@
 from plainbox.i18n import gettext as _
 from plainbox.impl.result import MemoryJobResult
 from plainbox.impl.secure import config
-from plainbox.impl.secure.qualifiers import select_jobs
+from plainbox.impl.secure.qualifiers import select_units
 from plainbox.impl.session import SessionManager
 from plainbox.impl.session.jobs import InhibitionCause
 
 
-# Deprecated, use plainbox.impl.secure.qualifiers.select_jobs() instead
+# Deprecated, use plainbox.impl.secure.qualifiers.select_units() instead
 def get_matching_job_list(job_list, qualifier):
     """
     Get a list of jobs that are designated by the specified qualifier.
 
     This is intended to be used with :class:`CompositeQualifier`
-    but works with any :class:`IJobQualifier` subclass.
+    but works with any :class:`IUnitQualifier` subclass.
     """
-    return select_jobs(job_list, [qualifier])
+    return select_units(job_list, [qualifier])
 
 
 def run_job_if_possible(session, runner, config, job, update=True, ui=None):