Introduce "exec-runnables-recipe" resolver

This resolver is somewhat of a hybrid between the "exec-test" and the
"runnables-recipe" resolvers.

It runs an executable, and attempts to read from its STDOUT content
that will be treated as runnables-recipe JSON content.  If that
succeeds, the content will be returned as test resolutions.  This is
useful for executable tests or test generators that will output the
tests dinamically.

Signed-off-by: Cleber Rosa <[email protected]>

avocado/plugins/list.py

@@ -214,6 +214,13 @@ def configure(self, parser):
             allow_multiple=True,
         )
 
+        settings.add_argparser_to_option(
+            namespace="resolver.run_executables",
+            parser=parser,
+            long_arg="--resolver-run-executables",
+            allow_multiple=True,
+        )
+
         help_msg = "Writes runnable recipe files to a directory."
         settings.register_option(
             section="list.recipes",

avocado/plugins/resolvers.py

@@ -19,10 +19,11 @@
 import json
 import os
 import re
+import subprocess
 
 from avocado.core.extension_manager import PluginPriority
 from avocado.core.nrunner.runnable import Runnable
-from avocado.core.plugin_interfaces import Resolver
+from avocado.core.plugin_interfaces import Init, Resolver
 from avocado.core.references import reference_split
 from avocado.core.resolver import (
     ReferenceResolution,
@@ -31,6 +32,7 @@
     get_file_assets,
 )
 from avocado.core.safeloader import find_avocado_tests, find_python_unittests
+from avocado.core.settings import settings
 
 
 class BaseExec:
@@ -195,3 +197,84 @@ def resolve(self, reference):
             return criteria_check
 
         return self._validate_and_load_runnables(reference)
+
+
+class ExecRunnablesRecipeInit(Init):
+    name = "exec-runnables-recipe"
+    description = 'Configuration for resolver plugin "exec-runnables-recipe" plugin'
+
+    def initialize(self):
+        help_msg = (
+            'Whether resolvers (such as "exec-runnables-recipe") should '
+            "execute files given as test references that have executable "
+            "permissions. This is disabled by default due to security "
+            "implications of running executables that may not be trusted."
+        )
+        settings.register_option(
+            section="resolver",
+            key="run_executables",
+            key_type=bool,
+            default=False,
+            help_msg=help_msg,
+        )
+
+
+class ExecRunnablesRecipeResolver(BaseExec, Resolver):
+    name = "exec-runnables-recipe"
+    description = "Test resolver for executables that output JSON runnable recipes"
+    priority = PluginPriority.LOW
+
+    def resolve(self, reference):
+        if not self.config.get("resolver.run_executables"):
+            return ReferenceResolution(
+                reference,
+                ReferenceResolutionResult.NOTFOUND,
+                info=(
+                    "Running executables is not enabled. Refer to "
+                    '"resolver.run_executables" configuration option'
+                ),
+            )
+
+        exec_criteria = self.check_exec(reference)
+        if exec_criteria is not None:
+            return exec_criteria
+
+        try:
+            process = subprocess.Popen(
+                reference,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+        except (FileNotFoundError, PermissionError) as exc:
+            return ReferenceResolution(
+                reference,
+                ReferenceResolutionResult.NOTFOUND,
+                info=(f'Failure while running running executable "{reference}": {exc}'),
+            )
+
+        content, _ = process.communicate()
+        try:
+            runnables = json.loads(content)
+        except json.JSONDecodeError:
+            return ReferenceResolution(
+                reference,
+                ReferenceResolutionResult.NOTFOUND,
+                info=f'Content generated by running executable "{reference}" is not JSON',
+            )
+
+        if not (
+            isinstance(runnables, list)
+            and all([isinstance(r, dict) for r in runnables])
+        ):
+            return ReferenceResolution(
+                reference,
+                ReferenceResolutionResult.NOTFOUND,
+                info=f"Content generated by running executable {reference} does not look like a runnables recipe JSON content",
+            )
+
+        return ReferenceResolution(
+            reference,
+            ReferenceResolutionResult.SUCCESS,
+            [Runnable.from_dict(r) for r in runnables],
+        )

avocado/plugins/run.py

@@ -267,6 +267,13 @@ def configure(self, parser):
             long_arg="--log-test-data-directories",
         )
 
+        settings.add_argparser_to_option(
+            namespace="resolver.run_executables",
+            parser=parser,
+            long_arg="--resolver-run-executables",
+            allow_multiple=True,
+        )
+
         parser_common_args.add_tag_filter_args(parser)
 
     def run(self, config):

docs/source/guides/writer/chapters/recipes.rst

@@ -76,3 +76,78 @@ That will be parsed by the ``runnables-recipe`` resolver, like in
 
   exec-test /bin/true
   exec-test /bin/false
+
+Using dynamically generated recipes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``exec-runnables-recipe`` resolver allows a user to point to a
+file that will be executed, and that is expected to generate (on its
+``STDOUT``) content compatible with the Runnable recipe format
+mentioned previously.
+
+For security reasons, Avocado won't execute files indiscriminately
+when looking for tests (at the resolution phase). One must set the
+``--resolver-run-executables`` command line option (or the underlying
+``resolver.run_executables`` configuration option) to allow running
+executables at the resolver stage.
+
+.. warning:: It's the user's responsibility to give test references
+             (to be resolved and thus executed) that are well behaved
+             in the sense that they will finish executing quickly,
+             won't execute unintended code (such as running tests),
+             won't destroy data, etc.
+
+A script such as:
+
+.. literalinclude:: ../../../../../examples/nrunner/resolvers/exec_runnables_recipe.sh
+
+Will output JSON that is compatible with the runnable recipe format.
+That can be used directly via either ``avocado list`` or ``avocado
+run``.  Example::
+
+  $ avocado list --resolver-run-executables examples/nrunner/resolvers/exec_runnables_recipe.sh
+
+  exec-test true-test
+  exec-test false-test
+
+Behavior of ``exec-runnables-recipe`` and ``exec-test`` resolvers
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+The ``exec-runnables-recipe`` resolver has a higher priority than
+(that is, it runs before) the ``exec-test`` resolver.  That means that
+if, and only if, a user enables the feature itself (by means of the
+``--resolver-run-executables`` command line option or the underlying
+``resolver.run_executables`` configuration option), it
+``exec-runnables-recipe`` will perform any meaningful action.
+
+Even if the ``exec-runnables-recipe`` is activated (through the
+command line or configuration option mentioned before), it may still
+coexist with ``exec-test`` resolver, example::
+
+  $ avocado list --resolver-run-executables examples/nrunner/resolvers/exec_runnables_recipe.sh /bin/uname
+
+  exec-test true-test
+  exec-test false-test
+  exec-test /bin/uname
+
+The reason (that can be seen with ``avocado -V list ...``) for that is
+the ``exec-runnables-recipe`` returns a "not found" resolution with
+the message::
+
+  Resolver              Reference  Info
+  ...
+  exec-runnables-recipe /bin/uname Content generated by running executable "/bin/uname" is not JSON
+
+.. warning:: Even though it's possible to have ``exec-test`` and
+             ``exec-runnable-recipes`` in the same Avocado test suite
+             (for instance in an ``avocado run`` command execution)
+             it's not recommended on most cases because ``exec-tests``
+             will end up being run at the test resolution phase
+             in addition to the test execution phase.  It's
+             recommended to use multiple ``avocado run``
+             commands or use the Job API and mulitple
+             :class:`avocado.core.suite.TestSuite`, one for
+             ``exec-runnable-recipes`` with the
+             ``resolver.run_executables`` options enabled, and
+             another for ``exec-tests`` with that option in its
+             default state (disabled).

examples/nrunner/resolvers/exec_runnables_recipe.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo '[{"kind": "exec-test","uri": "/bin/true","identifier": "true-test"},{"kind": "exec-test","uri": "/bin/false","identifier": "false-test"}]'

selftests/check.py

@@ -29,7 +29,7 @@
     "nrunner-requirement": 28,
     "unit": 678,
     "jobs": 11,
-    "functional-parallel": 309,
+    "functional-parallel": 311,
     "functional-serial": 7,
     "optional-plugins": 0,
     "optional-plugins-golang": 2,

selftests/functional/resolver.py

@@ -10,7 +10,13 @@
 # is also the same
 from selftests.functional.list import AVOCADO_TEST_OK as AVOCADO_INSTRUMENTED_TEST
 from selftests.functional.list import EXEC_TEST
-from selftests.utils import AVOCADO, BASEDIR, TestCaseTmpDir, python_module_available
+from selftests.utils import (
+    AVOCADO,
+    BASEDIR,
+    TestCaseTmpDir,
+    python_module_available,
+    skipUnlessPathExists,
+)
 
 
 class ResolverFunctional(unittest.TestCase):
@@ -157,6 +163,44 @@ def test_runnable_recipe_origin(self):
             result.stdout,
         )
 
+    @skipUnlessPathExists("/bin/sh")
+    def test_exec_runnable_recipe_disabled(self):
+        resolver_path = os.path.join(
+            BASEDIR,
+            "examples",
+            "nrunner",
+            "resolvers",
+            "exec_runnables_recipe.sh",
+        )
+        cmd_line = f"{AVOCADO} -V list {resolver_path}"
+        result = process.run(cmd_line)
+        self.assertIn(
+            b"examples/nrunner/resolvers/exec_runnables_recipe.sh exec-test",
+            result.stdout,
+        )
+        self.assertIn(b"exec-test: 1\n", result.stdout)
+
+    @skipUnlessPathExists("/bin/sh")
+    def test_exec_runnable_recipe_enabled(self):
+        resolver_path = os.path.join(
+            BASEDIR,
+            "examples",
+            "nrunner",
+            "resolvers",
+            "exec_runnables_recipe.sh",
+        )
+        cmd_line = f"{AVOCADO} -V list --resolver-run-executables {resolver_path}"
+        result = process.run(cmd_line)
+        self.assertIn(
+            b"exec-test true-test  /bin/true  exec-runnables-recipe",
+            result.stdout,
+        )
+        self.assertIn(
+            b"exec-test false-test /bin/false exec-runnables-recipe",
+            result.stdout,
+        )
+        self.assertIn(b"exec-test: 2\n", result.stdout)
+
 
 class ResolverFunctionalTmp(TestCaseTmpDir):
     def test_runnables_recipe(self):

setup.py

@@ -394,6 +394,7 @@ def run(self):
                 "nrunner = avocado.plugins.runner_nrunner:RunnerInit",
                 "testlogsui = avocado.plugins.testlogs:TestLogsUIInit",
                 "human = avocado.plugins.human:HumanInit",
+                "exec-runnables-recipe = avocado.plugins.resolvers:ExecRunnablesRecipeInit",
             ],
             "avocado.plugins.cli": [
                 "xunit = avocado.plugins.xunit:XUnitCLI",
@@ -461,6 +462,7 @@ def run(self):
                 "tap = avocado.plugins.resolvers:TapResolver",
                 "runnable-recipe = avocado.plugins.resolvers:RunnableRecipeResolver",
                 "runnables-recipe = avocado.plugins.resolvers:RunnablesRecipeResolver",
+                "exec-runnables-recipe = avocado.plugins.resolvers:ExecRunnablesRecipeResolver",
             ],
             "avocado.plugins.suite.runner": [
                 "nrunner = avocado.plugins.runner_nrunner:Runner",