Introducing "testing" object (#1852)

Signed-off-by: Fabrice Macagno <[email protected]>
Signed-off-by: Nathan Cheung <[email protected]>
Signed-off-by: Jean-Christophe Morin <[email protected]>
Co-authored-by: Fabrice Macagno <[email protected]>
Co-authored-by: Nathan Cheung <[email protected]>

docs/source/package_commands.rst

@@ -707,6 +707,19 @@ Following is a list of the objects and functions available.
       if test.name == "unit":
          info("My unit test is about to run yay")
 
+.. py:attribute:: testing
+   :type: bool
+
+   This boolean variable is ``True`` if a test is occurring (typically done via the :ref:`rez-test` tool),
+   and ``False`` otherwise.
+
+   A package can use this variable to set environment variables that are only relevant during test execution.
+
+   .. code-block:: python
+
+      if testing:
+          env.FOO_TEST_DATA_PATH = "{root}/tests/data"
+
 .. py:attribute:: this
 
    The ``this`` object represents the current package. The following attributes are most commonly used

docs/source/package_definition.rst

@@ -290,6 +290,7 @@ is ``True``:
 * **context**: the :class:`~rez.resolved_context.ResolvedContext` instance this package belongs to;
 * **system**: see :attr:`system`;
 * **building**: see :attr:`building`;
+* **testing**: see :attr:`testing`;
 * **request**: see :attr:`request`;
 * **implicits**: see :attr:`implicits`.
 

src/rez/data/tests/builds/packages/testing_obj/1.0.0/build.py

@@ -0,0 +1,24 @@
+from build_util import build_directory_recurse
+import os.path
+
+
+def build(source_path, build_path, install_path, targets):
+
+    if "install" not in (targets or []):
+        install_path = None
+
+    build_directory_recurse(src_dir="testing_obj",
+                            dest_dir=os.path.join("python", "testing_obj"),
+                            source_path=source_path,
+                            build_path=build_path,
+                            install_path=install_path)
+
+
+if __name__ == '__main__':
+    import os, sys
+    build(
+        source_path=os.environ['REZ_BUILD_SOURCE_PATH'],
+        build_path=os.environ['REZ_BUILD_PATH'],
+        install_path=os.environ['REZ_BUILD_INSTALL_PATH'],
+        targets=sys.argv[1:]
+    )

src/rez/data/tests/builds/packages/testing_obj/1.0.0/package.py

@@ -0,0 +1,42 @@
+name = 'testing obj'
+version = '1.0.0'
+authors = ["Dan Flashes"]
+
+description = "testing the 'testing' attribute available during rez test"
+
+@late()
+def requires():
+    if in_context() and testing:
+        return ["floob"]
+    return ["hello"]
+
+private_build_requires = ["build_util", "python"]
+
+def commands():
+    env.PYTHONPATH.append('{root}/python')
+    if testing:
+        env.CAR_IDEA = "STURDY STEERING WHEEL"
+    else:
+        env.SKIP_LUNCH = "False"
+
+build_command = 'python {root}/build.py {install}'
+
+tests = {
+    "command_as_string_success": {
+        "command": "exit 0"
+    },
+    "command_as_string_fail": {
+        "command": "exit 1"
+    },
+    "check_car_ideas": {
+        "command": ["python", "-c", "import os; assert os.environ.get('CAR_IDEA') == 'STURDY STEERING WHEEL'"],
+        "requires": ["python"]
+    },
+    "move_meeting_to_noon": {
+        # We want this test to fail. SKIP_LUNCH should not be set.
+        # TODO: We should not test for failures here. Testing failures, str vs lsit commands, etc
+        # should we tested separately.
+        "command": ["python", "-c", "import os; assert os.environ.get('SKIP_LUNCH') is not None"],
+        "requires": ["python"]
+    }
+}

src/rez/data/tests/builds/packages/testing_obj/1.0.0/testing_obj/__init__.py

@@ -0,0 +1,2 @@
+def hello():
+    return "This shirt was $150 out the door and the pattern's not that complicated"

src/rez/package_test.py

@@ -603,6 +603,7 @@ def _get_context(self, requires, quiet=False):
                     package_paths=self.package_paths,
                     buf=(f if quiet else None),
                     timestamp=self.timestamp,
+                    testing=True,
                     **self.context_kwargs
                 )
 

src/rez/resolved_context.py

@@ -163,7 +163,7 @@ def __call__(self, state):
             return SolverCallbackReturn.keep_going, ''
 
     def __init__(self, package_requests, verbosity=0, timestamp=None,
-                 building=False, caching=None, package_paths=None,
+                 building=False, testing=False, caching=None, package_paths=None,
                  package_filter=None, package_orderers=None, max_fails=-1,
                  add_implicit_packages=True, time_limit=-1, callback=None,
                  package_load_callback=None, buf=None, suppress_passive=False,
@@ -176,6 +176,7 @@ def __init__(self, package_requests, verbosity=0, timestamp=None,
             timestamp (float): Ignore packages released after this epoch time. Packages
                 released at exactly this time will not be ignored.
             building (bool): True if we're resolving for a build.
+            testing (bool): True if we're resolving for a test (rez-test).
             caching (bool): If True, cache(s) may be used to speed the resolve. If
                 False, caches will not be used. If None, :data:`resolve_caching`
                 is used.
@@ -214,6 +215,7 @@ def __init__(self, package_requests, verbosity=0, timestamp=None,
         self.requested_timestamp = timestamp
         self.timestamp = self.requested_timestamp or int(time.time())
         self.building = building
+        self.testing = testing
         self.implicit_packages = []
         self.caching = config.resolve_caching if caching is None else caching
         self.verbosity = verbosity
@@ -1553,6 +1555,7 @@ def _add(field):
             timestamp=self.timestamp,
             requested_timestamp=self.requested_timestamp,
             building=self.building,
+            testing=self.testing,
             caching=self.caching,
             implicit_packages=list(map(str, self.implicit_packages)),
             package_requests=list(map(str, self._package_requests)),
@@ -1627,6 +1630,7 @@ def _print_version(value):
 
         r.timestamp = d["timestamp"]
         r.building = d["building"]
+        r.testing = d["testing"]
         r.caching = d["caching"]
         r.implicit_packages = [PackageRequest(x) for x in d["implicit_packages"]]
         r._package_requests = [PackageRequest(x) for x in d["package_requests"]]
@@ -1959,6 +1963,7 @@ def _get_pre_resolve_bindings(self):
             self.pre_resolve_bindings = {
                 "system": system,
                 "building": self.building,
+                "testing": self.testing,
                 "request": RequirementsBinding(self._package_requests),
                 "implicits": RequirementsBinding(self.implicit_packages),
                 "intersects": intersects

src/rez/resolver.py

@@ -37,8 +37,8 @@ class Resolver(object):
     """
     def __init__(self, context, package_requests, package_paths, package_filter=None,
                  package_orderers=None, timestamp=0, callback=None, building=False,
-                 verbosity=False, buf=None, package_load_callback=None, caching=True,
-                 suppress_passive=False, print_stats=False):
+                 testing=False, verbosity=False, buf=None, package_load_callback=None,
+                 caching=True, suppress_passive=False, print_stats=False):
         """Create a Resolver.
 
         Args:
@@ -52,6 +52,7 @@ def __init__(self, context, package_requests, package_paths, package_filter=None
                 prior to each package being loaded. It is passed a single
                 `Package` object.
             building: True if we're resolving for a build.
+            testing: True if we're resolving for a rez (rez-test).
             caching: If True, cache(s) may be used to speed the resolve. If
                 False, caches will not be used.
             print_stats (bool): If true, print advanced solver stats at the end.
@@ -64,6 +65,7 @@ def __init__(self, context, package_requests, package_paths, package_filter=None
         self.package_orderers = package_orderers
         self.package_load_callback = package_load_callback
         self.building = building
+        self.testing = testing
         self.verbosity = verbosity
         self.caching = caching
         self.buf = buf
@@ -384,6 +386,7 @@ def _memcache_key(self, timestamped=False):
              self.package_filter_hash,
              self.package_orderers_hash,
              self.building,
+             self.testing,
              config.prune_failed_graph]
 
         if timestamped and self.timestamp:

src/rez/serialise.py

@@ -184,6 +184,7 @@ def _load_file(filepath, format_, update_data_callback, original_filepath=None):
 # Default variables to avoid not-defined errors in early-bound attribs
 default_objects = {
     "building": False,
+    "testing": False,
     "build_variant_index": 0,
     "build_variant_requires": []
 }

src/rez/tests/test_context.py

@@ -71,6 +71,26 @@ def test_execute_command(self):
         stdout = stdout.strip()
         self.assertEqual(stdout, "Hello Rez World!")
 
+    def test_resolved_packages_testing_environ(self):
+        """Test resolving packages within a testing environment behaves correctly"""
+        packages_path = self.data_path("builds", "packages")
+
+        # Note how we use testing=True
+        r = ResolvedContext(["testing_obj"], testing=True, package_paths=[packages_path])
+        resolvedPackages = [x.qualified_package_name for x in r.resolved_packages]
+        self.assertEqual(resolvedPackages, ["floob", "testing_obj-1.0.0"])
+
+    def test_execute_command_testing_environ(self):
+        """Test that execute_command properly sets test specific environ dict"""
+        self.inject_python_repo()
+        packages_path = self.data_path("builds", "packages")
+        r = ResolvedContext(
+            ["testing_obj", "python"],
+            testing=True,
+            package_paths=[packages_path] + self.settings["packages_path"]
+        )
+        self.assertEqual(r.get_environ().get("CAR_IDEA"), "STURDY STEERING WHEEL")
+
     def test_execute_command_environ(self):
         """Test that execute_command properly sets environ dict."""
         self.inject_python_repo()

src/rez/tests/test_test.py

@@ -0,0 +1,85 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright Contributors to the Rez Project
+
+
+"""
+test rez package.py unit tests
+"""
+from rez.tests.util import TestBase, TempdirMixin
+from rez.resolved_context import ResolvedContext
+from rez.package_test import PackageTestRunner
+
+
+class TestTest(TestBase, TempdirMixin):
+    @classmethod
+    def setUpClass(cls):
+        TempdirMixin.setUpClass()
+
+        packages_path = cls.data_path("builds", "packages")
+        cls.settings = dict(
+            packages_path=[packages_path],
+            package_filter=None,
+            implicit_packages=[],
+            warn_untimestamped=False,
+            resolve_caching=False
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        TempdirMixin.tearDownClass()
+
+    def test_1(self):
+        """package.py unit tests are correctly run in a testing environment"""
+        self.inject_python_repo()
+        context = ResolvedContext(["testing_obj", "python"])
+        self._run_tests(context)
+
+    def test_2(self):
+        """package.py unit tests are correctly run in a testing environment when no verbosity is set"""
+        self.inject_python_repo()
+        context = ResolvedContext(["testing_obj", "python"])
+        # This will get us more code coverage :)
+        self._run_tests(context, verbose=0)
+
+    def _run_tests(self, r, verbose=2):
+        """Run unit tests in package.py"""
+        self.inject_python_repo()
+        runner = PackageTestRunner(
+            package_request="testing_obj",
+            package_paths=r.package_paths,
+            stop_on_fail=False,
+            verbose=verbose
+        )
+
+        test_names = runner.get_test_names()
+
+        for test_name in test_names:
+            runner.run_test(test_name)
+
+        self.assertEqual(runner.test_results.num_tests, 4)
+        self.assertEqual(
+            self._get_test_result(runner, "check_car_ideas")["status"],
+            "success",
+            "check_car_ideas did not succeed",
+        )
+        self.assertEqual(
+            self._get_test_result(runner, "move_meeting_to_noon")["status"],
+            "failed",
+            "move_meeting_to_noon did not fail",
+        )
+        self.assertEqual(
+            self._get_test_result(runner, "command_as_string_success")["status"],
+            "success",
+            "command_as_string_success did not succeed",
+        )
+        self.assertEqual(
+            self._get_test_result(runner, "command_as_string_fail")["status"],
+            "failed",
+            "command_as_string_fail did not fail",
+        )
+
+    def _get_test_result(self, runner, test_name):
+        return next(
+            (result for result in runner.test_results.test_results if result.get("test_name") == test_name),
+            None
+        )