pythongh-80958: unittest: discovery support for namespace packages as…

… start directory (python#123820)

Doc/library/unittest.rst

@@ -340,28 +340,21 @@ Test modules and packages can customize test loading and discovery by through
 the `load_tests protocol`_.
 
 .. versionchanged:: 3.4
-   Test discovery supports :term:`namespace packages <namespace package>`
-   for the start directory. Note that you need to specify the top level
-   directory too (e.g.
-   ``python -m unittest discover -s root/namespace -t root``).
+   Test discovery supports :term:`namespace packages <namespace package>`.
 
 .. versionchanged:: 3.11
-   :mod:`unittest` dropped the :term:`namespace packages <namespace package>`
-   support in Python 3.11. It has been broken since Python 3.7. Start directory and
-   subdirectories containing tests must be regular package that have
-   ``__init__.py`` file.
+   Test discovery dropped the :term:`namespace packages <namespace package>`
+   support. It has been broken since Python 3.7.
+   Start directory and its subdirectories containing tests must be regular
+   package that have ``__init__.py`` file.
 
-   Directories containing start directory still can be a namespace package.
-   In this case, you need to specify start directory as dotted package name,
-   and target directory explicitly. For example::
+   If the start directory is the dotted name of the package, the ancestor packages
+   can be namespace packages.
 
-      # proj/  <-- current directory
-      #   namespace/
-      #     mypkg/
-      #       __init__.py
-      #       test_mypkg.py
-
-      python -m unittest discover -s namespace.mypkg -t .
+.. versionchanged:: 3.14
+   Test discovery supports :term:`namespace package` as start directory again.
+   To avoid scanning directories unrelated to Python,
+   tests are not searched in subdirectories that do not contain ``__init__.py``.
 
 
 .. _organizing-tests:
@@ -1915,10 +1908,8 @@ Loading and running tests
          Modules that raise :exc:`SkipTest` on import are recorded as skips,
          not errors.
 
-      .. versionchanged:: 3.4
          *start_dir* can be a :term:`namespace packages <namespace package>`.
 
-      .. versionchanged:: 3.4
          Paths are sorted before being imported so that execution order is the
          same even if the underlying file system's ordering is not dependent
          on file name.
@@ -1930,11 +1921,13 @@ Loading and running tests
 
       .. versionchanged:: 3.11
          *start_dir* can not be a :term:`namespace packages <namespace package>`.
-         It has been broken since Python 3.7 and Python 3.11 officially remove it.
+         It has been broken since Python 3.7, and Python 3.11 officially removes it.
 
       .. versionchanged:: 3.13
          *top_level_dir* is only stored for the duration of *discover* call.
 
+      .. versionchanged:: 3.14
+         *start_dir* can once again be a :term:`namespace package`.
 
    The following attributes of a :class:`TestLoader` can be configured either by
    subclassing or assignment on an instance:

Doc/whatsnew/3.14.rst

@@ -421,6 +421,15 @@ unicodedata
 
 * The Unicode database has been updated to Unicode 16.0.0.
 
+
+unittest
+--------
+
+* unittest discovery supports :term:`namespace package` as start
+  directory again. It was removed in Python 3.11.
+  (Contributed by Jacob Walls in :gh:`80958`.)
+
+
 .. Add improved modules above alphabetically, not here at the end.
 
 Optimizations

Lib/test/test_unittest/namespace_test_pkg/bar/test_bar.py

@@ -0,0 +1,5 @@
+import unittest
+
+class PassingTest(unittest.TestCase):
+    def test_true(self):
+        self.assertTrue(True)

Lib/test/test_unittest/namespace_test_pkg/noop/no2/test_no2.py

@@ -0,0 +1,5 @@
+import unittest
+
+class PassingTest(unittest.TestCase):
+    def test_true(self):
+        self.assertTrue(True)

Lib/test/test_unittest/namespace_test_pkg/noop/test_noop.py

@@ -0,0 +1,5 @@
+import unittest
+
+class PassingTest(unittest.TestCase):
+    def test_true(self):
+        self.assertTrue(True)

Lib/test/test_unittest/namespace_test_pkg/test_foo.py

@@ -0,0 +1,5 @@
+import unittest
+
+class PassingTest(unittest.TestCase):
+    def test_true(self):
+        self.assertTrue(True)

Lib/test/test_unittest/test_discovery.py

@@ -4,12 +4,14 @@
 import sys
 import types
 import pickle
+from importlib._bootstrap_external import NamespaceLoader
 from test import support
 from test.support import import_helper
 
 import unittest
 import unittest.mock
 import test.test_unittest
+from test.test_importlib import util as test_util
 
 
 class TestableTestProgram(unittest.TestProgram):
@@ -395,7 +397,7 @@ def restore_isdir():
         self.addCleanup(restore_isdir)
 
         _find_tests_args = []
-        def _find_tests(start_dir, pattern):
+        def _find_tests(start_dir, pattern, namespace=None):
             _find_tests_args.append((start_dir, pattern))
             return ['tests']
         loader._find_tests = _find_tests
@@ -815,7 +817,7 @@ def test_discovery_from_dotted_path(self):
         expectedPath = os.path.abspath(os.path.dirname(test.test_unittest.__file__))
 
         self.wasRun = False
-        def _find_tests(start_dir, pattern):
+        def _find_tests(start_dir, pattern, namespace=None):
             self.wasRun = True
             self.assertEqual(start_dir, expectedPath)
             return tests
@@ -848,6 +850,54 @@ def restore():
                          'Can not use builtin modules '
                          'as dotted module names')
 
+    def test_discovery_from_dotted_namespace_packages(self):
+        loader = unittest.TestLoader()
+
+        package = types.ModuleType('package')
+        package.__name__ = "tests"
+        package.__path__ = ['/a', '/b']
+        package.__file__ = None
+        package.__spec__ = types.SimpleNamespace(
+            name=package.__name__,
+            loader=NamespaceLoader(package.__name__, package.__path__, None),
+            submodule_search_locations=['/a', '/b']
+        )
+
+        def _import(packagename, *args, **kwargs):
+            sys.modules[packagename] = package
+            return package
+
+        _find_tests_args = []
+        def _find_tests(start_dir, pattern, namespace=None):
+            _find_tests_args.append((start_dir, pattern))
+            return ['%s/tests' % start_dir]
+
+        loader._find_tests = _find_tests
+        loader.suiteClass = list
+
+        with unittest.mock.patch('builtins.__import__', _import):
+            # Since loader.discover() can modify sys.path, restore it when done.
+            with import_helper.DirsOnSysPath():
+                # Make sure to remove 'package' from sys.modules when done.
+                with test_util.uncache('package'):
+                    suite = loader.discover('package')
+
+        self.assertEqual(suite, ['/a/tests', '/b/tests'])
+
+    def test_discovery_start_dir_is_namespace(self):
+        """Subdirectory discovery not affected if start_dir is a namespace pkg."""
+        loader = unittest.TestLoader()
+        with (
+            import_helper.DirsOnSysPath(os.path.join(os.path.dirname(__file__))),
+            test_util.uncache('namespace_test_pkg')
+        ):
+            suite = loader.discover('namespace_test_pkg')
+        self.assertEqual(
+            {list(suite)[0]._tests[0].__module__ for suite in suite._tests if list(suite)},
+            # files under namespace_test_pkg.noop not discovered.
+            {'namespace_test_pkg.test_foo', 'namespace_test_pkg.bar.test_bar'},
+        )
+
     def test_discovery_failed_discovery(self):
         from test.test_importlib import util
 

Lib/unittest/loader.py

@@ -274,6 +274,8 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None):
         self._top_level_dir = top_level_dir
 
         is_not_importable = False
+        is_namespace = False
+        tests = []
         if os.path.isdir(os.path.abspath(start_dir)):
             start_dir = os.path.abspath(start_dir)
             if start_dir != top_level_dir:
@@ -286,12 +288,25 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None):
                 is_not_importable = True
             else:
                 the_module = sys.modules[start_dir]
-                top_part = start_dir.split('.')[0]
-                try:
-                    start_dir = os.path.abspath(
-                        os.path.dirname((the_module.__file__)))
-                except AttributeError:
-                    if the_module.__name__ in sys.builtin_module_names:
+                if not hasattr(the_module, "__file__") or the_module.__file__ is None:
+                    # look for namespace packages
+                    try:
+                        spec = the_module.__spec__
+                    except AttributeError:
+                        spec = None
+
+                    if spec and spec.submodule_search_locations is not None:
+                        is_namespace = True
+
+                        for path in the_module.__path__:
+                            if (not set_implicit_top and
+                                not path.startswith(top_level_dir)):
+                                continue
+                            self._top_level_dir = \
+                                (path.split(the_module.__name__
+                                        .replace(".", os.path.sep))[0])
+                            tests.extend(self._find_tests(path, pattern, namespace=True))
+                    elif the_module.__name__ in sys.builtin_module_names:
                         # builtin module
                         raise TypeError('Can not use builtin modules '
                                         'as dotted module names') from None
@@ -300,14 +315,27 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None):
                             f"don't know how to discover from {the_module!r}"
                             ) from None
 
+                else:
+                    top_part = start_dir.split('.')[0]
+                    start_dir = os.path.abspath(os.path.dirname((the_module.__file__)))
+
                 if set_implicit_top:
-                    self._top_level_dir = self._get_directory_containing_module(top_part)
+                    if not is_namespace:
+                        if sys.modules[top_part].__file__ is None:
+                            self._top_level_dir = os.path.dirname(the_module.__file__)
+                            if self._top_level_dir not in sys.path:
+                                sys.path.insert(0, self._top_level_dir)
+                        else:
+                            self._top_level_dir = \
+                                self._get_directory_containing_module(top_part)
                     sys.path.remove(top_level_dir)
 
         if is_not_importable:
             raise ImportError('Start directory is not importable: %r' % start_dir)
 
-        tests = list(self._find_tests(start_dir, pattern))
+        if not is_namespace:
+            tests = list(self._find_tests(start_dir, pattern))
+
         self._top_level_dir = original_top_level_dir
         return self.suiteClass(tests)
 
@@ -343,7 +371,7 @@ def _match_path(self, path, full_path, pattern):
         # override this method to use alternative matching strategy
         return fnmatch(path, pattern)
 
-    def _find_tests(self, start_dir, pattern):
+    def _find_tests(self, start_dir, pattern, namespace=False):
         """Used by discovery. Yields test suites it loads."""
         # Handle the __init__ in this package
         name = self._get_name_from_path(start_dir)
@@ -352,7 +380,8 @@ def _find_tests(self, start_dir, pattern):
         if name != '.' and name not in self._loading_packages:
             # name is in self._loading_packages while we have called into
             # loadTestsFromModule with name.
-            tests, should_recurse = self._find_test_path(start_dir, pattern)
+            tests, should_recurse = self._find_test_path(
+                start_dir, pattern, namespace)
             if tests is not None:
                 yield tests
             if not should_recurse:
@@ -363,19 +392,20 @@ def _find_tests(self, start_dir, pattern):
         paths = sorted(os.listdir(start_dir))
         for path in paths:
             full_path = os.path.join(start_dir, path)
-            tests, should_recurse = self._find_test_path(full_path, pattern)
+            tests, should_recurse = self._find_test_path(
+                full_path, pattern, False)
             if tests is not None:
                 yield tests
             if should_recurse:
                 # we found a package that didn't use load_tests.
                 name = self._get_name_from_path(full_path)
                 self._loading_packages.add(name)
                 try:
-                    yield from self._find_tests(full_path, pattern)
+                    yield from self._find_tests(full_path, pattern, False)
                 finally:
                     self._loading_packages.discard(name)
 
-    def _find_test_path(self, full_path, pattern):
+    def _find_test_path(self, full_path, pattern, namespace=False):
         """Used by discovery.
 
         Loads tests from a single file, or a directories' __init__.py when
@@ -419,7 +449,8 @@ def _find_test_path(self, full_path, pattern):
                         msg % (mod_name, module_dir, expected_dir))
                 return self.loadTestsFromModule(module, pattern=pattern), False
         elif os.path.isdir(full_path):
-            if not os.path.isfile(os.path.join(full_path, '__init__.py')):
+            if (not namespace and
+                not os.path.isfile(os.path.join(full_path, '__init__.py'))):
                 return None, False
 
             load_tests = None

Makefile.pre.in

@@ -2534,6 +2534,10 @@ TESTSUBDIRS=	idlelib/idle_test \
 		test/test_tools \
 		test/test_ttk \
 		test/test_unittest \
+		test/test_unittest/namespace_test_pkg \
+		test/test_unittest/namespace_test_pkg/bar \
+		test/test_unittest/namespace_test_pkg/noop \
+		test/test_unittest/namespace_test_pkg/noop/no2 \
 		test/test_unittest/testmock \
 		test/test_warnings \
 		test/test_warnings/data \

Misc/NEWS.d/next/Library/2024-09-07-13-57-49.gh-issue-80958.fVYnqV.rst

@@ -0,0 +1 @@
+unittest discovery supports PEP 420 namespace packages as start directory again.