Merge branch 'main' into no_setattr

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
         args:
           - --ignore-words-list=wronly,afile
   - repo: https://github.com/psf/black
-    rev: 23.10.1
+    rev: 23.11.0
     hooks:
       - id: black
         args: [ --safe, --quiet ]
@@ -45,7 +45,7 @@ repos:
           - flake8-bugbear
         args: ["--extend-ignore=E203", "--max-line-length=88"]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.6.1
+    rev: v1.7.0
     hooks:
       - id: mypy
         exclude: (docs|pyfakefs/tests)

CHANGES.md

@@ -4,13 +4,18 @@ The released versions correspond to PyPI releases.
 ## Unreleased
 
 ### Fixes
-* Restores compatability with PyTorch 2.0 and above, as well as with other classes that have custom __setattr__ methods (see #905).
+* fixes the problem that filesystem patching was still active in the pytest
+  logreport phase (see [#904](../../issues/904))
+* Restores compatability with PyTorch 2.0 and above, as well as with other
+  classes that have custom __setattr__ methods (see #905).
 
 ## [Version 5.3.0](https://pypi.python.org/pypi/pyfakefs/5.3.0) (2023-10-11)
 Adds official support for Python 3.12.
 
 ### Changes
 * add official support for Python 3.12
+* changed behavior of `add_real_directory` to be able to map a real directory
+  to an existing directory in the fake filesystem (see #901)
 
 ### Fixes
 * removed a leftover debug print statement (see [#869](../../issues/869))

docs/usage.rst

@@ -726,7 +726,9 @@ files are never changed.
 
 ``add_real_file()``, ``add_real_directory()`` and ``add_real_symlink()`` also
 allow you to map a file or a directory tree into another location in the
-fake filesystem via the argument ``target_path``.
+fake filesystem via the argument ``target_path``. If the target directory already exists
+in the fake filesystem, the directory contents are merged. If a file in the fake filesystem
+would be overwritten by a file from the real filesystem, an exception is raised.
 
 .. code:: python
 

extra_requirements.txt

@@ -15,6 +15,6 @@ scandir>=1.8
 # we use the latest version to see any problems with new versions
 pandas==1.3.5; python_version == '3.7' # pyup: ignore
 pandas==2.0.3; python_version == '3.8' # pyup: ignore
-pandas==2.1.2; python_version > '3.8'
+pandas==2.1.3; python_version > '3.8'
 xlrd==2.0.1
 openpyxl==3.1.2

pyfakefs/fake_filesystem.py

@@ -2232,6 +2232,8 @@ def add_real_directory(
         :py:class:`FakeDirectory<pyfakefs.fake_file.FakeDirectory>` object.
         Add entries in the fake directory corresponding to
         the entries in the real directory.  Symlinks are supported.
+        If the target directory already exists in the fake filesystem, the directory
+        contents are merged. Overwriting existing files is not allowed.
 
         Args:
             source_path: The path to the existing directory.
@@ -2254,49 +2256,78 @@ def add_real_directory(
             :py:class:`FakeDirectory<pyfakefs.fake_file.FakeDirectory>` object.
 
         Raises:
-            OSError: if the directory does not exist in the real file system.
-            OSError: if the directory already exists in the fake file system.
+            OSError: if the directory does not exist in the real filesystem.
+            OSError: if a file or link exists in the fake filesystem where a real
+                file or directory shall be mapped.
         """
-        source_path_str = make_string_path(source_path)  # TODO: add test
+        source_path_str = make_string_path(source_path)
         source_path_str = self._path_without_trailing_separators(source_path_str)
         if not os.path.exists(source_path_str):
             self.raise_os_error(errno.ENOENT, source_path_str)
         target_path_str = make_string_path(target_path or source_path_str)
+
+        # get rid of inconsistencies between real and fake path separators
+        if os.altsep is not None:
+            target_path_str = os.path.normpath(target_path_str)
+        if os.sep != self.path_separator:
+            target_path_str = target_path_str.replace(os.sep, self.path_separator)
+
         self._auto_mount_drive_if_needed(target_path_str)
-        new_dir: FakeDirectory
         if lazy_read:
-            parent_path = os.path.split(target_path_str)[0]
-            if self.exists(parent_path):
-                parent_dir = self.get_object(parent_path)
-            else:
-                parent_dir = self.create_dir(parent_path)
-            new_dir = FakeDirectoryFromRealDirectory(
-                source_path_str, self, read_only, target_path_str
+            self._create_fake_from_real_dir_lazily(
+                source_path_str, target_path_str, read_only
             )
-            parent_dir.add_entry(new_dir)
         else:
-            new_dir = self.create_dir(target_path_str)
-            for base, _, files in os.walk(source_path_str):
-                new_base = os.path.join(
-                    new_dir.path,  # type: ignore[arg-type]
-                    os.path.relpath(base, source_path_str),
-                )
-                for fileEntry in os.listdir(base):
-                    abs_fileEntry = os.path.join(base, fileEntry)
-
-                    if not os.path.islink(abs_fileEntry):
-                        continue
-
-                    self.add_real_symlink(
-                        abs_fileEntry, os.path.join(new_base, fileEntry)
-                    )
-                for fileEntry in files:
-                    path = os.path.join(base, fileEntry)
-                    if os.path.islink(path):
-                        continue
+            self._create_fake_from_real_dir(source_path_str, target_path_str, read_only)
+        return cast(FakeDirectory, self.get_object(target_path_str))
+
+    def _create_fake_from_real_dir(self, source_path_str, target_path_str, read_only):
+        if not self.exists(target_path_str):
+            self.create_dir(target_path_str)
+        for base, _, files in os.walk(source_path_str):
+            new_base = os.path.join(
+                target_path_str,
+                os.path.relpath(base, source_path_str),
+            )
+            for file_entry in os.listdir(base):
+                file_path = os.path.join(base, file_entry)
+                if os.path.islink(file_path):
+                    self.add_real_symlink(file_path, os.path.join(new_base, file_entry))
+            for file_entry in files:
+                path = os.path.join(base, file_entry)
+                if not os.path.islink(path):
                     self.add_real_file(
-                        path, read_only, os.path.join(new_base, fileEntry)
+                        path, read_only, os.path.join(new_base, file_entry)
                     )
+
+    def _create_fake_from_real_dir_lazily(
+        self, source_path_str, target_path_str, read_only
+    ):
+        if self.exists(target_path_str):
+            if not self.isdir(target_path_str):
+                raise OSError(errno.ENOTDIR, "Mapping target is not a directory")
+            for entry in os.listdir(source_path_str):
+                src_entry_path = os.path.join(source_path_str, entry)
+                target_entry_path = os.path.join(target_path_str, entry)
+                if os.path.isdir(src_entry_path):
+                    self.add_real_directory(
+                        src_entry_path, read_only, True, target_entry_path
+                    )
+                elif os.path.islink(src_entry_path):
+                    self.add_real_symlink(src_entry_path, target_entry_path)
+                elif os.path.isfile(src_entry_path):
+                    self.add_real_file(src_entry_path, read_only, target_entry_path)
+            return self.get_object(target_path_str)
+
+        parent_path = os.path.split(target_path_str)[0]
+        if self.exists(parent_path):
+            parent_dir = self.get_object(parent_path)
+        else:
+            parent_dir = self.create_dir(parent_path)
+        new_dir = FakeDirectoryFromRealDirectory(
+            source_path_str, self, read_only, target_path_str
+        )
+        parent_dir.add_entry(new_dir)
         return new_dir
 
     def add_real_paths(
@@ -2322,8 +2353,8 @@ def add_real_paths(
         Raises:
             OSError: if any of the files and directories in the list
                 does not exist in the real file system.
-            OSError: if any of the files and directories in the list
-                already exists in the fake file system.
+            OSError: if a file or link exists in the fake filesystem where a real
+                file or directory shall be mapped.
         """
         for path in path_list:
             if os.path.isdir(path):

pyfakefs/fake_filesystem_unittest.py

@@ -628,6 +628,7 @@ def __init__(
         self._isStale = True
         self._dyn_patcher: Optional[DynamicPatcher] = None
         self._patching = False
+        self._paused = False
 
     @classmethod
     def clear_fs_cache(cls) -> None:
@@ -898,6 +899,7 @@ def setUp(self, doctester: Any = None) -> None:
     def start_patching(self) -> None:
         if not self._patching:
             self._patching = True
+            self._paused = False
 
             self.patch_modules()
             self.patch_functions()
@@ -975,17 +977,22 @@ def tearDown(self, doctester: Any = None):
         else:
             self.__class__.PATCHER = None
 
-    def stop_patching(self) -> None:
+    def stop_patching(self, temporary=False) -> None:
         if self._patching:
             self._isStale = True
             self._patching = False
+            self._paused = temporary
             if self._stubs:
                 self._stubs.smart_unset_all()
             self.unset_defaults()
             if self._dyn_patcher:
                 self._dyn_patcher.cleanup()
                 sys.meta_path.pop(0)
 
+    @property
+    def is_patching(self):
+        return self._patching
+
     def unset_defaults(self) -> None:
         for fct, idx, ft in self.FS_DEFARGS:
             new_defaults = []
@@ -1003,15 +1010,16 @@ def pause(self) -> None:
         Calling pause() twice is silently ignored.
 
         """
-        self.stop_patching()
+        self.stop_patching(temporary=True)
 
     def resume(self) -> None:
         """Resume the patching of the file system modules if `pause` has
         been called before. After that call, all file system calls are
         executed in the fake file system.
         Does nothing if patching is not paused.
         """
-        self.start_patching()
+        if self._paused:
+            self.start_patching()
 
 
 class Pause:

pyfakefs/pytest_plugin.py

@@ -80,3 +80,16 @@ def fs_session(request):
 def pytest_sessionfinish(session, exitstatus):
     """Make sure that the cache is cleared before the final test shutdown."""
     Patcher.clear_fs_cache()
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_runtest_logreport(report):
+    """Make sure that patching is not active during reporting."""
+    if report.when == "call" and Patcher.PATCHER is not None:
+        Patcher.PATCHER.pause()
+
+
+def pytest_runtest_call(item):
+    """Resume paused patching before test start."""
+    if Patcher.PATCHER is not None:
+        Patcher.PATCHER.resume()

pyfakefs/pytest_tests/pytest_module_fixture_test.py

@@ -20,7 +20,19 @@ def use_fs(fs_module):
     yield fs_module
 
 
+@pytest.fixture(autouse=True)
+def check_patching_stopped(fs):
+    # patching shall be paused at test end, even in module scope (see #904)
+    yield
+    assert not fs.patcher.is_patching
+
+
 @pytest.mark.usefixtures("fs")
-def test_fs_uses_fs_module():
+def test_fs_uses_fs_module1():
     # check that `fs` uses the same filesystem as `fs_module`
     assert os.path.exists(os.path.join("foo", "bar"))
+
+
+def test_fs_uses_fs_module2(fs):
+    # check that testing was not stopped by the first test
+    assert os.path.exists(os.path.join("foo", "bar"))