diff --git a/CHANGES.md b/CHANGES.md index 326ea714..e6483560 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,15 +14,19 @@ The release versions are PyPi releases. ``` #### New Features - * a module imported as another name (`import os as _os`) is now correctly - patched without the need of additional parameters + * improved automatic patching: + * automatically patch methods of a patched file system module imported like + `from os.path import exists` + * a module imported as another name (`import os as _os`) is now correctly + patched without the need of additional parameters + ([#434](../../pull/434)) + * automatically patch `Path` if imported like `from pathlib import Path` + ([#440](../../issues/440)) + * parameter `patch_path` has been removed from `UnitTest` and `Patcher`, + the correct patching of `path` imports is now done automatically + * `UnitTest` /`Patcher` arguments can now also be set in `setUpPyfakefs()` * added possibility to set root user ([#431](../../issues/431)) - * automatically patch `Path` if imported like `from pathlib import Path` - ([#440](../../issues/440)) * added side_effect option to fake files ([#433](../../pull/433)) - * parameter `patch_path` has been removed from `UnitTest` and `Patcher`, - the correct patching of `path` imports is now done automatically - * `UnitTest` /`Patcher` arguments can now also be set in `setUpPyfakefs()` * added pathlib2 support ([#408](../../issues/408)) ([#422](../../issues/422)) * added some support for extended filesystem attributes under Linux ([#423](../../issues/423)) diff --git a/docs/usage.rst b/docs/usage.rst index 71ee48da..6e7aeb5f 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -111,6 +111,14 @@ Pyfakefs automatically patches file system related modules that are: from os import path from pathlib import Path +Additionally, functions from file system related modules are patched +automatically if imported like: + +.. code:: python + + from os.path import exists + from os import stat + There are other cases where automatic patching does not work. Both ``fake_filesystem_unittest.Patcher`` and ``fake_filesystem_unittest.TestCase`` provide a few additional arguments to handle such cases. @@ -132,8 +140,15 @@ modules_to_reload This allows to pass a list of modules that shall be reloaded, thus allowing to patch modules not patched automatically. +Here is a simple example for a default argument that is not patched +automatically: + +.. code:: python + + def check_if_exists(filepath, file_exists=os.path.exists): + return file_exists(filepath) -If adding the module containing these imports to ``modules_to_reload``, they +If adding the module containing this code to ``modules_to_reload``, it will be correctly patched. modules_to_patch diff --git a/pyfakefs/fake_filesystem.py b/pyfakefs/fake_filesystem.py index a5d78506..1e5df50f 100644 --- a/pyfakefs/fake_filesystem.py +++ b/pyfakefs/fake_filesystem.py @@ -3026,6 +3026,23 @@ class FakePathModule(object): """ _OS_PATH_COPY = _copy_module(os.path) + @staticmethod + def dir(): + """Return the list of patched function names. Used for patching + functions imported from the module. + """ + dir = [ + 'abspath', 'dirname', 'exists', 'expanduser', 'getatime', + 'getctime', 'getmtime', 'getsize', 'isabs', 'isdir', 'isfile', + 'islink', 'ismount', 'join', 'lexists', 'normcase', 'normpath', + 'realpath', 'relpath', 'split', 'splitdrive' + ] + if IS_PY2: + dir.append('walk') + if sys.platform != 'win32' or not IS_PY2: + dir.append('samefile') + return dir + def __init__(self, filesystem, os_module=None): """Init. @@ -3422,6 +3439,31 @@ class FakeOsModule(object): devnull = None + @staticmethod + def dir(): + """Return the list of patched function names. Used for patching + functions imported from the module. + """ + dir = [ + 'access', 'chdir', 'chmod', 'chown', 'close', 'fstat', 'fsync', + 'getcwd', 'lchmod', 'link', 'listdir', 'lstat', 'makedirs', + 'mkdir', 'mknod', 'open', 'read', 'readlink', 'remove', + 'removedirs', 'rename', 'rmdir', 'stat', 'symlink', 'umask', + 'unlink', 'utime', 'walk', 'write' + ] + if IS_PY2: + dir += ['getcwdu'] + else: + dir += ['getcwdb', 'replace'] + if sys.platform.startswith('linux'): + dir += [ + 'fdatasync','getxattr', 'listxattr', + 'removexattr', 'setxattr' + ] + if use_scandir: + dir += ['scandir'] + return dir + def __init__(self, filesystem, os_path_module=None): """Also exposes self.path (to fake os.path). @@ -3843,7 +3885,6 @@ def setxattr(self, path, attribute, value, self.filesystem.raise_os_error(errno.EEXIST, file_obj.path) file_obj.xattr[attribute] = value - if use_scandir: def scandir(self, path=''): """Return an iterator of DirEntry objects corresponding to the @@ -4420,6 +4461,15 @@ class FakeIoModule(object): my_io_module = fake_filesystem.FakeIoModule(filesystem) """ + @staticmethod + def dir(): + """Return the list of patched function names. Used for patching + functions imported from the module. + """ + # `open` would clash with build-in `open`, so don't patch it + # if imported like `from io import open` + return () + def __init__(self, filesystem): """ Args: diff --git a/pyfakefs/fake_filesystem_shutil.py b/pyfakefs/fake_filesystem_shutil.py index ce22b21e..e7f857e2 100755 --- a/pyfakefs/fake_filesystem_shutil.py +++ b/pyfakefs/fake_filesystem_shutil.py @@ -30,11 +30,22 @@ import shutil import sys +from pyfakefs.helpers import IS_PY2 + class FakeShutilModule(object): """Uses a FakeFilesystem to provide a fake replacement for shutil module. """ + @staticmethod + def dir(): + """Return the list of patched function names. Used for patching + functions imported from the module. + """ + if not IS_PY2: + return 'disk_usage', + return () + def __init__(self, filesystem): """Construct fake shutil module using the fake filesystem. diff --git a/pyfakefs/fake_filesystem_unittest.py b/pyfakefs/fake_filesystem_unittest.py index 91025397..8ec696b9 100644 --- a/pyfakefs/fake_filesystem_unittest.py +++ b/pyfakefs/fake_filesystem_unittest.py @@ -77,6 +77,7 @@ else: import builtins +OS_MODULE = 'nt' if sys.platform == 'win32' else 'posix' PATH_MODULE = 'ntpath' if sys.platform == 'win32' else 'posixpath' def load_doctests(loader, tests, ignore, module, @@ -284,7 +285,7 @@ class Patcher(object): IS_WINDOWS = sys.platform in ('win32', 'cygwin') - SKIPNAMES = {'os', 'path', 'io', 'genericpath'} + SKIPNAMES = {'os', 'path', 'io', 'genericpath', OS_MODULE, PATH_MODULE} if pathlib: SKIPNAMES.add('pathlib') @@ -338,8 +339,32 @@ def __init__(self, additional_skip_names=None, module_name) self._fake_module_classes[name] = fake_module + # handle patching function imported separately like + # `from os import stat` + # each patched function name has to be looked up separately + self._fake_module_functions = {} + for mod_name, fake_module in self._fake_module_classes.items(): + modnames = ( + (mod_name, OS_MODULE) if mod_name == 'os' else (mod_name,) + ) + for fct_name in fake_module.dir(): + self._fake_module_functions[fct_name] = ( + modnames, + getattr(fake_module, fct_name), + mod_name + ) + # special handling for functions in os.path + fake_module = fake_filesystem.FakePathModule + for fct_name in fake_module.dir(): + self._fake_module_functions[fct_name] = ( + ('genericpath', PATH_MODULE), + getattr(fake_module, fct_name), + PATH_MODULE + ) + # Attributes set by _refresh() self._modules = {} + self._fct_modules = {} self._stubs = None self.fs = None self.fake_open = None @@ -366,6 +391,12 @@ def _find_modules(self): Later, `setUp()` will stub these with the fake file system modules. """ + def is_fct(module, name): + fct = module.__dict__.get(name) + return (fct is not None and + (inspect.isfunction(fct) or inspect.isbuiltin(fct)) and + fct.__module__ in self._fake_module_functions[name][0]) + module_names = list(self._fake_module_classes.keys()) + [PATH_MODULE] for name, module in set(sys.modules.items()): try: @@ -378,6 +409,7 @@ def _find_modules(self): # where py.error has no __name__ attribute # see https://github.com/pytest-dev/py/issues/73 continue + modules = {name: mod for name, mod in module.__dict__.items() if inspect.ismodule(mod) and mod.__name__ in module_names @@ -387,6 +419,11 @@ def _find_modules(self): self._modules.setdefault(name, set()).add((module, mod.__name__)) + functions = [name for name in self._fake_module_functions + if is_fct(module, name)] + for name in functions: + self._fct_modules.setdefault(name, set()).add(module) + def _refresh(self): """Renew the fake file system and set the _isStale flag to `False`.""" if self._stubs is not None: @@ -416,10 +453,17 @@ def setUp(self, doctester=None): # file() was eliminated in Python3 self._stubs.smart_set(builtins, 'file', self.fake_open) self._stubs.smart_set(builtins, 'open', self.fake_open) - for name in self._modules: - for module, attr in self._modules[name]: + for name, modules in self._modules.items(): + for module, attr in modules: self._stubs.smart_set(module, name, self.fake_modules[attr]) + for name, modules in self._fct_modules.items(): + _, method, mod_name = self._fake_module_functions[name] + fake_module = self.fake_modules[mod_name] + attr = method.__get__(fake_module, fake_module.__class__) + for module in modules: + self._stubs.smart_set(module, name, attr) + self._dyn_patcher = DynamicPatcher(self) sys.meta_path.insert(0, self._dyn_patcher) diff --git a/pyfakefs/fake_pathlib.py b/pyfakefs/fake_pathlib.py index bd514fd6..4880b6bc 100644 --- a/pyfakefs/fake_pathlib.py +++ b/pyfakefs/fake_pathlib.py @@ -633,6 +633,13 @@ class FakePathlibModule(object): `fake_pathlib_module = fake_filesystem.FakePathlibModule(filesystem)` """ + @staticmethod + def dir(): + """Return an empty list as `Path` methods will always be called + on the instances. + """ + return () + def __init__(self, filesystem): """ Initializes the module with the given filesystem. @@ -676,6 +683,10 @@ class FakePathlibPathModule(object): """Patches `pathlib.Path` by passing all calls to FakePathlibModule.""" fake_pathlib = None + @staticmethod + def dir(): + return () + def __init__(self, filesystem): if self.fake_pathlib is None: self.__class__.fake_pathlib = FakePathlibModule(filesystem) diff --git a/pyfakefs/fake_scandir.py b/pyfakefs/fake_scandir.py index f00463d1..bf95cb98 100644 --- a/pyfakefs/fake_scandir.py +++ b/pyfakefs/fake_scandir.py @@ -258,6 +258,13 @@ class FakeScanDirModule(object): `fake_scandir_module = fake_filesystem.FakeScanDirModule(filesystem)` """ + @staticmethod + def dir(): + """Return the list of patched function names. Used for patching + functions imported from the module. + """ + return 'scandir', 'walk' + def __init__(self, filesystem): self.filesystem = filesystem diff --git a/pyfakefs/tests/fake_filesystem_unittest_test.py b/pyfakefs/tests/fake_filesystem_unittest_test.py index c654c0d1..4b9c596d 100644 --- a/pyfakefs/tests/fake_filesystem_unittest_test.py +++ b/pyfakefs/tests/fake_filesystem_unittest_test.py @@ -160,6 +160,18 @@ def test_import_path_from_pathlib(self): self.assertTrue( pyfakefs.tests.import_as_example.check_if_exists3(file_path)) + def test_import_function_from_os_path(self): + file_path = '/foo/bar' + self.fs.create_dir(file_path) + self.assertTrue( + pyfakefs.tests.import_as_example.check_if_exists5(file_path)) + + def test_import_function_from_os(self): + file_path = '/foo/bar' + self.fs.create_file(file_path, contents=b'abc') + stat_result = pyfakefs.tests.import_as_example.file_stat(file_path) + self.assertEqual(3, stat_result.st_size) + class TestAttributesWithFakeModuleNames(TestPyfakefsUnittestBase): """Test that module attributes with names like `path` or `io` are not diff --git a/pyfakefs/tests/import_as_example.py b/pyfakefs/tests/import_as_example.py index 46fe9903..ae301424 100644 --- a/pyfakefs/tests/import_as_example.py +++ b/pyfakefs/tests/import_as_example.py @@ -15,7 +15,9 @@ to be patched under another name. """ from os import path +from os.path import exists import os as my_os +from os import stat try: from pathlib import Path @@ -45,3 +47,13 @@ def check_if_exists3(filepath): def check_if_exists4(filepath, exists=my_os.path.exists): # this is a similar case as in the tempfile implementation under Posix return exists(filepath) + + +def check_if_exists5(filepath): + # tests patching `exists` imported from os.path + return exists(filepath) + + +def file_stat(filepath): + # tests patching `stat` imported from os + return stat(filepath)