From 761f72443cf0ed898c2d70490b76428e9cd14d2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Fri, 10 Jan 2020 21:32:52 +0100 Subject: [PATCH 01/24] initial add_dll_directory --- fiona/__init__.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/fiona/__init__.py b/fiona/__init__.py index ecef54c2b..a50cb8ebe 100644 --- a/fiona/__init__.py +++ b/fiona/__init__.py @@ -67,7 +67,7 @@ import os import sys import warnings - +import platform from six import string_types from collections import OrderedDict @@ -77,6 +77,18 @@ class Path: pass + +# Add gdal dll directory on Windows and Python >= 3.8, see https://github.com/Toblerity/Fiona/issues/851 +if platform.system() == 'Windows' and (3, 8) <= sys.version_info: + + # Add GDAL_HOME/bin, if present + gdal_home = os.getenv('GDAL_HOME', None) + + if gdal_home is not None and os.path.exists(gdal_home): + os.add_dll_directory(os.path.join(gdal_home, "bin")) + + + # TODO: remove this? Or at least move it, flake8 complains. if sys.platform == "win32": libdir = os.path.join(os.path.dirname(__file__), ".libs") From 7c8967edb457901c107dd63c418180a8a81c6e84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Fri, 10 Jan 2020 21:34:55 +0100 Subject: [PATCH 02/24] remove empty line --- fiona/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fiona/__init__.py b/fiona/__init__.py index a50cb8ebe..45db964d2 100644 --- a/fiona/__init__.py +++ b/fiona/__init__.py @@ -88,7 +88,6 @@ class Path: os.add_dll_directory(os.path.join(gdal_home, "bin")) - # TODO: remove this? Or at least move it, flake8 complains. if sys.platform == "win32": libdir = os.path.join(os.path.dirname(__file__), ".libs") From 69c95b5fae930642746aa5b7e8b4ea689780c475 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Mon, 13 Jan 2020 12:46:28 +0100 Subject: [PATCH 03/24] search for gdal/bin in PATH, search for dll directories only when import of ogrext failed with ImportError --- fiona/__init__.py | 59 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/fiona/__init__.py b/fiona/__init__.py index 45db964d2..c6aec4c9f 100644 --- a/fiona/__init__.py +++ b/fiona/__init__.py @@ -71,28 +71,58 @@ from six import string_types from collections import OrderedDict + +# TODO: remove this? Or at least move it, flake8 complains. +if sys.platform == "win32": + libdir = os.path.join(os.path.dirname(__file__), ".libs") + os.environ["PATH"] = os.environ["PATH"] + ";" + libdir + try: - from pathlib import Path -except ImportError: # pragma: no cover - class Path: - pass + from fiona.ogrext import _bounds, _listlayers, FIELD_TYPES_MAP, _remove, \ + _remove_layer -# Add gdal dll directory on Windows and Python >= 3.8, see https://github.com/Toblerity/Fiona/issues/851 -if platform.system() == 'Windows' and (3, 8) <= sys.version_info: +except ImportError as e: + """ + With Python >= 3.8 on Windows directories in PATH are not automatically + searched for DLL dependencies and must be added manually with + os.add_dll_directory. - # Add GDAL_HOME/bin, if present - gdal_home = os.getenv('GDAL_HOME', None) + see https://github.com/Toblerity/Fiona/issues/851 - if gdal_home is not None and os.path.exists(gdal_home): - os.add_dll_directory(os.path.join(gdal_home, "bin")) + We check if a */gdal/bin directory is found in PATH and + if none is found if GDAL_HOME is set. + """ + if platform.system() == 'Windows' and (3, 8) <= sys.version_info: + dll_directory = None -# TODO: remove this? Or at least move it, flake8 complains. -if sys.platform == "win32": - libdir = os.path.join(os.path.dirname(__file__), ".libs") - os.environ["PATH"] = os.environ["PATH"] + ";" + libdir + # Parse PATH for gdal/bin + for path in os.getenv('PATH', '').split(';'): + p = Path(path.lower()) + + if p.parts[-2:] == ('gdal', 'bin') and os.path.exists(path): + dll_directory = path + break + del path, p + + # Use GDAL_HOME if present + if dll_directory is not None: + gdal_home = os.getenv('GDAL_HOME', None) + if gdal_home is not None and os.path.exists(gdal_home): + dll_directory = os.path.join(gdal_home, "bin") + del gdal_home + + if dll_directory is not None: + os.add_dll_directory(dll_directory) + del dll_directory + + from fiona.ogrext import _bounds, _listlayers, FIELD_TYPES_MAP, \ + _remove, _remove_layer + + else: + raise e from fiona.collection import BytesCollection, Collection from fiona.drvsupport import supported_drivers from fiona.env import ensure_env_with_credentials, Env @@ -102,7 +132,6 @@ class Path: calc_gdal_version_num, get_gdal_version_num, get_gdal_release_name, get_gdal_version_tuple) from fiona.io import MemoryFile -from fiona.ogrext import _bounds, _listlayers, FIELD_TYPES_MAP, _remove, _remove_layer from fiona.path import ParsedPath, parse_path, vsi_path from fiona.vfs import parse_paths as vfs_parse_paths From 81de199170e59147df0e0faff07e982c21fb5f20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Mon, 13 Jan 2020 12:50:31 +0100 Subject: [PATCH 04/24] move del p --- fiona/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fiona/__init__.py b/fiona/__init__.py index c6aec4c9f..11fddaee9 100644 --- a/fiona/__init__.py +++ b/fiona/__init__.py @@ -104,7 +104,8 @@ if p.parts[-2:] == ('gdal', 'bin') and os.path.exists(path): dll_directory = path break - del path, p + del p + del path # Use GDAL_HOME if present if dll_directory is not None: From a1a7f8c04b1051eb5e83f5617d7137da455265e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Mon, 13 Jan 2020 12:58:11 +0100 Subject: [PATCH 05/24] add missing pathlib import --- fiona/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fiona/__init__.py b/fiona/__init__.py index 11fddaee9..363a151e2 100644 --- a/fiona/__init__.py +++ b/fiona/__init__.py @@ -70,7 +70,7 @@ import platform from six import string_types from collections import OrderedDict - +from pathlib import Path # TODO: remove this? Or at least move it, flake8 complains. if sys.platform == "win32": From efcd995deac75e5e70a523addf44ca915677fad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Mon, 13 Jan 2020 13:01:22 +0100 Subject: [PATCH 06/24] move del path --- fiona/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fiona/__init__.py b/fiona/__init__.py index 363a151e2..cb33dde20 100644 --- a/fiona/__init__.py +++ b/fiona/__init__.py @@ -104,8 +104,7 @@ if p.parts[-2:] == ('gdal', 'bin') and os.path.exists(path): dll_directory = path break - del p - del path + del path, p # Use GDAL_HOME if present if dll_directory is not None: From 8a40a9853b68b24762b17664a1d4820f6fe8518e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Mon, 13 Jan 2020 13:23:44 +0100 Subject: [PATCH 07/24] add python 3.8 / gdal 3.0.2 to build matrix, switch to stable gisinternal builds for gdal 3.0.x --- appveyor.yml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 6ffd51301..55b85348e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -38,13 +38,12 @@ environment: GIS_INTERNALS: "release-1911-x64-gdal-2-4-2-mapserver-7-4-0.zip" GIS_INTERNALS_LIBS: "release-1911-x64-gdal-2-4-2-mapserver-7-4-0-libs.zip" - # Use daily stable branch of gdal 3.0.x series until release with gdal >= 3.0.1 is released - PYTHON: "C:\\Python36-x64" PYTHON_VERSION: "3.6" PYTHON_ARCH: "64" - GDAL_VERSION: "3.0.0" - GIS_INTERNALS: "release-1911-x64-gdal-3-0-mapserver-7-4.zip" - GIS_INTERNALS_LIBS: "release-1911-x64-gdal-3-0-mapserver-7-4-libs.zip" + GDAL_VERSION: "3.0.2" + GIS_INTERNALS: "release-1911-x64-gdal-3-0-2-mapserver-7-4-2.zip" + GIS_INTERNALS_LIBS: "release-1911-x64-gdal-3-0-2-mapserver-7-4-2-libs.zip" PROJ_LIB: "C:\\gdal\\bin\\proj6\\share" - PYTHON: "C:\\Python37-x64" @@ -57,9 +56,9 @@ environment: - PYTHON: "C:\\Python37-x64" PYTHON_VERSION: "3.7" PYTHON_ARCH: "64" - GDAL_VERSION: "3.0.0" - GIS_INTERNALS: "release-1911-x64-gdal-3-0-mapserver-7-4.zip" - GIS_INTERNALS_LIBS: "release-1911-x64-gdal-3-0-mapserver-7-4-libs.zip" + GDAL_VERSION: "3.0.2" + GIS_INTERNALS: "release-1911-x64-gdal-3-0-2-mapserver-7-4-2.zip" + GIS_INTERNALS_LIBS: "release-1911-x64-gdal-3-0-2-mapserver-7-4-2-libs.zip" PROJ_LIB: "C:\\gdal\\bin\\proj6\\share" - PYTHON: "C:\\Python37-x64" @@ -70,6 +69,14 @@ environment: GIS_INTERNALS_LIBS: "release-1911-x64-gdal-3-0-mapserver-7-4-libs.zip" PROJ_LIB: "C:\\gdal\\bin\\proj6\\share" + - PYTHON: "C:\\Python38-x64" + PYTHON_VERSION: "3.8" + PYTHON_ARCH: "64" + GDAL_VERSION: "3.0.2" + GIS_INTERNALS: "release-1911-x64-gdal-3-0-2-mapserver-7-4-2.zip" + GIS_INTERNALS_LIBS: "release-1911-x64-gdal-3-0-2-mapserver-7-4-2-libs.zip" + PROJ_LIB: "C:\\gdal\\bin\\proj6\\share" + - PYTHON: "C:\\Python38-x64" PYTHON_VERSION: "3.8" PYTHON_ARCH: "64" From bb3f8d4cb5e6c019902d455f3bdc0b8ae89488ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Tue, 24 Mar 2020 09:57:07 +0100 Subject: [PATCH 08/24] refactor import --- fiona/__init__.py | 51 +++++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/fiona/__init__.py b/fiona/__init__.py index cb33dde20..263b6f018 100644 --- a/fiona/__init__.py +++ b/fiona/__init__.py @@ -77,10 +77,10 @@ libdir = os.path.join(os.path.dirname(__file__), ".libs") os.environ["PATH"] = os.environ["PATH"] + ";" + libdir + try: - from fiona.ogrext import _bounds, _listlayers, FIELD_TYPES_MAP, _remove, \ - _remove_layer + import fiona.ogrext except ImportError as e: """ @@ -89,40 +89,43 @@ os.add_dll_directory. see https://github.com/Toblerity/Fiona/issues/851 - - We check if a */gdal/bin directory is found in PATH and - if none is found if GDAL_HOME is set. """ if platform.system() == 'Windows' and (3, 8) <= sys.version_info: + + def add_dll_directory_win(): + """ + Check if a */gdal/bin directory is found in PATH. + If none is found, use GDAL_HOME if present. + """ - dll_directory = None + dll_directory = None - # Parse PATH for gdal/bin - for path in os.getenv('PATH', '').split(';'): - p = Path(path.lower()) + # Parse PATH for gdal/bin + for _path in os.getenv('PATH', '').split(';'): + p = Path(_path.lower()) - if p.parts[-2:] == ('gdal', 'bin') and os.path.exists(path): - dll_directory = path - break - del path, p + if p.parts[-2:] == ('gdal', 'bin') and os.path.exists(_path): + dll_directory = _path + break - # Use GDAL_HOME if present - if dll_directory is not None: - gdal_home = os.getenv('GDAL_HOME', None) + # Use GDAL_HOME if present + if dll_directory is not None: + gdal_home = os.getenv('GDAL_HOME', None) - if gdal_home is not None and os.path.exists(gdal_home): - dll_directory = os.path.join(gdal_home, "bin") - del gdal_home + if gdal_home is not None and os.path.exists(gdal_home): + dll_directory = os.path.join(gdal_home, "bin") - if dll_directory is not None: - os.add_dll_directory(dll_directory) - del dll_directory + if dll_directory is not None: + os.add_dll_directory(dll_directory) - from fiona.ogrext import _bounds, _listlayers, FIELD_TYPES_MAP, \ - _remove, _remove_layer + add_dll_directory_win() else: raise e + +from fiona.ogrext import _bounds, _listlayers, FIELD_TYPES_MAP, _remove, \ + _remove_layer + from fiona.collection import BytesCollection, Collection from fiona.drvsupport import supported_drivers from fiona.env import ensure_env_with_credentials, Env From 4241697f08656e3d3270dbd8f661fcb17c1f08bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Tue, 24 Mar 2020 10:05:44 +0100 Subject: [PATCH 09/24] remove python 3.8 constraint for finding dll directories --- fiona/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fiona/__init__.py b/fiona/__init__.py index 263b6f018..1108ad53d 100644 --- a/fiona/__init__.py +++ b/fiona/__init__.py @@ -90,7 +90,7 @@ see https://github.com/Toblerity/Fiona/issues/851 """ - if platform.system() == 'Windows' and (3, 8) <= sys.version_info: + if platform.system() == 'Windows': def add_dll_directory_win(): """ From 932bfa03235fbe8f67b5b8888d4fd311d3c8d52e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Tue, 24 Mar 2020 10:17:11 +0100 Subject: [PATCH 10/24] update documentation --- README.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 87571fa1d..fafba6982 100644 --- a/README.rst +++ b/README.rst @@ -300,8 +300,11 @@ the ``GDAL_VERSION`` environment variable (e.g. ``set GDAL_VERSION=2.1``). $ python setup.py build_ext -I -lgdal_i -L install --gdalversion 2.1 -Note: The GDAL DLL (``gdal111.dll`` or similar) and gdal-data directory need to -be in your Windows PATH otherwise Fiona will fail to work. + +Note: The directory containing the GDAL DLL (``gdal111.dll`` or similar) need to be in your Windows ``PATH`` +(e.g. ``C:\gdal\bin``). Alternatively, ``GDAL_HOME`` needs to be set (e.g. ``C:\gdal``). Furthermore, the +gdal-data directory needs to be in your Windows PATH or the environment variable ``GDAL_DATA`` must be +set (e.g. ``C:\gdal\bin\gdal-data``), otherwise Fiona will fail to work. The `Appveyor CI build `__ uses the GISInternals GDAL binaries to build Fiona. This produces a binary wheel From cdb433845e5d880f3749baba20f609b1ff8393cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Tue, 24 Mar 2020 10:25:02 +0100 Subject: [PATCH 11/24] update documentation --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index fafba6982..48a4f7c53 100644 --- a/README.rst +++ b/README.rst @@ -301,10 +301,10 @@ the ``GDAL_VERSION`` environment variable (e.g. ``set GDAL_VERSION=2.1``). $ python setup.py build_ext -I -lgdal_i -L install --gdalversion 2.1 -Note: The directory containing the GDAL DLL (``gdal111.dll`` or similar) need to be in your Windows ``PATH`` -(e.g. ``C:\gdal\bin``). Alternatively, ``GDAL_HOME`` needs to be set (e.g. ``C:\gdal``). Furthermore, the -gdal-data directory needs to be in your Windows PATH or the environment variable ``GDAL_DATA`` must be -set (e.g. ``C:\gdal\bin\gdal-data``), otherwise Fiona will fail to work. +Note: The directory containing the GDAL DLL (``gdal111.dll`` or similar) needs to be in your Windows ``PATH`` +(e.g. ``C:\gdal\bin``). Alternatively, ``GDAL_HOME`` needs to be set (e.g. ``C:\gdal``). A ``*\gdal\bin`` directory in ``Path`` +has priority over ``GDAL_HOME``. Furthermore, the gdal-data directory needs to be in your Windows PATH or the environment variable +``GDAL_DATA`` must be set (e.g. ``C:\gdal\bin\gdal-data``), otherwise Fiona will fail to work. The `Appveyor CI build `__ uses the GISInternals GDAL binaries to build Fiona. This produces a binary wheel From e4757708b53280a11588674d355e4426a47349f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Tue, 24 Mar 2020 10:26:31 +0100 Subject: [PATCH 12/24] formatting --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 48a4f7c53..f8f33733b 100644 --- a/README.rst +++ b/README.rst @@ -302,8 +302,8 @@ the ``GDAL_VERSION`` environment variable (e.g. ``set GDAL_VERSION=2.1``). Note: The directory containing the GDAL DLL (``gdal111.dll`` or similar) needs to be in your Windows ``PATH`` -(e.g. ``C:\gdal\bin``). Alternatively, ``GDAL_HOME`` needs to be set (e.g. ``C:\gdal``). A ``*\gdal\bin`` directory in ``Path`` -has priority over ``GDAL_HOME``. Furthermore, the gdal-data directory needs to be in your Windows PATH or the environment variable +(e.g. ``C:\gdal\bin``). Alternatively, ``GDAL_HOME`` needs to be set (e.g. ``C:\gdal``). A ``*\gdal\bin`` directory in ``PATH`` +has priority over ``GDAL_HOME``. Furthermore, the gdal-data directory needs to be in your Windows ``PATH`` or the environment variable ``GDAL_DATA`` must be set (e.g. ``C:\gdal\bin\gdal-data``), otherwise Fiona will fail to work. The `Appveyor CI build `__ From ee46b235c0e144dde3b4c3c21e324b4bf44df458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Tue, 24 Mar 2020 10:35:11 +0100 Subject: [PATCH 13/24] add missing import --- fiona/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fiona/__init__.py b/fiona/__init__.py index 1108ad53d..974100b85 100644 --- a/fiona/__init__.py +++ b/fiona/__init__.py @@ -120,6 +120,8 @@ def add_dll_directory_win(): add_dll_directory_win() + import fiona.ogrext + else: raise e From 472a87d0009dafd482658910b6ac4cd9a4b764e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Tue, 31 Mar 2020 10:16:02 +0200 Subject: [PATCH 14/24] refactor dll directory search and add --- fiona/__init__.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/fiona/__init__.py b/fiona/__init__.py index 974100b85..e1990964b 100644 --- a/fiona/__init__.py +++ b/fiona/__init__.py @@ -77,31 +77,30 @@ libdir = os.path.join(os.path.dirname(__file__), ".libs") os.environ["PATH"] = os.environ["PATH"] + ";" + libdir - try: import fiona.ogrext except ImportError as e: - """ - With Python >= 3.8 on Windows directories in PATH are not automatically - searched for DLL dependencies and must be added manually with - os.add_dll_directory. - see https://github.com/Toblerity/Fiona/issues/851 - """ - if platform.system() == 'Windows': - + # With Python >= 3.8 on Windows directories in PATH are not automatically + # searched for DLL dependencies and must be added manually with + # os.add_dll_directory. + # see https://github.com/Toblerity/Fiona/issues/851 + + if platform.system() == 'Windows' and (3, 8) <= sys.version_info: + def add_dll_directory_win(): - """ - Check if a */gdal/bin directory is found in PATH. - If none is found, use GDAL_HOME if present. + """ Finds and adds dll directories on Windows + + Checks if a */gdal/bin directory is present in PATH. + If none is found, GDAL_HOME is used if available. """ dll_directory = None # Parse PATH for gdal/bin - for _path in os.getenv('PATH', '').split(';'): + for _path in os.getenv('PATH', '').split(os.pathsep): p = Path(_path.lower()) if p.parts[-2:] == ('gdal', 'bin') and os.path.exists(_path): @@ -109,7 +108,8 @@ def add_dll_directory_win(): break # Use GDAL_HOME if present - if dll_directory is not None: + if dll_directory is None: + gdal_home = os.getenv('GDAL_HOME', None) if gdal_home is not None and os.path.exists(gdal_home): @@ -120,7 +120,7 @@ def add_dll_directory_win(): add_dll_directory_win() - import fiona.ogrext + import fiona.ogrext else: raise e From bd9791996d569e355c5f7684302d1255739dc01a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Tue, 31 Mar 2020 10:34:28 +0200 Subject: [PATCH 15/24] doc update --- README.rst | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index f8f33733b..d2223930a 100644 --- a/README.rst +++ b/README.rst @@ -301,10 +301,16 @@ the ``GDAL_VERSION`` environment variable (e.g. ``set GDAL_VERSION=2.1``). $ python setup.py build_ext -I -lgdal_i -L install --gdalversion 2.1 -Note: The directory containing the GDAL DLL (``gdal111.dll`` or similar) needs to be in your Windows ``PATH`` -(e.g. ``C:\gdal\bin``). Alternatively, ``GDAL_HOME`` needs to be set (e.g. ``C:\gdal``). A ``*\gdal\bin`` directory in ``PATH`` -has priority over ``GDAL_HOME``. Furthermore, the gdal-data directory needs to be in your Windows ``PATH`` or the environment variable -``GDAL_DATA`` must be set (e.g. ``C:\gdal\bin\gdal-data``), otherwise Fiona will fail to work. +Note: The following environment variables needs to be set so that Fiona works correctly: + +* The directory containing the GDAL DLL (``gdal300.dll`` or similar) needs to be in your + Windows ``PATH`` (e.g. ``C:\gdal\bin``). Alternatively, the environment variable + ``GDAL_HOME`` can be used (e.g. ``C:\gdal``). A ``*\gdal\bin`` directory in ``PATH`` has + priority over ``GDAL_HOME``. +* The gdal-data directory needs to be in your Windows ``PATH`` or the environment variable + ``GDAL_DATA`` must be set (e.g. ``C:\gdal\bin\gdal-data``). +* The environment variable ``PROJ_LIB`` must be set to the proj library directory (e.g. + ``C:\gdal\bin\proj6\share``) The `Appveyor CI build `__ uses the GISInternals GDAL binaries to build Fiona. This produces a binary wheel From 94588c7ee1c06ec8862641a50eb3ccd08eee1d97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Wed, 1 Apr 2020 10:51:20 +0200 Subject: [PATCH 16/24] bump appveyor gdal versions --- appveyor.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 55b85348e..f4788f2ed 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -41,9 +41,9 @@ environment: - PYTHON: "C:\\Python36-x64" PYTHON_VERSION: "3.6" PYTHON_ARCH: "64" - GDAL_VERSION: "3.0.2" - GIS_INTERNALS: "release-1911-x64-gdal-3-0-2-mapserver-7-4-2.zip" - GIS_INTERNALS_LIBS: "release-1911-x64-gdal-3-0-2-mapserver-7-4-2-libs.zip" + GDAL_VERSION: "3.0.4" + GIS_INTERNALS: "release-1911-x64-gdal-3-0-4-mapserver-7-4-3.zip" + GIS_INTERNALS_LIBS: "release-1911-x64-gdal-3-0-4-mapserver-7-4-3-libs.zip" PROJ_LIB: "C:\\gdal\\bin\\proj6\\share" - PYTHON: "C:\\Python37-x64" @@ -56,9 +56,9 @@ environment: - PYTHON: "C:\\Python37-x64" PYTHON_VERSION: "3.7" PYTHON_ARCH: "64" - GDAL_VERSION: "3.0.2" - GIS_INTERNALS: "release-1911-x64-gdal-3-0-2-mapserver-7-4-2.zip" - GIS_INTERNALS_LIBS: "release-1911-x64-gdal-3-0-2-mapserver-7-4-2-libs.zip" + GDAL_VERSION: "3.0.4" + GIS_INTERNALS: "release-1911-x64-gdal-3-0-4-mapserver-7-4-3.zip" + GIS_INTERNALS_LIBS: "release-1911-x64-gdal-3-0-4-mapserver-7-4-3-libs.zip" PROJ_LIB: "C:\\gdal\\bin\\proj6\\share" - PYTHON: "C:\\Python37-x64" @@ -72,9 +72,9 @@ environment: - PYTHON: "C:\\Python38-x64" PYTHON_VERSION: "3.8" PYTHON_ARCH: "64" - GDAL_VERSION: "3.0.2" - GIS_INTERNALS: "release-1911-x64-gdal-3-0-2-mapserver-7-4-2.zip" - GIS_INTERNALS_LIBS: "release-1911-x64-gdal-3-0-2-mapserver-7-4-2-libs.zip" + GDAL_VERSION: "3.0.4" + GIS_INTERNALS: "release-1911-x64-gdal-3-0-4-mapserver-7-4-3.zip" + GIS_INTERNALS_LIBS: "release-1911-x64-gdal-3-0-4-mapserver-7-4-3-libs.zip" PROJ_LIB: "C:\\gdal\\bin\\proj6\\share" - PYTHON: "C:\\Python38-x64" From b608b2ce9bd4f5635dc890dfb6e0ecad2995c424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Wed, 1 Apr 2020 10:51:56 +0200 Subject: [PATCH 17/24] refactor to search for gdal*.dll --- fiona/__init__.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/fiona/__init__.py b/fiona/__init__.py index e1990964b..99ae08ff7 100644 --- a/fiona/__init__.py +++ b/fiona/__init__.py @@ -71,6 +71,7 @@ from six import string_types from collections import OrderedDict from pathlib import Path +import glob # TODO: remove this? Or at least move it, flake8 complains. if sys.platform == "win32": @@ -90,21 +91,25 @@ if platform.system() == 'Windows' and (3, 8) <= sys.version_info: + def directory_contains_gdal_dll(path): + """ Checks if a directory contains a gdal dll """ + return len(glob.glob(os.path.join(path, "gdal*.dll"))) > 0 + def add_dll_directory_win(): """ Finds and adds dll directories on Windows - Checks if a */gdal/bin directory is present in PATH. + Checks if a */gdal/* directory is present in PATH + and contains a gdal*.dll file. If none is found, GDAL_HOME is used if available. """ dll_directory = None # Parse PATH for gdal/bin - for _path in os.getenv('PATH', '').split(os.pathsep): - p = Path(_path.lower()) + for path in os.getenv('PATH', '').split(os.pathsep): - if p.parts[-2:] == ('gdal', 'bin') and os.path.exists(_path): - dll_directory = _path + if "gdal" in path.lower() and directory_contains_gdal_dll(path): + dll_directory = path break # Use GDAL_HOME if present @@ -113,7 +118,11 @@ def add_dll_directory_win(): gdal_home = os.getenv('GDAL_HOME', None) if gdal_home is not None and os.path.exists(gdal_home): - dll_directory = os.path.join(gdal_home, "bin") + + if directory_contains_gdal_dll(gdal_home): + dll_directory = gdal_home + elif directory_contains_gdal_dll(os.path.join(gdal_home, "bin")): + dll_directory = os.path.join(gdal_home, "bin") if dll_directory is not None: os.add_dll_directory(dll_directory) From 51bee20a0db807232904387c2115fd1ba2f33c31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Wed, 1 Apr 2020 11:09:31 +0200 Subject: [PATCH 18/24] add logging --- fiona/__init__.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/fiona/__init__.py b/fiona/__init__.py index 99ae08ff7..1f3527249 100644 --- a/fiona/__init__.py +++ b/fiona/__init__.py @@ -73,6 +73,9 @@ from pathlib import Path import glob +log = logging.getLogger(__name__) +log.addHandler(logging.NullHandler()) + # TODO: remove this? Or at least move it, flake8 complains. if sys.platform == "win32": libdir = os.path.join(os.path.dirname(__file__), ".libs") @@ -124,8 +127,14 @@ def add_dll_directory_win(): elif directory_contains_gdal_dll(os.path.join(gdal_home, "bin")): dll_directory = os.path.join(gdal_home, "bin") + elif gdal_home is not None and not os.path.exists(gdal_home): + log.warn("GDAL_HOME directory does not exist.") + if dll_directory is not None: + log.info("Adding dll directory: {}".format(dll_directory)) os.add_dll_directory(dll_directory) + else: + log.warn("No dll directory found to add.") add_dll_directory_win() @@ -161,10 +170,6 @@ def add_dll_directory_win(): gdal_version = get_gdal_version_tuple() -log = logging.getLogger(__name__) -log.addHandler(logging.NullHandler()) - - @ensure_env_with_credentials def open(fp, mode='r', driver=None, schema=None, crs=None, encoding=None, layer=None, vfs=None, enabled_drivers=None, crs_wkt=None, From 19cf456e219c5dad852171d2d8d020552c52e857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Wed, 1 Apr 2020 11:22:58 +0200 Subject: [PATCH 19/24] update readme --- README.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index d2223930a..c51ec86e8 100644 --- a/README.rst +++ b/README.rst @@ -303,10 +303,11 @@ the ``GDAL_VERSION`` environment variable (e.g. ``set GDAL_VERSION=2.1``). Note: The following environment variables needs to be set so that Fiona works correctly: -* The directory containing the GDAL DLL (``gdal300.dll`` or similar) needs to be in your - Windows ``PATH`` (e.g. ``C:\gdal\bin``). Alternatively, the environment variable - ``GDAL_HOME`` can be used (e.g. ``C:\gdal``). A ``*\gdal\bin`` directory in ``PATH`` has - priority over ``GDAL_HOME``. +* The directory containing the GDAL DLL (``gdal304.dll`` or similar) needs to be in your + Windows ``PATH`` (e.g. ``C:\gdal\bin``). Only directories containing ``gdal`` in + their names are considered. Alternatively, the environment variable ``GDAL_HOME`` can + be used (e.g. ``C:\gdal``). A ``*\*gdal*\*`` directory in ``PATH`` has priority over + ``GDAL_HOME``. * The gdal-data directory needs to be in your Windows ``PATH`` or the environment variable ``GDAL_DATA`` must be set (e.g. ``C:\gdal\bin\gdal-data``). * The environment variable ``PROJ_LIB`` must be set to the proj library directory (e.g. From d6d04ee2c5c865981bd7d43985b3aeebd8831e8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Wed, 1 Apr 2020 13:40:42 +0200 Subject: [PATCH 20/24] add gdal_home directory to logging message --- fiona/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fiona/__init__.py b/fiona/__init__.py index 1f3527249..cc0b8888b 100644 --- a/fiona/__init__.py +++ b/fiona/__init__.py @@ -128,7 +128,7 @@ def add_dll_directory_win(): dll_directory = os.path.join(gdal_home, "bin") elif gdal_home is not None and not os.path.exists(gdal_home): - log.warn("GDAL_HOME directory does not exist.") + log.warn("GDAL_HOME directory ({}) does not exist.".format(gdal_home)) if dll_directory is not None: log.info("Adding dll directory: {}".format(dll_directory)) From e94e7e13bef2009197f58bf2ab98513f2f9be626 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Fri, 3 Apr 2020 12:55:26 +0200 Subject: [PATCH 21/24] add dlls using contextmanager --- appveyor.yml | 2 +- fiona/__init__.py | 61 +++++--------------------------------- fiona/_loading.py | 75 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 55 deletions(-) create mode 100644 fiona/_loading.py diff --git a/appveyor.yml b/appveyor.yml index f4788f2ed..97a3cd21c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -170,4 +170,4 @@ matrix: artifacts: - path: dist\*.whl - name: wheel + name: wheel \ No newline at end of file diff --git a/fiona/__init__.py b/fiona/__init__.py index cc0b8888b..e456cadcf 100644 --- a/fiona/__init__.py +++ b/fiona/__init__.py @@ -71,7 +71,7 @@ from six import string_types from collections import OrderedDict from pathlib import Path -import glob +import fiona._loading log = logging.getLogger(__name__) log.addHandler(logging.NullHandler()) @@ -84,64 +84,16 @@ try: import fiona.ogrext + import fiona._env except ImportError as e: - # With Python >= 3.8 on Windows directories in PATH are not automatically - # searched for DLL dependencies and must be added manually with - # os.add_dll_directory. - # see https://github.com/Toblerity/Fiona/issues/851 + import fiona._loading - if platform.system() == 'Windows' and (3, 8) <= sys.version_info: + with fiona._loading.add_gdal_dll_directories(): - def directory_contains_gdal_dll(path): - """ Checks if a directory contains a gdal dll """ - return len(glob.glob(os.path.join(path, "gdal*.dll"))) > 0 - - def add_dll_directory_win(): - """ Finds and adds dll directories on Windows - - Checks if a */gdal/* directory is present in PATH - and contains a gdal*.dll file. - If none is found, GDAL_HOME is used if available. - """ - - dll_directory = None - - # Parse PATH for gdal/bin - for path in os.getenv('PATH', '').split(os.pathsep): - - if "gdal" in path.lower() and directory_contains_gdal_dll(path): - dll_directory = path - break - - # Use GDAL_HOME if present - if dll_directory is None: - - gdal_home = os.getenv('GDAL_HOME', None) - - if gdal_home is not None and os.path.exists(gdal_home): - - if directory_contains_gdal_dll(gdal_home): - dll_directory = gdal_home - elif directory_contains_gdal_dll(os.path.join(gdal_home, "bin")): - dll_directory = os.path.join(gdal_home, "bin") - - elif gdal_home is not None and not os.path.exists(gdal_home): - log.warn("GDAL_HOME directory ({}) does not exist.".format(gdal_home)) - - if dll_directory is not None: - log.info("Adding dll directory: {}".format(dll_directory)) - os.add_dll_directory(dll_directory) - else: - log.warn("No dll directory found to add.") - - add_dll_directory_win() - - import fiona.ogrext - - else: - raise e + import fiona.ogrext + import fiona._env from fiona.ogrext import _bounds, _listlayers, FIELD_TYPES_MAP, _remove, \ _remove_layer @@ -158,6 +110,7 @@ def add_dll_directory_win(): from fiona.path import ParsedPath, parse_path, vsi_path from fiona.vfs import parse_paths as vfs_parse_paths + # These modules are imported by fiona.ogrext, but are also import here to # help tools like cx_Freeze find them automatically from fiona import _geometry, _err, rfc3339 diff --git a/fiona/_loading.py b/fiona/_loading.py new file mode 100644 index 000000000..a86023a34 --- /dev/null +++ b/fiona/_loading.py @@ -0,0 +1,75 @@ +import glob +import os +import logging +import contextlib +import platform +import sys + +log = logging.getLogger(__name__) +log.addHandler(logging.NullHandler()) + +# With Python >= 3.8 on Windows directories in PATH are not automatically +# searched for DLL dependencies and must be added manually with +# os.add_dll_directory. +# see https://github.com/Toblerity/Fiona/issues/851 + +dll_directories = [] + + +def directory_contains_gdal_dll(path): + """ Checks if a directory contains a gdal dll """ + return len(glob.glob(os.path.join(path, "gdal*.dll"))) > 0 + + +def search_gdal_dll_directories(): + """ Search for gdal dlls + + Checks if a */*gdal*/* directory is present in PATH + and contains a gdal*.dll file. + If none is found, GDAL_HOME is used if available. + """ + + # Parse PATH for gdal/bin + for path in os.getenv('PATH', '').split(os.pathsep): + + if "gdal" in path.lower() and directory_contains_gdal_dll(path): + dll_directories.append(path) + + # Use GDAL_HOME if present + if len(dll_directories) == 0: + + gdal_home = os.getenv('GDAL_HOME', None) + + if gdal_home is not None and os.path.exists(gdal_home): + + if directory_contains_gdal_dll(gdal_home): + dll_directories.append(gdal_home) + elif directory_contains_gdal_dll(os.path.join(gdal_home, "bin")): + dll_directories.append(os.path.join(gdal_home, "bin")) + + elif gdal_home is not None and not os.path.exists(gdal_home): + log.warning("GDAL_HOME directory ({}) does not exist.".format(gdal_home)) + + if len(dll_directories) == 0: + log.warning("No dll directory found.") + + +if platform.system() == 'Windows' and (3, 8) <= sys.version_info: + search_gdal_dll_directories() + + +@contextlib.contextmanager +def add_gdal_dll_directories(): + + dll_dirs = [] + for dll_directory in dll_directories: + dll_dirs.append(os.add_dll_directory(dll_directory)) + + try: + + yield None + + finally: + + for dll_dir in dll_dirs: + dll_dir.close() From 987bc9986d59016214b8b43bb0da7cf0fd027e4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Sat, 4 Apr 2020 17:33:39 +0200 Subject: [PATCH 22/24] move extension import test to _loading --- appveyor.yml | 2 +- fiona/__init__.py | 43 ++++++++++++++++--------------------------- fiona/_loading.py | 13 +++++++------ 3 files changed, 24 insertions(+), 34 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 97a3cd21c..f4788f2ed 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -170,4 +170,4 @@ matrix: artifacts: - path: dist\*.whl - name: wheel \ No newline at end of file + name: wheel diff --git a/fiona/__init__.py b/fiona/__init__.py index e456cadcf..ad7ce6781 100644 --- a/fiona/__init__.py +++ b/fiona/__init__.py @@ -67,50 +67,35 @@ import os import sys import warnings -import platform + from six import string_types from collections import OrderedDict -from pathlib import Path -import fiona._loading -log = logging.getLogger(__name__) -log.addHandler(logging.NullHandler()) +try: + from pathlib import Path +except ImportError: # pragma: no cover + class Path: + pass # TODO: remove this? Or at least move it, flake8 complains. if sys.platform == "win32": libdir = os.path.join(os.path.dirname(__file__), ".libs") os.environ["PATH"] = os.environ["PATH"] + ";" + libdir -try: - - import fiona.ogrext - import fiona._env - -except ImportError as e: - - import fiona._loading - - with fiona._loading.add_gdal_dll_directories(): - - import fiona.ogrext - import fiona._env - -from fiona.ogrext import _bounds, _listlayers, FIELD_TYPES_MAP, _remove, \ - _remove_layer - +import fiona._loading +with fiona._loading.add_gdal_dll_directories(): + from fiona._env import ( + calc_gdal_version_num, get_gdal_version_num, get_gdal_release_name, + get_gdal_version_tuple, driver_count) + from fiona.ogrext import _bounds, _listlayers, FIELD_TYPES_MAP, _remove, _remove_layer from fiona.collection import BytesCollection, Collection from fiona.drvsupport import supported_drivers from fiona.env import ensure_env_with_credentials, Env from fiona.errors import FionaDeprecationWarning -from fiona._env import driver_count -from fiona._env import ( - calc_gdal_version_num, get_gdal_version_num, get_gdal_release_name, - get_gdal_version_tuple) from fiona.io import MemoryFile from fiona.path import ParsedPath, parse_path, vsi_path from fiona.vfs import parse_paths as vfs_parse_paths - # These modules are imported by fiona.ogrext, but are also import here to # help tools like cx_Freeze find them automatically from fiona import _geometry, _err, rfc3339 @@ -123,6 +108,10 @@ gdal_version = get_gdal_version_tuple() +log = logging.getLogger(__name__) +log.addHandler(logging.NullHandler()) + + @ensure_env_with_credentials def open(fp, mode='r', driver=None, schema=None, crs=None, encoding=None, layer=None, vfs=None, enabled_drivers=None, crs_wkt=None, diff --git a/fiona/_loading.py b/fiona/_loading.py index a86023a34..bb0ded7ae 100644 --- a/fiona/_loading.py +++ b/fiona/_loading.py @@ -51,11 +51,16 @@ def search_gdal_dll_directories(): log.warning("GDAL_HOME directory ({}) does not exist.".format(gdal_home)) if len(dll_directories) == 0: - log.warning("No dll directory found.") + log.warning("No dll directories found.") if platform.system() == 'Windows' and (3, 8) <= sys.version_info: - search_gdal_dll_directories() + + # if loading of extension modules fails, search for gdal dll directories + try: + import fiona.ogrext + except ImportError as e: + search_gdal_dll_directories() @contextlib.contextmanager @@ -64,12 +69,8 @@ def add_gdal_dll_directories(): dll_dirs = [] for dll_directory in dll_directories: dll_dirs.append(os.add_dll_directory(dll_directory)) - try: - yield None - finally: - for dll_dir in dll_dirs: dll_dir.close() From b1f7dbecac22326c2420ee3ff1e439bc4769a104 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Sat, 4 Apr 2020 17:56:45 +0200 Subject: [PATCH 23/24] search in all directories in PATH for gdal libraries --- fiona/_loading.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fiona/_loading.py b/fiona/_loading.py index bb0ded7ae..9e6cf778a 100644 --- a/fiona/_loading.py +++ b/fiona/_loading.py @@ -24,15 +24,14 @@ def directory_contains_gdal_dll(path): def search_gdal_dll_directories(): """ Search for gdal dlls - Checks if a */*gdal*/* directory is present in PATH - and contains a gdal*.dll file. - If none is found, GDAL_HOME is used if available. + Checks if a gdal dll is present in PATH directory. + If no directory is found, GDAL_HOME is used if available. """ # Parse PATH for gdal/bin for path in os.getenv('PATH', '').split(os.pathsep): - if "gdal" in path.lower() and directory_contains_gdal_dll(path): + if directory_contains_gdal_dll(path): dll_directories.append(path) # Use GDAL_HOME if present From 3247c694151144e064c7d46995323671600a62cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Buffat?= Date: Sat, 4 Apr 2020 17:57:32 +0200 Subject: [PATCH 24/24] use add_gdal_dll_directories also for import of _geometry, _err --- fiona/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fiona/__init__.py b/fiona/__init__.py index ad7ce6781..39f547db2 100644 --- a/fiona/__init__.py +++ b/fiona/__init__.py @@ -67,7 +67,6 @@ import os import sys import warnings - from six import string_types from collections import OrderedDict @@ -98,7 +97,8 @@ class Path: # These modules are imported by fiona.ogrext, but are also import here to # help tools like cx_Freeze find them automatically -from fiona import _geometry, _err, rfc3339 +with fiona._loading.add_gdal_dll_directories(): + from fiona import _geometry, _err, rfc3339 import uuid