From 664fdadb1ad9ef1718e1c7d6c7f12b9c0fe4ac66 Mon Sep 17 00:00:00 2001 From: Henrik Andersson Date: Tue, 12 Sep 2023 08:27:59 +0200 Subject: [PATCH 1/3] Pathlib instead of os.path --- mikeio/__init__.py | 12 +++--- mikeio/_track.py | 30 +++++++------- mikeio/dataset/_dataarray.py | 3 +- mikeio/dataset/_dataset.py | 5 ++- mikeio/dfs/_dfs.py | 2 +- mikeio/dfs/_dfs0.py | 23 ++++++----- mikeio/dfs/_dfs1.py | 27 ++++++------ mikeio/dfs/_dfs2.py | 12 ++---- mikeio/dfs/_dfs3.py | 8 +--- mikeio/dfsu/_dfsu.py | 8 ++-- mikeio/dfsu/_factory.py | 4 +- tests/test_dataset.py | 14 +++---- tests/test_dfs0.py | 6 +-- tests/test_dfsu_layered.py | 6 +-- tests/test_dfsu_plot.py | 80 ++++++++++++++---------------------- tests/test_mesh.py | 6 +-- 16 files changed, 111 insertions(+), 135 deletions(-) diff --git a/mikeio/__init__.py b/mikeio/__init__.py index de29f3fb6..1756484ec 100644 --- a/mikeio/__init__.py +++ b/mikeio/__init__.py @@ -1,4 +1,4 @@ -import os +from pathlib import Path from platform import architecture # PEP0440 compatible formatted version, see: @@ -117,7 +117,7 @@ def read(filename, *, items=None, time=None, keepdims=False, **kwargs) -> Datase >>> ds = mikeio.read("HD2D.dfsu", error_bad_data=False, fill_bad_data_value=0.0) # replace corrupt data with 0.0 """ - ext = os.path.splitext(filename)[1].lower() + ext = Path(filename).suffix.lower() if "dfs" not in ext: raise ValueError("mikeio.read() is only supported for Dfs files") @@ -156,7 +156,7 @@ def open(filename: str, **kwargs): >>> dfs = mikeio.open("pt_spectra.dfs2", type="spectral") """ - file_format = os.path.splitext(filename)[1].lower()[1:] + ext = Path(filename).suffix.lower()[1:] READERS = { "dfs0": Dfs0, @@ -167,13 +167,13 @@ def open(filename: str, **kwargs): "mesh": Mesh, } - if file_format not in READERS: + if ext not in READERS: valid_formats = ", ".join(READERS.keys()) raise Exception( - f"{file_format} is not a supported format for mikeio.open. Valid formats are {valid_formats}" + f"{ext} is not a supported format for mikeio.open. Valid formats are {valid_formats}" ) - reader_klass = READERS[file_format] + reader_klass = READERS[ext] return reader_klass(filename, **kwargs) diff --git a/mikeio/_track.py b/mikeio/_track.py index 345827809..4612c0e53 100644 --- a/mikeio/_track.py +++ b/mikeio/_track.py @@ -1,5 +1,5 @@ from __future__ import annotations -import os +from pathlib import Path from datetime import datetime from typing import Callable, Sequence, Tuple @@ -36,20 +36,22 @@ def _extract_track( if isinstance(track, str): filename = track - if os.path.exists(filename): - ext = os.path.splitext(filename)[1].lower() - if ext == ".dfs0": - df = Dfs0(filename).to_dataframe() - elif ext == ".csv": - df = pd.read_csv(filename, index_col=0, parse_dates=True) - df.index = pd.DatetimeIndex(df.index) - else: - raise ValueError(f"{ext} files not supported (dfs0, csv)") - - times = df.index - coords = df.iloc[:, 0:2].to_numpy(copy=True) - else: + path = Path(filename) + if not path.exists(): raise ValueError(f"{filename} does not exist") + + ext = path.suffix.lower() + if ext == ".dfs0": + df = Dfs0(filename).to_dataframe() + elif ext == ".csv": + df = pd.read_csv(filename, index_col=0, parse_dates=True) + df.index = pd.DatetimeIndex(df.index) + else: + raise ValueError(f"{ext} files not supported (dfs0, csv)") + + times = df.index + coords = df.iloc[:, 0:2].to_numpy(copy=True) + elif isinstance(track, Dataset): times = track.time coords = np.zeros(shape=(len(times), 2)) diff --git a/mikeio/dataset/_dataarray.py b/mikeio/dataset/_dataarray.py index b3c35a047..76d65e704 100644 --- a/mikeio/dataset/_dataarray.py +++ b/mikeio/dataset/_dataarray.py @@ -291,7 +291,8 @@ def _is_compatible(self, other, raise_error=False): problems.append("Number of timesteps must be the same") if self.start_time != other.start_time: problems.append("start_time must be the same") - if type(self.geometry) != type(other.geometry): + #if type(self.geometry) != type(other.geometry): + if not isinstance(self.geometry, other.geometry.__class__): problems.append("The type of geometry must be the same") if hasattr(self.geometry, "__eq__"): if not (self.geometry == self.geometry): diff --git a/mikeio/dataset/_dataset.py b/mikeio/dataset/_dataset.py index a55776616..c4d5fdf2f 100644 --- a/mikeio/dataset/_dataset.py +++ b/mikeio/dataset/_dataset.py @@ -1,5 +1,5 @@ from __future__ import annotations -import os +from pathlib import Path import warnings from copy import deepcopy from datetime import datetime @@ -1838,7 +1838,8 @@ def to_dfs(self, filename, **kwargs): @staticmethod def _validate_extension(filename, valid_extension): - ext = os.path.splitext(filename)[1].lower() + path = Path(filename) + ext = path.suffix.lower() if ext != valid_extension: raise ValueError(f"File extension must be {valid_extension}") diff --git a/mikeio/dfs/_dfs.py b/mikeio/dfs/_dfs.py index 85d31a108..728b59780 100644 --- a/mikeio/dfs/_dfs.py +++ b/mikeio/dfs/_dfs.py @@ -281,7 +281,7 @@ class _Dfs123: # TODO add all common arguments def __init__(self, filename=None): - self._filename = str(filename) + self._filename = str(filename) if filename else None self._projstr = None self._start_time = None self._end_time = None diff --git a/mikeio/dfs/_dfs0.py b/mikeio/dfs/_dfs0.py index 23567d032..ee21baddf 100644 --- a/mikeio/dfs/_dfs0.py +++ b/mikeio/dfs/_dfs0.py @@ -1,4 +1,4 @@ -import os +from pathlib import Path import warnings from datetime import datetime, timedelta @@ -104,12 +104,13 @@ def __init__(self, filename=None): self._filename = str(filename) if filename: - self._read_header() + self._read_header(Path(filename)) def __repr__(self): out = [""] - if os.path.isfile(self._filename): + # TODO does this make sense: + if self._filename: out.append(f"timeaxis: {repr(self._timeaxistype)}") if self._n_items is not None: @@ -122,11 +123,11 @@ def __repr__(self): return str.join("\n", out) - def _read_header(self): - if not os.path.exists(self._filename): - raise FileNotFoundError(self._filename) + def _read_header(self, path: Path): + if not path.exists(): + raise FileNotFoundError(path) - dfs = DfsFileFactory.DfsGenericOpen(self._filename) + dfs = DfsFileFactory.DfsGenericOpen(str(path)) self._source = dfs self._deletevalue = dfs.FileInfo.DeleteValueDouble # NOTE: changed in cutil @@ -164,10 +165,10 @@ def read(self, items=None, time=None, keepdims=False) -> Dataset: ------- Dataset A Dataset with data dimensions [t] - """ - - if not os.path.exists(self._filename): - raise FileNotFoundError(f"File {self._filename} not found.") + """ + path = Path(self._filename) + if not path.exists(): + raise FileNotFoundError(f"File {path} not found") # read data from file fdata, ftime, fitems = self.__read(self._filename) diff --git a/mikeio/dfs/_dfs1.py b/mikeio/dfs/_dfs1.py index 1b3aec69a..3a8ac65d8 100644 --- a/mikeio/dfs/_dfs1.py +++ b/mikeio/dfs/_dfs1.py @@ -1,4 +1,4 @@ -import os +from pathlib import Path from mikecore.DfsBuilder import DfsBuilder # type: ignore from mikecore.DfsFileFactory import DfsFileFactory # type: ignore @@ -20,7 +20,7 @@ def __init__(self, filename=None): self._x0 = 0 if filename: - self._read_dfs1_header() + self._read_dfs1_header(path=Path(filename)) origin = self._longitude, self._latitude self.geometry = Grid1D( x0=self._x0, @@ -34,7 +34,8 @@ def __init__(self, filename=None): def __repr__(self): out = [""] - if os.path.isfile(self._filename): + # TODO does this make sense + if self._filename: out.append(f"dx: {self.dx:.5f}") if self._n_items is not None: @@ -44,20 +45,20 @@ def __repr__(self): out.append(f" {i}: {item}") else: out.append(f"number of items: {self._n_items}") - if os.path.isfile(self._filename): - if self._n_timesteps == 1: - out.append("time: time-invariant file (1 step)") - else: - out.append(f"time: {self._n_timesteps} steps") - out.append(f"start time: {self._start_time}") + + if self._n_timesteps == 1: + out.append("time: time-invariant file (1 step)") + else: + out.append(f"time: {self._n_timesteps} steps") + out.append(f"start time: {self._start_time}") return str.join("\n", out) - def _read_dfs1_header(self): - if not os.path.isfile(self._filename): - raise FileNotFoundError(self._filename) + def _read_dfs1_header(self, path: Path): + if not path.exists(): + raise FileNotFoundError(path) - self._dfs = DfsFileFactory.Dfs1FileOpen(self._filename) + self._dfs = DfsFileFactory.Dfs1FileOpen(str(path)) self._x0 = self._dfs.SpatialAxis.X0 self._dx = self._dfs.SpatialAxis.Dx self._nx = self._dfs.SpatialAxis.XCount diff --git a/mikeio/dfs/_dfs2.py b/mikeio/dfs/_dfs2.py index 0a7f773d8..92f5c3a04 100644 --- a/mikeio/dfs/_dfs2.py +++ b/mikeio/dfs/_dfs2.py @@ -1,4 +1,3 @@ -import os from typing import List, Tuple from copy import deepcopy @@ -120,7 +119,7 @@ def __init__(self, filename=None, type: str = "horizontal"): if filename: is_spectral = type.lower() in ["spectral", "spectra", "spectrum"] - self._read_dfs2_header(read_x0y0=is_spectral) + self._read_dfs2_header(filename=filename, read_x0y0=is_spectral) self._validate_no_orientation_in_geo() origin, orientation = self._origin_and_orientation_in_CRS() @@ -140,7 +139,7 @@ def __init__(self, filename=None, type: str = "horizontal"): def __repr__(self): out = [""] - if os.path.isfile(self._filename): + if self._filename: out.append(f"dx: {self.dx:.5f}") out.append(f"dy: {self.dy:.5f}") @@ -160,11 +159,8 @@ def __repr__(self): return str.join("\n", out) - def _read_dfs2_header(self, read_x0y0: bool = False): - if not os.path.isfile(self._filename): - raise Exception(f"file {self._filename} does not exist!") - - self._dfs = DfsFileFactory.Dfs2FileOpen(self._filename) + def _read_dfs2_header(self, filename, read_x0y0: bool = False): + self._dfs = DfsFileFactory.Dfs2FileOpen(str(filename)) self._source = self._dfs if read_x0y0: self._x0 = self._dfs.SpatialAxis.X0 diff --git a/mikeio/dfs/_dfs3.py b/mikeio/dfs/_dfs3.py index 00d7d0601..01693200a 100644 --- a/mikeio/dfs/_dfs3.py +++ b/mikeio/dfs/_dfs3.py @@ -1,5 +1,3 @@ -import os - from typing import Tuple import numpy as np @@ -142,7 +140,8 @@ def __init__(self, filename=None): def __repr__(self): out = [""] - if os.path.isfile(self._filename): + + if self._filename: out.append(f"geometry: {self.geometry}") if self._n_items is not None: @@ -162,9 +161,6 @@ def __repr__(self): return str.join("\n", out) def _read_dfs3_header(self, read_x0y0z0: bool = False): - if not os.path.isfile(self._filename): - raise Exception(f"file {self._filename} does not exist!") - self._dfs = DfsFileFactory.Dfs3FileOpen(self._filename) self._source = self._dfs diff --git a/mikeio/dfsu/_dfsu.py b/mikeio/dfsu/_dfsu.py index d4a389a9f..1efedad0e 100644 --- a/mikeio/dfsu/_dfsu.py +++ b/mikeio/dfsu/_dfsu.py @@ -1,5 +1,6 @@ from __future__ import annotations import os +from pathlib import Path import warnings from datetime import datetime, timedelta from functools import wraps @@ -175,10 +176,11 @@ def _read_header(self, input): return filename = input - if not os.path.isfile(filename): - raise Exception(f"file {filename} does not exist!") + path = Path(input) + if not path.exists(): + raise FileNotFoundError(f"file {path} does not exist!") - ext = os.path.splitext(filename)[1].lower() + ext = path.suffix.lower() if ext == ".mesh": self._read_mesh_header(filename) diff --git a/mikeio/dfsu/_factory.py b/mikeio/dfsu/_factory.py index daae36d31..1e6d7d074 100644 --- a/mikeio/dfsu/_factory.py +++ b/mikeio/dfsu/_factory.py @@ -1,4 +1,4 @@ -import os +from pathlib import Path from mikecore.DfsuFile import DfsuFile, DfsuFileType # type: ignore @@ -25,7 +25,7 @@ def __new__(self, filename, *args, **kwargs): @staticmethod def _get_DfsuFileType_n_Obj(filename: str): - ext = os.path.splitext(filename)[-1].lower() + ext = Path(filename).suffix.lower() if "dfs" in ext: dfs = DfsuFile.Open(filename) type = DfsuFileType(dfs.DfsuFileType) diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 24d8e8c2b..353efcf09 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -1,4 +1,4 @@ -import os +from pathlib import Path from datetime import datetime import numpy as np import pandas as pd @@ -840,19 +840,19 @@ def test_to_dfs_extension_validation(tmp_path): assert "dfsu" in str(excinfo.value) -def test_weighted_average(tmp_path): +def test_weighted_average(tmp_path: Path): # TODO move to integration tests - filename = "tests/testdata/HD2D.dfsu" - dfs = mikeio.open(filename) + fp = Path("tests/testdata/HD2D.dfsu") + dfs = mikeio.open(fp) ds = dfs.read(items=["Surface elevation", "Current speed"]) area = dfs.get_element_area() ds2 = ds.average(weights=area, axis=1) - outfilename = tmp_path / "average.dfs0" - ds2.to_dfs(outfilename) - assert os.path.isfile(outfilename) + out_path = tmp_path / "average.dfs0" + ds2.to_dfs(out_path) + assert out_path.exists def test_quantile_axis1(ds1): diff --git a/tests/test_dfs0.py b/tests/test_dfs0.py index 17e8aa47b..4c969773c 100644 --- a/tests/test_dfs0.py +++ b/tests/test_dfs0.py @@ -10,8 +10,7 @@ def test_repr(): - filename = os.path.join("tests", "testdata", "da_diagnostic.dfs0") - dfs = Dfs0(filename) + dfs = Dfs0("tests/testdata/da_diagnostic.dfs0") text = repr(dfs) @@ -19,8 +18,7 @@ def test_repr(): def test_repr_equidistant(): - filename = os.path.join("tests", "testdata", "random.dfs0") - dfs = Dfs0(filename) + dfs = Dfs0("tests/testdata/random.dfs0") text = repr(dfs) diff --git a/tests/test_dfsu_layered.py b/tests/test_dfsu_layered.py index ee7db254a..61c85f8f8 100644 --- a/tests/test_dfsu_layered.py +++ b/tests/test_dfsu_layered.py @@ -506,7 +506,7 @@ def test_write_from_dfsu3D(tmp_path): dfs.write(fp, ds) - assert os.path.exists(fp) + assert fp.exists() def test_extract_top_layer_to_2d(tmp_path): @@ -549,12 +549,12 @@ def test_to_mesh_3d(tmp_path): fp = tmp_path / "oresund_from_dfs.mesh" dfs.to_mesh(fp) - assert os.path.exists(fp) + assert fp.exists() Mesh(fp) fp = tmp_path / "oresund_from_geometry.mesh" dfs.geometry.to_mesh(fp) - assert os.path.exists(fp) + assert fp.exists() Mesh(fp) diff --git a/tests/test_dfsu_plot.py b/tests/test_dfsu_plot.py index c34f54ca7..c395177a7 100644 --- a/tests/test_dfsu_plot.py +++ b/tests/test_dfsu_plot.py @@ -18,8 +18,12 @@ pytest.importorskip("matplotlib") +@pytest.fixture +def hd2d_dfs(): + return mikeio.open("tests/testdata/HD2D.dfsu") + def test_plot_bathymetry(): - filename = os.path.join("tests", "testdata", "oresund_sigma_z.dfsu") + filename = "tests/testdata/oresund_sigma_z.dfsu" dfs = mikeio.open(filename) with pytest.warns(FutureWarning): dfs.plot() @@ -27,40 +31,37 @@ def test_plot_bathymetry(): def test_plot_bathymetry_no_colorbar(): - filename = os.path.join("tests", "testdata", "oresund_sigma_z.dfsu") + filename = "tests/testdata/oresund_sigma_z.dfsu" dfs = mikeio.open(filename) with pytest.warns(FutureWarning): dfs.plot(add_colorbar=False) assert True -def test_plot_2d(): - filename = os.path.join("tests", "testdata", "HD2D.dfsu") - dfs = mikeio.open(filename) +def test_plot_2d(hd2d_dfs): + dfs = hd2d_dfs with pytest.warns(FutureWarning): dfs.plot(cmap="plasma") assert True def test_plot_3d(): - filename = os.path.join("tests", "testdata", "oresund_sigma_z.dfsu") + filename = "tests/testdata/oresund_sigma_z.dfsu" dfs = mikeio.open(filename) with pytest.warns(FutureWarning): dfs.plot() assert True -def test_plot_dfsu_contour(): - filename = os.path.join("tests", "testdata", "HD2D.dfsu") - dfs = mikeio.open(filename) +def test_plot_dfsu_contour(hd2d_dfs): + dfs = hd2d_dfs with pytest.warns(FutureWarning): dfs.plot(plot_type="contour", levels=5) assert True -def test_plot_dfsu_contourf_levels(): - filename = os.path.join("tests", "testdata", "HD2D.dfsu") - dfs = mikeio.open(filename) +def test_plot_dfsu_contourf_levels(hd2d_dfs): + dfs = hd2d_dfs cmap = mpl.colors.ListedColormap(["red", "green", "blue"]) bounds = [-3, 1, 2, 100] with pytest.warns(FutureWarning): @@ -71,8 +72,7 @@ def test_plot_dfsu_contourf_levels(): def test_plot_dfsu_contour_mixedmesh(): - filename = os.path.join("tests", "testdata", "FakeLake.dfsu") - msh = Mesh(filename) + msh = Mesh("tests/testdata/FakeLake.dfsu") msh.plot(plot_type="contour", levels=5) msh.plot( plot_type="contourf", @@ -84,15 +84,13 @@ def test_plot_dfsu_contour_mixedmesh(): def test_plot_dfsu_n_refinements(): - filename = os.path.join("tests", "testdata", "FakeLake.dfsu") - msh = Mesh(filename) + msh = Mesh("tests/testdata/FakeLake.dfsu") msh.plot(plot_type="contourf", levels=None, n_refinements=1) assert True -def test_plot_dfsu_shaded(): - filename = os.path.join("tests", "testdata", "HD2D.dfsu") - dfs = mikeio.open(filename) +def test_plot_dfsu_shaded(hd2d_dfs): + dfs = hd2d_dfs da = dfs.read(items="Surface elevation", time=0)[0] elem40 = np.arange(40) @@ -103,9 +101,8 @@ def test_plot_dfsu_shaded(): with pytest.warns(FutureWarning): dfs.plot(wl_40, elements=elem40, plot_type="shaded", levels=5) -def test_plot_dfsu_contour_subset_not_allowed(): - filename = os.path.join("tests", "testdata", "HD2D.dfsu") - dfs = mikeio.open(filename) +def test_plot_dfsu_contour_subset_not_allowed(hd2d_dfs): + dfs = hd2d_dfs da = dfs.read(items="Surface elevation", time=0)[0] elem40 = np.arange(40) with pytest.raises(Exception): @@ -113,18 +110,16 @@ def test_plot_dfsu_contour_subset_not_allowed(): -def test_plot_dfsu(): - filename = os.path.join("tests", "testdata", "HD2D.dfsu") - dfs = mikeio.open(filename) +def test_plot_dfsu(hd2d_dfs): + dfs = hd2d_dfs data = dfs.read() with pytest.warns(FutureWarning): dfs.plot(z=data[1][0, :], figsize=(3, 3), plot_type="mesh_only") assert True -def test_plot_dfsu_squeeze(): - filename = os.path.join("tests", "testdata", "HD2D.dfsu") - dfs = mikeio.open(filename) +def test_plot_dfsu_squeeze(hd2d_dfs): + dfs = hd2d_dfs data = dfs.read(items=0, time=0) with pytest.warns(FutureWarning): dfs.plot(z=data) # 1 item-dataset @@ -132,8 +127,7 @@ def test_plot_dfsu_squeeze(): def test_plot_dfsu_arguments(): - filename = os.path.join("tests", "testdata", "NorthSea_HD_and_windspeed.dfsu") - dfs = mikeio.open(filename) + dfs = mikeio.open("tests/testdata/NorthSea_HD_and_windspeed.dfsu") dfs.read() with pytest.warns(FutureWarning): dfs.plot(title="test", label="test", vmin=-23, vmax=23) @@ -141,50 +135,37 @@ def test_plot_dfsu_arguments(): def test_plot_mesh(): - filename = os.path.join("tests", "testdata", "odense_rough.mesh") - msh = Mesh(filename) + msh = Mesh("tests/testdata/odense_rough.mesh") msh.plot(show_mesh=False) assert True def test_plot_mesh_outline(): - filename = os.path.join("tests", "testdata", "odense_rough.mesh") - msh = Mesh(filename) + msh = Mesh("tests/testdata/odense_rough.mesh") msh.plot(plot_type="outline_only") assert True msh.plot(plot_type=None) assert True -# TODO: no longer supported? -# def test_plot_mesh_part(): -# filename = os.path.join("tests", "testdata", "odense_rough.mesh") -# msh = Mesh(filename) -# msh.plot(elements=list(range(0, 100))) -# assert True - - def test_plot_mesh_ax(): import matplotlib.pyplot as plt - filename = os.path.join("tests", "testdata", "odense_rough.mesh") - msh = Mesh(filename) + msh = Mesh("tests/testdata/odense_rough.mesh") _, ax = plt.subplots() msh.plot(ax=ax) assert True def test_plot_mesh_boundary_nodes(): - filename = os.path.join("tests", "testdata", "odense_rough.mesh") - msh = Mesh(filename) + msh = Mesh("tests/testdata/odense_rough.mesh") msh.plot_boundary_nodes() msh.plot_boundary_nodes(["Land", "Sea"]) assert True def test_plot_invalid(): - filename = os.path.join("tests", "testdata", "odense_rough.mesh") - mesh = Mesh(filename) + msh = Mesh("tests/testdata/odense_rough.mesh") with pytest.raises(Exception): mesh.plot(plot_type="invalid") with pytest.raises(Exception): @@ -194,8 +175,7 @@ def test_plot_invalid(): def test_plot_dfsu_vertical_profile(): import matplotlib.pyplot as plt - filename = os.path.join("tests", "testdata", "oresund_vertical_slice.dfsu") - dfs = mikeio.open(filename) + dfs = mikeio.open("tests/testdata/oresund_vertical_slice.dfsu") time_step = 1 item_number = 1 data = dfs.read()[item_number].to_numpy()[time_step, :] diff --git a/tests/test_mesh.py b/tests/test_mesh.py index 2e3267038..a9c70fee7 100644 --- a/tests/test_mesh.py +++ b/tests/test_mesh.py @@ -6,14 +6,12 @@ @pytest.fixture def tri_mesh(): - filename = os.path.join("tests", "testdata", "odense_rough.mesh") - return Mesh(filename) + return Mesh("tests/testdata/odense_rough.mesh") @pytest.fixture def mixed_mesh(): - filename = os.path.join("tests", "testdata", "quad_tri.mesh") - return Mesh(filename) + return Mesh("tests/testdata/quad_tri.mesh") def test_get_number_of_elements(tri_mesh): From 60155a73ad7945ac4fa09f86991b0cc55032e528 Mon Sep 17 00:00:00 2001 From: Henrik Andersson Date: Tue, 12 Sep 2023 08:46:04 +0200 Subject: [PATCH 2/3] Fix typo --- tests/test_dfsu_plot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_dfsu_plot.py b/tests/test_dfsu_plot.py index c395177a7..08ba2d6f2 100644 --- a/tests/test_dfsu_plot.py +++ b/tests/test_dfsu_plot.py @@ -167,9 +167,9 @@ def test_plot_mesh_boundary_nodes(): def test_plot_invalid(): msh = Mesh("tests/testdata/odense_rough.mesh") with pytest.raises(Exception): - mesh.plot(plot_type="invalid") + msh.plot(plot_type="invalid") with pytest.raises(Exception): - mesh.plot(plot_type="invalid") + msh.plot(plot_type="invalid") def test_plot_dfsu_vertical_profile(): From d64b9cf7e0390bdc10f59697f85222a467fe71c4 Mon Sep 17 00:00:00 2001 From: Henrik Andersson Date: Tue, 12 Sep 2023 10:44:47 +0200 Subject: [PATCH 3/3] No more os.path --- docs/conf.py | 10 --- tests/notebooks/test_notebooks.py | 48 +++++------- tests/performance/test_performance_dfs0.py | 14 ++-- tests/test_mesh.py | 27 ++++--- tests/test_pfs.py | 90 +++++++++++----------- 5 files changed, 84 insertions(+), 105 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index d7e02e19d..93db8da90 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,16 +4,6 @@ # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - # -- Project information ----------------------------------------------------- diff --git a/tests/notebooks/test_notebooks.py b/tests/notebooks/test_notebooks.py index 2123ec85c..78c4c6db9 100644 --- a/tests/notebooks/test_notebooks.py +++ b/tests/notebooks/test_notebooks.py @@ -1,55 +1,47 @@ -import os -import subprocess +from pathlib import Path +from typing import List + import nbformat from nbconvert.preprocessors import ExecutePreprocessor from nbconvert.preprocessors import CellExecutionError -_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -PARENT_DIR = os.path.join(_TEST_DIR, "../..") -SKIP_LIST = [] - -def _process_notebook(notebook_filename, notebook_path="notebooks"): +def _process_notebook(fp: Path): """Checks if an IPython notebook runs without error from start to finish. If so, writes the notebook to HTML (with outputs) and overwrites the .ipynb file (without outputs).""" - with open(notebook_filename) as f: + with open(fp) as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(timeout=600, kernel_name="python3") try: # Check that the notebook runs - ep.preprocess(nb, {"metadata": {"path": notebook_path}}) + ep.preprocess(nb, {"metadata": {"path": "notebooks"}}) except CellExecutionError as e: - print(f"Failed executing {notebook_filename}") + print(f"Failed executing {fp}") print(e) raise - print(f"Successfully executed {notebook_filename}") + print(f"Successfully executed {fp}") return -def _get_all_notebooks_in_repo(skip=[]): - """Get all files .ipynb included in the git repository""" - git_files = ( - subprocess.check_output( - "git ls-tree --full-tree --name-only -r HEAD", shell=True - ) - .decode("utf-8") - .splitlines() - ) +def _get_all_notebooks_in_repo() -> List[Path]: + ROOT_DIR = Path(__file__).parent.parent.parent + NOTEBOOK_DIR = ROOT_DIR / "notebooks" - return [ - fn - for fn in git_files - if fn.endswith(".ipynb") and not any(s in fn for s in skip) - ] + return list(NOTEBOOK_DIR.glob("*.ipynb")) def test_notebook(notebook): - _process_notebook(os.path.join(PARENT_DIR, notebook)) - + _process_notebook(notebook) def pytest_generate_tests(metafunc): - notebooks = _get_all_notebooks_in_repo(skip=SKIP_LIST) + notebooks = _get_all_notebooks_in_repo() metafunc.parametrize("notebook", notebooks) + + +if __name__ == "__main__": + notebooks = _get_all_notebooks_in_repo() + for notebook in notebooks: + print(notebook) \ No newline at end of file diff --git a/tests/performance/test_performance_dfs0.py b/tests/performance/test_performance_dfs0.py index 526451c30..91c83d6d6 100644 --- a/tests/performance/test_performance_dfs0.py +++ b/tests/performance/test_performance_dfs0.py @@ -1,12 +1,12 @@ -import os +from pathlib import Path import numpy as np import pandas as pd import mikeio -def test_write_read_long_dfs0(tmpdir): +def test_write_read_long_dfs0(tmp_path): - filename = os.path.join(tmpdir.dirname, "big.dfs0") + filename = tmp_path / "big.dfs0" nt = 10_000_000 data = np.random.random([nt]) @@ -15,16 +15,16 @@ def test_write_read_long_dfs0(tmpdir): ) da.to_dfs(filename) - assert os.path.exists(filename) + assert filename.exists() ds = mikeio.read(filename) assert len(ds.time) == nt -def test_write_read_many_items_dataset_pandas(tmpdir): +def test_write_read_many_items_dataset_pandas(tmp_path): - filename = os.path.join(tmpdir.dirname, "many_items.dfs0") + filename = tmp_path / "many_items.dfs0" n_items = 10_000 nt = 200 @@ -38,7 +38,7 @@ def test_write_read_many_items_dataset_pandas(tmpdir): ds = mikeio.Dataset(das) ds.to_dfs(filename) - assert os.path.exists(filename) + assert filename.exists() # read to dataset ds = mikeio.read(filename) diff --git a/tests/test_mesh.py b/tests/test_mesh.py index a9c70fee7..e2303a3cd 100644 --- a/tests/test_mesh.py +++ b/tests/test_mesh.py @@ -40,14 +40,14 @@ def test_read_mixed_mesh(mixed_mesh): assert np.all(el_tbl_vec < msh.n_nodes) -def test_read_write_mixed_mesh(mixed_mesh, tmpdir): +def test_read_write_mixed_mesh(mixed_mesh, tmp_path): msh = mixed_mesh - outfilename = os.path.join(tmpdir.dirname, "quad_tri_v2.mesh") + outfilename = tmp_path / "quad_tri_v2.mesh" msh.write(outfilename) msh2 = Mesh(outfilename) - assert os.path.exists(outfilename) + assert outfilename.exists() assert np.all(np.hstack(msh2.element_table) == np.hstack(msh.element_table)) assert np.all(msh2.element_coordinates == msh.element_coordinates) @@ -78,7 +78,6 @@ def test_get_bad_node_coordinates(tri_mesh): def test_set_z(tri_mesh): - os.path.join("tests", "testdata", "odense_rough.mesh") msh = tri_mesh zn = msh.node_coordinates[:, 2] zn[zn < -3] = -3 @@ -107,28 +106,28 @@ def test_set_codes(tri_mesh): msh.geometry.codes = codes[0:4] -def test_write(tri_mesh, tmpdir): - outfilename = os.path.join(tmpdir.dirname, "simple.mesh") +def test_write(tri_mesh, tmp_path): + outfilename = tmp_path / "simple.mesh" msh = tri_mesh msh.write(outfilename) - assert os.path.exists(outfilename) + assert outfilename.exists() -def test_write_part(tri_mesh, tmpdir): - outfilename = os.path.join(tmpdir.dirname, "simple_sub.mesh") +def test_write_part(tri_mesh, tmp_path): + outfilename = tmp_path / "simple_sub.mesh" msh = tri_mesh msh.write(outfilename, elements=list(range(0, 100))) - assert os.path.exists(outfilename) + assert outfilename.exists() -def test_write_mesh_from_dfsu(tmpdir): - outfilename = os.path.join(tmpdir.dirname, "quad_tri.mesh") - dfsufilename = os.path.join("tests", "testdata", "FakeLake.dfsu") +def test_write_mesh_from_dfsu(tmp_path): + outfilename = tmp_path / "quad_tri.mesh" + dfsufilename = "tests/testdata/FakeLake.dfsu" msh = Mesh(dfsufilename) @@ -136,6 +135,6 @@ def test_write_mesh_from_dfsu(tmpdir): msh2 = Mesh(outfilename) - assert os.path.exists(outfilename) + assert outfilename.exists() assert np.all(np.hstack(msh2.element_table) == np.hstack(msh.element_table)) diff --git a/tests/test_pfs.py b/tests/test_pfs.py index 881172c7d..b3cca537d 100644 --- a/tests/test_pfs.py +++ b/tests/test_pfs.py @@ -250,10 +250,10 @@ def test_pfssection_find_replace(d1): assert sct.FILE_6.lst == [0.11, 0.22, 0.33] -def test_pfssection_write(d1, tmpdir): +def test_pfssection_write(d1, tmp_path): sct = mikeio.PfsSection(d1) pfs = mikeio.PfsDocument({"root": sct}) - fn = os.path.join(tmpdir.dirname, "pfssection.pfs") + fn = tmp_path / "pfssection.pfs" pfs.write(fn) pfs2 = mikeio.PfsDocument(fn) @@ -328,53 +328,51 @@ def assert_txt_files_match(f1, f2, comment="//") -> None: assert s1 == s2 -def test_read_write(tmpdir): +def test_read_write(tmp_path): infilename = "tests/testdata/pfs/concat.mzt" pfs1 = mikeio.PfsDocument(infilename) - outfilename = os.path.join(tmpdir.dirname, "concat_out.mzt") + outfilename = tmp_path / "concat_out.mzt" pfs1.write(outfilename) assert_txt_files_match(infilename, outfilename) _ = mikeio.PfsDocument(outfilename) # try to parse it also -def test_read_write_she(tmpdir): +def test_read_write_she(tmp_path): infilename = "tests/testdata/pfs/Karup_basic.she" pfs1 = mikeio.PfsDocument(infilename, unique_keywords=False) - outfilename = os.path.join(tmpdir.dirname, "Karup_basic_out.she") + outfilename = tmp_path / "Karup_basic_out.she" pfs1.write(outfilename) - # assert_txt_files_match(infilename, outfilename) pfs2 = mikeio.PfsDocument(outfilename) assert pfs1.MIKESHE_FLOWMODEL == pfs2.MIKESHE_FLOWMODEL -def test_read_write_she2(tmpdir): +def test_read_write_she2(tmp_path): infilename = "tests/testdata/pfs/Karup_mini.she" with pytest.warns(match="contains a single quote character"): pfs1 = mikeio.PfsDocument(infilename) - outfilename = os.path.join(tmpdir.dirname, "Karup_mini_out.she") + outfilename = tmp_path / "Karup_mini_out.she" pfs1.write(outfilename) with pytest.warns(match="contains a single quote character"): pfs2 = mikeio.PfsDocument(outfilename) assert pfs1.MIKESHE_FLOWMODEL == pfs2.MIKESHE_FLOWMODEL - -def test_read_write_filenames(tmpdir): +def test_read_write_filenames(tmp_path): infilename = "tests/testdata/pfs/filenames.pfs" pfs1 = mikeio.PfsDocument(infilename) - outfilename = os.path.join(tmpdir.dirname, "filenames_out.pfs") + outfilename = tmp_path / "filenames_out.pfs" pfs1.write(outfilename) assert_txt_files_match(infilename, outfilename) _ = mikeio.PfsDocument(outfilename) # try to parse it also -def test_read_write_filenames_modified(tmpdir): +def test_read_write_filenames_modified(tmp_path): infilename = "tests/testdata/pfs/filenames.pfs" pfs1 = mikeio.PfsDocument(infilename) pfs1.FILE_NAMES.file5 = r"..\..\newfile5.dfs0" pfs1.FILE_NAMES.file6 = "|../newfile6.dfs0|" - outfilename = os.path.join(tmpdir.dirname, "filenames_out.pfs") + outfilename = tmp_path / "filenames_out.pfs" pfs1.write(outfilename) pfs2 = mikeio.PfsDocument(outfilename) @@ -517,11 +515,11 @@ def test_non_unique_keywords_allowed(): assert pfs.BoundaryExtractor.z_min == [-3000, 9, 19] -def test_non_unique_keywords_read_write(tmpdir): +def test_non_unique_keywords_read_write(tmp_path): fn1 = "tests/testdata/pfs/nonunique.pfs" pfs1 = mikeio.PfsDocument(fn1, unique_keywords=False) - fn2 = os.path.join(tmpdir.dirname, "nonunique_out.pfs") + fn2 = tmp_path / "nonunique_out.pfs" pfs1.write(fn2) pfs2 = mikeio.PfsDocument(fn2, unique_keywords=False) @@ -626,7 +624,7 @@ def test_read_array(): assert pfs.ENGINE.fill_list[1] == 2 -def test_empty(tmpdir): +def test_empty(tmp_path): text = """ [ENGINE] @@ -642,7 +640,7 @@ def test_empty(tmpdir): assert isinstance(pfs.ENGINE.B, mikeio.PfsSection) assert len(pfs.ENGINE.B) == 0 - outfile = os.path.join(tmpdir, "empty.pfs") + outfile = tmp_path / "empty.pfs" pfs.write(outfile) with open(outfile) as f: @@ -653,7 +651,7 @@ def test_empty(tmpdir): assert outlines[7].strip() == "EndSect // B" -def test_difficult_chars_in_str(tmpdir): +def test_difficult_chars_in_str(tmp_path): text = """ [ENGINE] @@ -678,7 +676,7 @@ def test_difficult_chars_in_str(tmpdir): assert isinstance(pfs.ENGINE.D, str) assert pfs.ENGINE.E == "|str,s\U0001F600+-s_d.dfs0|" - outfile = os.path.join(tmpdir, "difficult_chars_in_str.pfs") + outfile = tmp_path / "difficult_chars_in_str.pfs" pfs.write(outfile) with open(outfile) as f: @@ -691,7 +689,7 @@ def test_difficult_chars_in_str(tmpdir): assert outlines[9].strip() == "E = |str,s'+-s_d.dfs0|" -def test_difficult_chars_in_str2(tmpdir): +def test_difficult_chars_in_str2(tmp_path): text = """ [ENGINE] @@ -704,7 +702,7 @@ def test_difficult_chars_in_str2(tmpdir): with pytest.warns(match="contains a single quote character"): pfs = mikeio.PfsDocument(StringIO(text)) - outfile = os.path.join(tmpdir, "difficult_chars_in_str2.pfs") + outfile = tmp_path / "difficult_chars_in_str2.pfs" pfs.write(outfile) with open(outfile) as f: @@ -744,7 +742,7 @@ def test_read_string_array(): assert pfs.ENGINE.fill_list[2] == "baz" -def test_read_write_list_list(tmpdir): +def test_read_write_list_list(tmp_path): text = """ [ENGINE] RGB_Color_Value = 128, 0, 128 @@ -755,7 +753,7 @@ def test_read_write_list_list(tmpdir): assert len(pfs.ENGINE.RGB_Color_Value) == 2 assert len(pfs.ENGINE.RGB_Color_Value[0]) == 3 - outfile = os.path.join(tmpdir, "mini.pal") + outfile = tmp_path / "mini.pal" pfs.write(outfile) @@ -780,7 +778,7 @@ def test_pfs_repr_contains_name_of_target(): assert "ENGINE" in text -def test_double_single_quotes_in_string(tmpdir): +def test_double_single_quotes_in_string(tmp_path): text = """ [DERIVED_VARIABLE_106] name = 'alfa_PC_T' @@ -800,7 +798,7 @@ def test_double_single_quotes_in_string(tmpdir): == 'alfa_PC_T, "light" adjusted alfa_PC, ugC/gC*m2/uE' ) - filename = os.path.join(tmpdir, "quotes.pfs") + filename = tmp_path / "quotes.pfs" pfs.write(filename) @@ -813,7 +811,7 @@ def test_double_single_quotes_in_string(tmpdir): ) -def test_str_in_str_projection(tmpdir): +def test_str_in_str_projection(tmp_path): text = """ [ROOT] Proj = 'PROJCS["ETRS_1989",GEOGCS["GCS_1989",DATUM["D_ETRS_1"]]]' @@ -824,7 +822,7 @@ def test_str_in_str_projection(tmpdir): assert isinstance(pfs.ROOT.Proj, str) assert pfs.ROOT.Proj[8] == "E" - filename = os.path.join(tmpdir, "str_in_str.pfs") + filename = tmp_path / "str_in_str.pfs" pfs.write(filename) with open(filename) as f: @@ -836,7 +834,7 @@ def test_str_in_str_projection(tmpdir): ) -def test_number_in_str(tmpdir): +def test_number_in_str(tmp_path): text = """ [ROOT] ID1 = '1' @@ -850,7 +848,7 @@ def test_number_in_str(tmpdir): assert isinstance(pfs.ROOT.ID2, str) assert not isinstance(pfs.ROOT.Number, str) - filename = os.path.join(tmpdir, "number_in_str.pfs") + filename = tmp_path / "number_in_str.pfs" pfs.write(filename) with open(filename) as f: @@ -861,7 +859,7 @@ def test_number_in_str(tmpdir): assert line.strip() == "ID2 = '1'" -def test_floatlike_strings(tmpdir): +def test_floatlike_strings(tmp_path): text = """ [WELLNO_424] ID_A = '1E-3' @@ -874,7 +872,7 @@ def test_floatlike_strings(tmpdir): assert pfs.WELLNO_424.ID_B == "1-E" assert pfs.WELLNO_424.ID_C == "1-E3" - filename = os.path.join(tmpdir, "float_like_strings.pfs") + filename = tmp_path / "float_like_strings.pfs" pfs.write(filename) pfs = mikeio.read_pfs(filename) # assert pfs.WELLNO_424.ID_A == "1E-3" # not possible to distinguish @@ -882,7 +880,7 @@ def test_floatlike_strings(tmpdir): assert pfs.WELLNO_424.ID_C == "1-E3" -def test_nested_quotes(tmpdir): +def test_nested_quotes(tmp_path): text = """ [Weir_0] Properties = '' @@ -894,7 +892,7 @@ def test_nested_quotes(tmpdir): == '' ) - filename = os.path.join(tmpdir, "nested_quotes.pfs") + filename = tmp_path / "nested_quotes.pfs" pfs.write(filename) with open(filename) as f: @@ -906,7 +904,7 @@ def test_nested_quotes(tmpdir): ) -def test_filename_in_list(tmpdir): +def test_filename_in_list(tmp_path): text = """ [EcolabTemplateSpecification] TemplateFile_A = |.\Test1_OLSZ_OL_WQsetups.ecolab| @@ -921,7 +919,7 @@ def test_filename_in_list(tmpdir): == r"|.\Test1_OLSZ_OL_WQsetups.ecolab|" ) - filename = os.path.join(tmpdir, "filename_in_list.pfs") + filename = tmp_path / "filename_in_list.pfs" pfs.write(filename) with open(filename) as f: @@ -933,7 +931,7 @@ def test_filename_in_list(tmpdir): ) -def test_multiple_empty_strings_in_list(tmpdir): +def test_multiple_empty_strings_in_list(tmp_path): text = """ [Engine] A = '', '', '', '' @@ -950,7 +948,7 @@ def test_multiple_empty_strings_in_list(tmpdir): assert pfs.Engine.B[2] == "||" assert pfs.Engine.B[-1] == "" - filename = os.path.join(tmpdir, "multiple_empty_strings_in_list.pfs") + filename = tmp_path / "multiple_empty_strings_in_list.pfs" pfs.write(filename) with open(filename) as f: @@ -961,7 +959,7 @@ def test_multiple_empty_strings_in_list(tmpdir): assert line.strip() == "B = '', '', ||, ||, '', ''" -def test_vertical_lines_in_list(tmpdir): +def test_vertical_lines_in_list(tmp_path): text = """ [EcolabTemplateSpecification] TemplateFile_OL = ||, -1, -1, ||, -1, -1, || @@ -975,7 +973,7 @@ def test_vertical_lines_in_list(tmpdir): assert pfs.EcolabTemplateSpecification.TemplateFile_OL[3] == "||" assert pfs.EcolabTemplateSpecification.TemplateFile_OL[-1] == "||" - filename = os.path.join(tmpdir, "vertical_lines_in_list.pfs") + filename = tmp_path / "vertical_lines_in_list.pfs" pfs.write(filename) with open(filename) as f: @@ -984,7 +982,7 @@ def test_vertical_lines_in_list(tmpdir): assert line.strip() == "TemplateFile_OL = ||, -1, -1, ||, -1, -1, ||" -def test_nonunique_mixed_keywords_sections1(tmpdir): +def test_nonunique_mixed_keywords_sections1(tmp_path): text = """ [ROOT] A = '1' @@ -1004,7 +1002,7 @@ def test_nonunique_mixed_keywords_sections1(tmpdir): assert pfs.ROOT.A[2].B[0] == 0 assert pfs.ROOT.A[-1] == 3 - filename = os.path.join(tmpdir, "nonunique_mixed_keywords_sections.pfs") + filename = tmp_path / "nonunique_mixed_keywords_sections.pfs" pfs.write(filename) with open(filename) as f: @@ -1017,7 +1015,7 @@ def test_nonunique_mixed_keywords_sections1(tmpdir): assert outlines[9].strip() == "B = 2" -def test_nonunique_mixed_keywords_sections2(tmpdir): +def test_nonunique_mixed_keywords_sections2(tmp_path): text = """ [ROOT] [A] @@ -1043,7 +1041,7 @@ def test_nonunique_mixed_keywords_sections2(tmpdir): assert pfs.ROOT.A[2].B == 0 assert pfs.ROOT.A[-1] == 3 - filename = os.path.join(tmpdir, "nonunique_mixed_keywords_sections.pfs") + filename = tmp_path / "nonunique_mixed_keywords_sections.pfs" pfs.write(filename) @@ -1057,9 +1055,9 @@ def test_parse_mike_she_pfs(): ) # TODO Is this sensible to check? -def test_read_write_grid_editor_color_palette(tmpdir): +def test_read_write_grid_editor_color_palette(tmp_path): infile = "tests/testdata/pfs/grid1.gsf" - outfile = os.path.join(tmpdir, "grid.gsf") + outfile = tmp_path / "grid.gsf" pfs = mikeio.PfsDocument(infile) pal = pfs.GRID_EDITOR.GRID_EDIT_VIEW.MIKEZero_Palette_Definition assert len(pal.RGB_Color_Value) == 16