From 67eeba7e1448e1fd9d43b7fc22fec84d5d6f1706 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sat, 27 Jan 2024 13:51:01 +0100 Subject: [PATCH 001/186] TYP: upgrade mypy and type-check extras --- pyproject.toml | 8 ++++---- yt/frontends/rockstar/definitions.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d048bf9ff2..d365105185 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -225,10 +225,10 @@ test = [ "nose-timer~=1.0.0; python_version < '3.10'", ] typecheck = [ - "mypy==1.5.1", - "types-PyYAML==6.0.12.2", - "types-chardet==5.0.4", - "types-requests==2.28.11.5", + "mypy==1.8.0", + "types-PyYAML==6.0.12.12", + "types-chardet==5.0.4.6", + "types-requests==2.31.0.20240125", "typing-extensions==4.4.0; python_version < '3.12'", ] diff --git a/yt/frontends/rockstar/definitions.py b/yt/frontends/rockstar/definitions.py index 7e6753f830..578f29b77e 100644 --- a/yt/frontends/rockstar/definitions.py +++ b/yt/frontends/rockstar/definitions.py @@ -108,7 +108,7 @@ if len(item) == 2: halo_dts_tmp[rev].append(item) elif len(item) == 3: - mi, ma = item[2] # type: ignore + mi, ma = item[2] if (mi <= rev) and (rev <= ma): halo_dts_tmp[rev].append(item[:2]) halo_dts[rev] = np.dtype(halo_dts_tmp[rev], align=True) From 22c7e34f86766ae71da639573d53c7de8f5a9d01 Mon Sep 17 00:00:00 2001 From: biboyd Date: Mon, 15 Apr 2024 18:49:30 -0400 Subject: [PATCH 002/186] add path to reduce images without considering opacity --- yt/utilities/amr_kdtree/amr_kdtools.py | 45 ++++++++++--------- yt/utilities/amr_kdtree/amr_kdtree.py | 4 +- .../volume_rendering/render_source.py | 4 +- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/yt/utilities/amr_kdtree/amr_kdtools.py b/yt/utilities/amr_kdtree/amr_kdtools.py index f42e3a36f7..fb933291f4 100644 --- a/yt/utilities/amr_kdtree/amr_kdtools.py +++ b/yt/utilities/amr_kdtree/amr_kdtools.py @@ -3,34 +3,39 @@ from yt.funcs import mylog -def receive_and_reduce(comm, incoming_rank, image, add_to_front): +def receive_and_reduce(comm, incoming_rank, image, add_to_front, use_opacity): mylog.debug("Receiving image from %04i", incoming_rank) # mylog.debug( '%04i receiving image from %04i'%(self.comm.rank,back.owner)) arr2 = comm.recv_array(incoming_rank, incoming_rank).reshape( (image.shape[0], image.shape[1], image.shape[2]) ) - if add_to_front: - front = arr2 - back = image + if use_opacity: + if add_to_front: + front = arr2 + back = image + else: + front = image + back = arr2 + + if image.shape[2] == 3: + # Assume Projection Camera, Add + np.add(image, front, image) + return image + + ta = 1.0 - front[:, :, 3] + np.maximum(ta, 0.0, ta) + # This now does the following calculation, but in a memory + # conservative fashion + # image[:,:,i ] = front[:,:,i] + ta*back[:,:,i] + image = back.copy() + for i in range(4): + np.multiply(image[:, :, i], ta, image[:, :, i]) + np.add(image, front, image) + else: - front = image - back = arr2 + np.add(image, arr2, image) - if image.shape[2] == 3: - # Assume Projection Camera, Add - np.add(image, front, image) - return image - - ta = 1.0 - front[:, :, 3] - np.maximum(ta, 0.0, ta) - # This now does the following calculation, but in a memory - # conservative fashion - # image[:,:,i ] = front[:,:,i] + ta*back[:,:,i] - image = back.copy() - for i in range(4): - np.multiply(image[:, :, i], ta, image[:, :, i]) - np.add(image, front, image) return image diff --git a/yt/utilities/amr_kdtree/amr_kdtree.py b/yt/utilities/amr_kdtree/amr_kdtree.py index b8378c6671..527e93e3ae 100644 --- a/yt/utilities/amr_kdtree/amr_kdtree.py +++ b/yt/utilities/amr_kdtree/amr_kdtree.py @@ -294,7 +294,7 @@ def get_reduce_owners(self): owners[temp.node_id] = owners[temp.left.node_id] return owners - def reduce_tree_images(self, image, viewpoint): + def reduce_tree_images(self, image, viewpoint, use_opacity): if self.comm.size <= 1: return image myrank = self.comm.rank @@ -307,7 +307,7 @@ def reduce_tree_images(self, image, viewpoint): split_pos = node.parent.get_split_pos() add_to_front = viewpoint[split_dim] >= split_pos image = receive_and_reduce( - self.comm, owners[node.parent.right.node_id], image, add_to_front + self.comm, owners[node.parent.right.node_id], image, add_to_front, use_opacity, ) if node.parent.node_id == 1: break diff --git a/yt/visualization/volume_rendering/render_source.py b/yt/visualization/volume_rendering/render_source.py index 65d6cd2e55..42ce43c067 100644 --- a/yt/visualization/volume_rendering/render_source.py +++ b/yt/visualization/volume_rendering/render_source.py @@ -607,7 +607,9 @@ def render(self, camera, zbuffer=None): def finalize_image(self, camera, image): if self._volume is not None: - image = self.volume.reduce_tree_images(image, camera.lens.viewpoint) + image = self.volume.reduce_tree_images( + image, camera.lens.viewpoint, self.transfer_function.grey_opacity + ) return super().finalize_image(camera, image) From 0de8d6329ab640917a532f2dbfbf6a3663eb946a Mon Sep 17 00:00:00 2001 From: biboyd Date: Wed, 17 Apr 2024 12:01:36 -0400 Subject: [PATCH 003/186] make use_opacity a kwarg --- yt/utilities/amr_kdtree/amr_kdtree.py | 8 ++++++-- yt/visualization/volume_rendering/render_source.py | 6 ++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/yt/utilities/amr_kdtree/amr_kdtree.py b/yt/utilities/amr_kdtree/amr_kdtree.py index 527e93e3ae..30be6e3c86 100644 --- a/yt/utilities/amr_kdtree/amr_kdtree.py +++ b/yt/utilities/amr_kdtree/amr_kdtree.py @@ -294,7 +294,7 @@ def get_reduce_owners(self): owners[temp.node_id] = owners[temp.left.node_id] return owners - def reduce_tree_images(self, image, viewpoint, use_opacity): + def reduce_tree_images(self, image, viewpoint, use_opacity=True): if self.comm.size <= 1: return image myrank = self.comm.rank @@ -307,7 +307,11 @@ def reduce_tree_images(self, image, viewpoint, use_opacity): split_pos = node.parent.get_split_pos() add_to_front = viewpoint[split_dim] >= split_pos image = receive_and_reduce( - self.comm, owners[node.parent.right.node_id], image, add_to_front, use_opacity, + self.comm, + owners[node.parent.right.node_id], + image, + add_to_front, + use_opacity, ) if node.parent.node_id == 1: break diff --git a/yt/visualization/volume_rendering/render_source.py b/yt/visualization/volume_rendering/render_source.py index 42ce43c067..c7d0beed2e 100644 --- a/yt/visualization/volume_rendering/render_source.py +++ b/yt/visualization/volume_rendering/render_source.py @@ -608,8 +608,10 @@ def render(self, camera, zbuffer=None): def finalize_image(self, camera, image): if self._volume is not None: image = self.volume.reduce_tree_images( - image, camera.lens.viewpoint, self.transfer_function.grey_opacity - ) + image, + camera.lens.viewpoint, + use_opacity=self.transfer_function.grey_opacity, + ) return super().finalize_image(camera, image) From 10f3ef40d72c34184f0688732a43bcb1bfd70a1a Mon Sep 17 00:00:00 2001 From: Brendan Boyd <42776109+biboyd@users.noreply.github.com> Date: Wed, 17 Apr 2024 14:53:52 -0400 Subject: [PATCH 004/186] add more/force keyword arguments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Robert --- yt/utilities/amr_kdtree/amr_kdtools.py | 2 +- yt/utilities/amr_kdtree/amr_kdtree.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/yt/utilities/amr_kdtree/amr_kdtools.py b/yt/utilities/amr_kdtree/amr_kdtools.py index fb933291f4..990010c51a 100644 --- a/yt/utilities/amr_kdtree/amr_kdtools.py +++ b/yt/utilities/amr_kdtree/amr_kdtools.py @@ -3,7 +3,7 @@ from yt.funcs import mylog -def receive_and_reduce(comm, incoming_rank, image, add_to_front, use_opacity): +def receive_and_reduce(comm, incoming_rank, image, add_to_front, *, use_opacity=True): mylog.debug("Receiving image from %04i", incoming_rank) # mylog.debug( '%04i receiving image from %04i'%(self.comm.rank,back.owner)) arr2 = comm.recv_array(incoming_rank, incoming_rank).reshape( diff --git a/yt/utilities/amr_kdtree/amr_kdtree.py b/yt/utilities/amr_kdtree/amr_kdtree.py index 30be6e3c86..861f4320c1 100644 --- a/yt/utilities/amr_kdtree/amr_kdtree.py +++ b/yt/utilities/amr_kdtree/amr_kdtree.py @@ -294,7 +294,7 @@ def get_reduce_owners(self): owners[temp.node_id] = owners[temp.left.node_id] return owners - def reduce_tree_images(self, image, viewpoint, use_opacity=True): + def reduce_tree_images(self, image, viewpoint, *, use_opacity=True): if self.comm.size <= 1: return image myrank = self.comm.rank @@ -311,7 +311,7 @@ def reduce_tree_images(self, image, viewpoint, use_opacity=True): owners[node.parent.right.node_id], image, add_to_front, - use_opacity, + use_opacity=use_opacity, ) if node.parent.node_id == 1: break From a104859c9d86800af28451d46c2ea5e3c6dd8d8a Mon Sep 17 00:00:00 2001 From: biboyd Date: Wed, 17 Apr 2024 15:01:29 -0400 Subject: [PATCH 005/186] return image early --- yt/utilities/amr_kdtree/amr_kdtools.py | 46 +++++++++++++------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/yt/utilities/amr_kdtree/amr_kdtools.py b/yt/utilities/amr_kdtree/amr_kdtools.py index 990010c51a..01d7327a26 100644 --- a/yt/utilities/amr_kdtree/amr_kdtools.py +++ b/yt/utilities/amr_kdtree/amr_kdtools.py @@ -10,31 +10,31 @@ def receive_and_reduce(comm, incoming_rank, image, add_to_front, *, use_opacity= (image.shape[0], image.shape[1], image.shape[2]) ) - if use_opacity: - if add_to_front: - front = arr2 - back = image - else: - front = image - back = arr2 - - if image.shape[2] == 3: - # Assume Projection Camera, Add - np.add(image, front, image) - return image - - ta = 1.0 - front[:, :, 3] - np.maximum(ta, 0.0, ta) - # This now does the following calculation, but in a memory - # conservative fashion - # image[:,:,i ] = front[:,:,i] + ta*back[:,:,i] - image = back.copy() - for i in range(4): - np.multiply(image[:, :, i], ta, image[:, :, i]) - np.add(image, front, image) + if not use_opacity: + np.add(image, arr2, image) + return image + if add_to_front: + front = arr2 + back = image else: - np.add(image, arr2, image) + front = image + back = arr2 + + if image.shape[2] == 3: + # Assume Projection Camera, Add + np.add(image, front, image) + return image + + ta = 1.0 - front[:, :, 3] + np.maximum(ta, 0.0, ta) + # This now does the following calculation, but in a memory + # conservative fashion + # image[:,:,i ] = front[:,:,i] + ta*back[:,:,i] + image = back.copy() + for i in range(4): + np.multiply(image[:, :, i], ta, image[:, :, i]) + np.add(image, front, image) return image From c971f4ca14d8f5cebcd1cac0b415c7016be0fc9a Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Fri, 26 Apr 2024 09:51:36 -0500 Subject: [PATCH 006/186] On-demand file-handle opening. --- yt/frontends/tipsy/io.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/yt/frontends/tipsy/io.py b/yt/frontends/tipsy/io.py index da54326f87..fee0ce315a 100644 --- a/yt/frontends/tipsy/io.py +++ b/yt/frontends/tipsy/io.py @@ -157,9 +157,12 @@ def _read_particle_data_file(self, data_file, ptf, selector=None): f = open(data_file.filename, "rb") # we need to open all aux files for chunking to work - aux_fh = {} - for afield in self._aux_fields: - aux_fh[afield] = open(data_file.filename + "." + afield, "rb") + _aux_fh = {} + + def aux_fh(afield): + if afield not in _aux_fh: + _aux_fh[afield] = open(data_file.filename + "." + afield, "rb") + return _aux_fh[afield] for ptype, field_list in sorted(ptf.items(), key=lambda a: poff.get(a[0], -1)): if data_file.total_particles[ptype] == 0: @@ -170,22 +173,22 @@ def _read_particle_data_file(self, data_file, ptf, selector=None): p = np.fromfile(f, self._pdtypes[ptype], count=count) auxdata = [] for afield in afields: - aux_fh[afield].seek(aux_fields_offsets[afield][ptype]) + aux_fh(afield).seek(aux_fields_offsets[afield][ptype]) if isinstance(self._aux_pdtypes[afield], np.dtype): auxdata.append( np.fromfile( - aux_fh[afield], self._aux_pdtypes[afield], count=count + aux_fh(afield), self._aux_pdtypes[afield], count=count ) ) else: par = self.ds.parameters nlines = 1 + par["nsph"] + par["ndark"] + par["nstar"] - aux_fh[afield].seek(0) + aux_fh(afield).seek(0) sh = aux_fields_offsets[afield][ptype] sf = nlines - count - sh if tp[ptype] > 0: aux = np.genfromtxt( - aux_fh[afield], skip_header=sh, skip_footer=sf + aux_fh(afield), skip_header=sh, skip_footer=sf ) if aux.ndim < 1: aux = np.array([aux]) @@ -212,7 +215,7 @@ def _read_particle_data_file(self, data_file, ptf, selector=None): # close all file handles f.close() - for fh in list(aux_fh.values()): + for fh in list(_aux_fh.values()): fh.close() return return_data From 5c9407a17de814ab79751a34eb5642d83f58af46 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 17:09:59 +0000 Subject: [PATCH 007/186] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.5.0 → v4.6.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.5.0...v4.6.0) - [github.com/astral-sh/ruff-pre-commit: v0.4.0 → v0.4.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.0...v0.4.3) - [github.com/MarcoGorelli/cython-lint: v0.16.0 → v0.16.2](https://github.com/MarcoGorelli/cython-lint/compare/v0.16.0...v0.16.2) --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d507680f64..08cf87f28b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ ci: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -36,7 +36,7 @@ repos: additional_dependencies: [black==24.3.0] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.0 + rev: v0.4.3 hooks: - id: ruff-format types_or: [ python, pyi, jupyter ] @@ -50,7 +50,7 @@ repos: - id: rst-backticks - repo: https://github.com/MarcoGorelli/cython-lint - rev: v0.16.0 + rev: v0.16.2 hooks: - id: cython-lint args: [--no-pycodestyle] From 690203276cb69eea001a08db9793f831b4264b60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Mon, 6 May 2024 20:41:20 +0200 Subject: [PATCH 008/186] STY: manual fixes for UP031 rule (use string format specifiers) --- yt/_maintenance/backports.py | 2 +- yt/data_objects/analyzer_objects.py | 2 +- .../construction_data_containers.py | 5 +-- .../level_sets/clump_info_items.py | 2 +- yt/data_objects/static_output.py | 2 +- yt/fields/derived_field.py | 2 +- yt/fields/particle_fields.py | 8 ++--- yt/fields/vector_operations.py | 2 +- yt/frontends/adaptahop/data_structures.py | 4 +-- yt/frontends/boxlib/fields.py | 4 +-- yt/frontends/enzo/simulation_handling.py | 2 +- yt/frontends/fits/misc.py | 2 +- yt/frontends/flash/data_structures.py | 2 +- yt/frontends/gadget/simulation_handling.py | 2 +- yt/frontends/ramses/data_structures.py | 6 ++-- yt/frontends/sph/data_structures.py | 4 +-- yt/frontends/tipsy/data_structures.py | 2 +- yt/funcs.py | 31 +++++++++++-------- .../coordinates/cartesian_coordinates.py | 4 +-- yt/utilities/cosmology.py | 2 +- yt/utilities/on_demand_imports.py | 4 +-- yt/utilities/particle_generator.py | 2 +- yt/visualization/_commons.py | 4 +-- yt/visualization/fixed_resolution.py | 2 +- yt/visualization/image_writer.py | 2 +- yt/visualization/plot_modifications.py | 4 +-- yt/visualization/plot_window.py | 4 +-- yt/visualization/tests/test_save.py | 4 +-- yt/visualization/volume_rendering/lens.py | 7 +++-- .../volume_rendering/render_source.py | 2 +- yt/visualization/volume_rendering/scene.py | 4 +-- yt/visualization/volume_rendering/utils.py | 2 +- 32 files changed, 69 insertions(+), 62 deletions(-) diff --git a/yt/_maintenance/backports.py b/yt/_maintenance/backports.py index f3ee778243..78577af3e4 100644 --- a/yt/_maintenance/backports.py +++ b/yt/_maintenance/backports.py @@ -52,7 +52,7 @@ def __new__(cls, *values): if len(values) == 3: # check that errors argument is a string if not isinstance(values[2], str): - raise TypeError("errors must be a string, not %r" % (values[2])) + raise TypeError("errors must be a string, not %r" % (values[2])) # noqa: UP031 value = str(*values) member = str.__new__(cls, value) member._value_ = value diff --git a/yt/data_objects/analyzer_objects.py b/yt/data_objects/analyzer_objects.py index b6824b6249..97610f8273 100644 --- a/yt/data_objects/analyzer_objects.py +++ b/yt/data_objects/analyzer_objects.py @@ -67,7 +67,7 @@ class QuantityProxy(AnalysisTask): def __repr__(self): # Stolen from YTDataContainer.__repr__ s = f"{self.__class__.__name__}: " - s += ", ".join(["%s" % list(self.args)]) + s += ", ".join([str(list(self.args))]) s += ", ".join(f"{k}={v}" for k, v in self.kwargs.items()) return s diff --git a/yt/data_objects/construction_data_containers.py b/yt/data_objects/construction_data_containers.py index adcf01532c..03c10f92c1 100644 --- a/yt/data_objects/construction_data_containers.py +++ b/yt/data_objects/construction_data_containers.py @@ -880,10 +880,11 @@ def get_data(self, fields=None): fill, gen, part, alias = self._split_fields(fields_to_get) except NeedsGridType as e: if self._num_ghost_zones == 0: + num_ghost_zones = self._num_ghost_zones raise RuntimeError( "Attempting to access a field that needs ghost zones, but " - "num_ghost_zones = %s. You should create the covering grid " - "with nonzero num_ghost_zones." % self._num_ghost_zones + f"{num_ghost_zones = }. You should create the covering grid " + "with nonzero num_ghost_zones." ) from e else: raise diff --git a/yt/data_objects/level_sets/clump_info_items.py b/yt/data_objects/level_sets/clump_info_items.py index 798aecc30b..d0c727f310 100644 --- a/yt/data_objects/level_sets/clump_info_items.py +++ b/yt/data_objects/level_sets/clump_info_items.py @@ -109,7 +109,7 @@ def _distance_to_main_clump(clump, units="pc"): distance = np.sqrt(((master_com - my_com) ** 2).sum()) distance.convert_to_units("pc") return ( - "Distance from master center of mass: %%.6e %s." % units, + f"Distance from master center of mass: %.6e {units}.", distance.in_units(units), ) diff --git a/yt/data_objects/static_output.py b/yt/data_objects/static_output.py index 5c546a1849..57f6507dcd 100644 --- a/yt/data_objects/static_output.py +++ b/yt/data_objects/static_output.py @@ -1395,7 +1395,7 @@ def set_units(self): new_unit, my_u.base_value / (1 + self.current_redshift), dimensions.length, - "\\rm{%s}/(1+z)" % my_unit, + f"\\rm{{{my_unit}}}/(1+z)", prefixable=True, ) self.unit_registry.modify("a", 1 / (1 + self.current_redshift)) diff --git a/yt/fields/derived_field.py b/yt/fields/derived_field.py index 42b6ff6636..7e90277b2d 100644 --- a/yt/fields/derived_field.py +++ b/yt/fields/derived_field.py @@ -303,7 +303,7 @@ def get_label(self, projected=False): name = self.display_name # Start with the field name - data_label = r"$\rm{%s}" % name + data_label = rf"$\rm{{{name}}}" # Grab the correct units if projected: diff --git a/yt/fields/particle_fields.py b/yt/fields/particle_fields.py index c52bd2a0b4..7db9bfabea 100644 --- a/yt/fields/particle_fields.py +++ b/yt/fields/particle_fields.py @@ -110,7 +110,7 @@ def particle_count(field, data): function=particle_count, validators=[ValidateSpatial()], units="", - display_name=r"\mathrm{%s Count}" % ptype_dn, + display_name=rf"\mathrm{{{ptype_dn} Count}}", ) def particle_mass(field, data): @@ -125,7 +125,7 @@ def particle_mass(field, data): sampling_type="cell", function=particle_mass, validators=[ValidateSpatial()], - display_name=r"\mathrm{%s Mass}" % ptype_dn, + display_name=rf"\mathrm{{{ptype_dn} Mass}}", units=unit_system["mass"], ) @@ -144,7 +144,7 @@ def particle_density(field, data): sampling_type="cell", function=particle_density, validators=[ValidateSpatial()], - display_name=r"\mathrm{%s Density}" % ptype_dn, + display_name=rf"\mathrm{{{ptype_dn} Density}}", units=unit_system["density"], ) @@ -160,7 +160,7 @@ def particle_cic(field, data): sampling_type="cell", function=particle_cic, validators=[ValidateSpatial()], - display_name=r"\mathrm{%s CIC Density}" % ptype_dn, + display_name=rf"\mathrm{{{ptype_dn} CIC Density}}", units=unit_system["density"], ) diff --git a/yt/fields/vector_operations.py b/yt/fields/vector_operations.py index 0c3cbe0a2f..99e0544539 100644 --- a/yt/fields/vector_operations.py +++ b/yt/fields/vector_operations.py @@ -133,7 +133,7 @@ def _los_field(field, data): function=_los_field, units=field_units, validators=validators, - display_name=r"\mathrm{Line of Sight %s}" % basename.capitalize(), + display_name=rf"\mathrm{{Line of Sight {basename.capitalize()}}}", ) diff --git a/yt/frontends/adaptahop/data_structures.py b/yt/frontends/adaptahop/data_structures.py index d36bcff4a2..e8cb853568 100644 --- a/yt/frontends/adaptahop/data_structures.py +++ b/yt/frontends/adaptahop/data_structures.py @@ -107,7 +107,7 @@ def _guess_headers_from_file(self, filename) -> None: pass if not ok: - raise OSError("Could not read headers from file %s" % filename) + raise OSError(f"Could not read headers from file {filename}") istart = fpu.tell() fpu.seek(0, 2) @@ -130,7 +130,7 @@ def _guess_headers_from_file(self, filename) -> None: continue if not ok: - raise OSError("Could not guess fields from file %s" % filename) + raise OSError(f"Could not guess fields from file {filename}") self._header_attributes = header_attributes self._halo_attributes = attributes diff --git a/yt/frontends/boxlib/fields.py b/yt/frontends/boxlib/fields.py index 5e32ad83e7..8ce65068a7 100644 --- a/yt/frontends/boxlib/fields.py +++ b/yt/frontends/boxlib/fields.py @@ -488,7 +488,7 @@ def setup_fluid_fields(self): sampling_type="cell", function=func, units=unit_system["density"], - display_name=r"\rho %s" % tex_label, + display_name=rf"\rho {tex_label}", ) # Most of the time our species will be of the form @@ -507,7 +507,7 @@ def setup_fluid_fields(self): elif field.startswith("omegadot("): nice_name, tex_label = _nice_species_name(field) - display_name = r"\dot{\omega}\left[%s\right]" % tex_label + display_name = rf"\dot{{\omega}}\left[{tex_label}\right]" # Overwrite field to use nicer tex_label'ed display_name self.add_output_field( ("boxlib", field), diff --git a/yt/frontends/enzo/simulation_handling.py b/yt/frontends/enzo/simulation_handling.py index 8733e40f84..5fb432650a 100644 --- a/yt/frontends/enzo/simulation_handling.py +++ b/yt/frontends/enzo/simulation_handling.py @@ -88,7 +88,7 @@ def _set_units(self): new_unit, self.unit_registry.lut[my_unit][0], dimensions.length, - "\\rm{%s}/(1+z)" % my_unit, + f"\\rm{{{my_unit}}}/(1+z)", prefixable=True, ) self.length_unit = self.quan( diff --git a/yt/frontends/fits/misc.py b/yt/frontends/fits/misc.py index 2da062b203..8362c6fa81 100644 --- a/yt/frontends/fits/misc.py +++ b/yt/frontends/fits/misc.py @@ -287,6 +287,6 @@ def _repr_html_(self): img = base64.b64encode(f.read()).decode() ret += ( r'
' % img + rf'src="data:image/png;base64,{img}">
' ) return ret diff --git a/yt/frontends/flash/data_structures.py b/yt/frontends/flash/data_structures.py index 7d617f07a1..abc8d0d654 100644 --- a/yt/frontends/flash/data_structures.py +++ b/yt/frontends/flash/data_structures.py @@ -246,7 +246,7 @@ def _set_code_unit_attributes(self): else: raise RuntimeError( "Runtime parameter unitsystem with " - "value %s is unrecognized" % self["unitsystem"] + f"value {self['unitsystem']} is unrecognized" ) else: b_factor = 1.0 diff --git a/yt/frontends/gadget/simulation_handling.py b/yt/frontends/gadget/simulation_handling.py index 505382aeca..b53c08ec89 100644 --- a/yt/frontends/gadget/simulation_handling.py +++ b/yt/frontends/gadget/simulation_handling.py @@ -81,7 +81,7 @@ def _set_units(self): new_unit, self.unit_registry.lut[my_unit][0], dimensions.length, - "\\rm{%s}/(1+z)" % my_unit, + f"\\rm{{{my_unit}}}/(1+z)", prefixable=True, ) self.length_unit = self.quan( diff --git a/yt/frontends/ramses/data_structures.py b/yt/frontends/ramses/data_structures.py index 97cc7c5af2..851cf29b88 100644 --- a/yt/frontends/ramses/data_structures.py +++ b/yt/frontends/ramses/data_structures.py @@ -417,7 +417,7 @@ def __init__( elif num_ghost_zones < 0: raise RuntimeError( "Cannot initialize a domain subset with a negative number " - "of ghost zones, was called with num_ghost_zones=%s" % num_ghost_zones + f"of ghost zones, was called with {num_ghost_zones=}" ) @property @@ -1038,8 +1038,8 @@ def caster(val): if rheader["ordering type"] != "hilbert" and self._bbox is not None: raise NotImplementedError( - "The ordering %s is not compatible with the `bbox` argument." - % rheader["ordering type"] + f"The ordering {rheader['ordering type']} " + "is not compatible with the `bbox` argument." ) self.parameters.update(rheader) self.domain_left_edge = np.zeros(3, dtype="float64") diff --git a/yt/frontends/sph/data_structures.py b/yt/frontends/sph/data_structures.py index 7d0e9c696d..3c65b7171d 100644 --- a/yt/frontends/sph/data_structures.py +++ b/yt/frontends/sph/data_structures.py @@ -59,8 +59,8 @@ def sph_smoothing_style(self): def sph_smoothing_style(self, value): if value not in self._sph_smoothing_styles: raise ValueError( - "Smoothing style not implemented: %s, please " - "select one of the following: " % value, + f"Smoothing style not implemented: {value}, " + "please select one of the following: ", self._sph_smoothing_styles, ) diff --git a/yt/frontends/tipsy/data_structures.py b/yt/frontends/tipsy/data_structures.py index 70d3e1cb77..d27c7f38d5 100644 --- a/yt/frontends/tipsy/data_structures.py +++ b/yt/frontends/tipsy/data_structures.py @@ -118,7 +118,7 @@ def _parse_parameter_file(self): # the snapshot time and particle counts. f = open(self.parameter_filename, "rb") - hh = self.endian + "".join("%s" % (b) for a, b in self._header_spec) + hh = self.endian + "".join(str(b) for a, b in self._header_spec) hvals = { a: c for (a, b), c in zip( diff --git a/yt/funcs.py b/yt/funcs.py index 94547af091..2c44d521f8 100644 --- a/yt/funcs.py +++ b/yt/funcs.py @@ -1086,12 +1086,20 @@ def array_like_field(data, x, field): return data.ds.quan(x, units) +def _full_type_name(obj: object = None, /, *, cls: Optional[type] = None) -> str: + if cls is not None and obj is not None: + raise TypeError("_full_type_name takes an object or a class, but not both") + if cls is None: + cls = obj.__class__ + prefix = f"{cls.__module__}." if cls.__module__ != "builtins" else "" + return f"{prefix}{cls.__name__}" + + def validate_3d_array(obj): if not is_sequence(obj) or len(obj) != 3: raise TypeError( - "Expected an array of size (3,), received '{}' of length {}".format( - str(type(obj)).split("'")[1], len(obj) - ) + f"Expected an array of size (3,), " + f"received {_full_type_name(obj)!r} of length {len(obj)}" ) @@ -1135,23 +1143,21 @@ def validate_float(obj): ): raise TypeError( "Expected a numeric value (or tuple of format " - "(float, String)), received an inconsistent tuple " - "'%s'." % str(obj) + f"(float, String)), received an inconsistent tuple {str(obj)!r}." ) else: return if is_sequence(obj) and (len(obj) != 1 or not isinstance(obj[0], numeric_type)): raise TypeError( "Expected a numeric value (or size-1 array), " - "received '{}' of length {}".format(str(type(obj)).split("'")[1], len(obj)) + f"received {_full_type_name(obj)!r} of length {len(obj)}" ) def validate_sequence(obj): if obj is not None and not is_sequence(obj): raise TypeError( - "Expected an iterable object, " - "received '%s'" % str(type(obj)).split("'")[1] + "Expected an iterable object, " f"received {_full_type_name(obj)!r}" ) @@ -1181,9 +1187,8 @@ def is_valid_field_key(key): def validate_object(obj, data_type): if obj is not None and not isinstance(obj, data_type): raise TypeError( - "Expected an object of '{}' type, received '{}'".format( - str(data_type).split("'")[1], str(type(obj)).split("'")[1] - ) + f"Expected an object of {_full_type_name(cls=data_type)!r} type, " + f"received {_full_type_name(obj)!r}" ) @@ -1209,13 +1214,13 @@ def validate_center(center): raise TypeError( "Expected 'center' to be in ['c', 'center', " "'m', 'max', 'min'] or the prefix to be " - "'max_'/'min_', received '%s'." % center + f"'max_'/'min_', received {center!r}." ) elif not isinstance(center, (numeric_type, YTQuantity)) and not is_sequence(center): raise TypeError( "Expected 'center' to be a numeric object of type " "list/tuple/np.ndarray/YTArray/YTQuantity, " - "received '%s'." % str(type(center)).split("'")[1] + f"received {_full_type_name(center)}." ) diff --git a/yt/geometry/coordinates/cartesian_coordinates.py b/yt/geometry/coordinates/cartesian_coordinates.py index 95812c4d91..775de8c6a5 100644 --- a/yt/geometry/coordinates/cartesian_coordinates.py +++ b/yt/geometry/coordinates/cartesian_coordinates.py @@ -593,8 +593,8 @@ def _ortho_pixelize( mask = mask.transpose() else: raise NotImplementedError( - "A pixelization routine has not been implemented for %s " - "data objects" % str(type(data_source)) + "A pixelization routine has not been implemented for " + f"{type(data_source)} data objects" ) buff = buff.transpose() mask = mask.transpose() diff --git a/yt/utilities/cosmology.py b/yt/utilities/cosmology.py index cb415bc766..691d86e1c4 100644 --- a/yt/utilities/cosmology.py +++ b/yt/utilities/cosmology.py @@ -95,7 +95,7 @@ def __init__( new_unit, my_u.base_value, dimensions.length, - "\\rm{%s}/(1+z)" % my_unit, + f"\\rm{{{my_unit}}}/(1+z)", prefixable=True, ) self.unit_registry = unit_registry diff --git a/yt/utilities/on_demand_imports.py b/yt/utilities/on_demand_imports.py index e77a482e34..839617f4a5 100644 --- a/yt/utilities/on_demand_imports.py +++ b/yt/utilities/on_demand_imports.py @@ -180,8 +180,8 @@ def __init__(self, pkg_name, exc: Optional[BaseException] = None): # relatively short. Discussion related to this is in # yt-project/yt#1966 self.error = ImportError( - "This functionality requires the %s " - "package to be installed." % self.pkg_name + f"This functionality requires the {self.pkg_name} " + "package to be installed." ) else: self.error = ImportError( diff --git a/yt/utilities/particle_generator.py b/yt/utilities/particle_generator.py index c41bd68a95..a0957e9b38 100644 --- a/yt/utilities/particle_generator.py +++ b/yt/utilities/particle_generator.py @@ -36,7 +36,7 @@ def __init__(self, ds, num_particles, field_list, ptype="io"): except Exception as e: raise KeyError( "You must specify position fields: " - + " ".join("particle_position_%s" % ax for ax in "xyz") + + " ".join(f"particle_position_{ax}" for ax in "xyz") ) from e self.index_index = self.field_list.index((ptype, "particle_index")) diff --git a/yt/visualization/_commons.py b/yt/visualization/_commons.py index 49fdc33c15..99d4b919a2 100644 --- a/yt/visualization/_commons.py +++ b/yt/visualization/_commons.py @@ -258,8 +258,8 @@ def get_default_from_config(data_source, *, field, keys, defaults): def _get_units_label(units: str) -> str: if r"\frac" in units: - return r"$\ \ \left(%s\right)$" % units + return rf"$\ \ \left({units}\right)$" elif units: - return r"$\ \ (%s)$" % units + return rf"$\ \ ({units})$" else: return "" diff --git a/yt/visualization/fixed_resolution.py b/yt/visualization/fixed_resolution.py index 05fdae3db8..d2da6a29e1 100644 --- a/yt/visualization/fixed_resolution.py +++ b/yt/visualization/fixed_resolution.py @@ -825,7 +825,7 @@ def _generate_image_and_mask(self, item) -> None: norm = self.ds.quan(dpx * dpy, "code_length**2").in_base() buff /= norm.v units = data.units / norm.units - info["label"] = "%s $\\rm{Density}$" % info["label"] + info["label"] += " $\\rm{Density}$" else: units = data.units diff --git a/yt/visualization/image_writer.py b/yt/visualization/image_writer.py index e9326d23cc..87bd9189e9 100644 --- a/yt/visualization/image_writer.py +++ b/yt/visualization/image_writer.py @@ -137,7 +137,7 @@ def write_bitmap(bitmap_array, filename, max_val=None, transpose=False): if len(bitmap_array.shape) != 3 or bitmap_array.shape[-1] not in (3, 4): raise RuntimeError( "Expecting image array of shape (N,M,3) or " - "(N,M,4), received %s" % str(bitmap_array.shape) + f"(N,M,4), received {str(bitmap_array.shape)}" ) if bitmap_array.dtype != np.uint8: diff --git a/yt/visualization/plot_modifications.py b/yt/visualization/plot_modifications.py index fd90ca03cc..ca69ff6549 100644 --- a/yt/visualization/plot_modifications.py +++ b/yt/visualization/plot_modifications.py @@ -1244,9 +1244,9 @@ def __call__(self, plot): y[i] = right_edge_y[n] - (12 * (yy1 - yy0) / ypix) else: raise RuntimeError( - "Unrecognized id_loc value ('%s'). " + f"Unrecognized id_loc value ({self.id_loc!r}). " "Allowed values are 'lower left', lower right', " - "'upper left', and 'upper right'." % self.id_loc + "'upper left', and 'upper right'." ) xi, yi = self._sanitize_xy_order(plot, x[i], y[i]) plot._axes.text(xi, yi, "%d" % block_ids[n], clip_on=True) diff --git a/yt/visualization/plot_window.py b/yt/visualization/plot_window.py index bb6cae9c9d..4534d7325c 100644 --- a/yt/visualization/plot_window.py +++ b/yt/visualization/plot_window.py @@ -1195,9 +1195,9 @@ def _setup_plots(self): if colorbar_label is None: colorbar_label = image.info["label"] if getattr(self, "moment", 1) == 2: - colorbar_label = "%s \\rm{Standard Deviation}" % colorbar_label + colorbar_label = f"{colorbar_label} \\rm{{Standard Deviation}}" if hasattr(self, "projected"): - colorbar_label = "$\\rm{Projected }$ %s" % colorbar_label + colorbar_label = f"$\\rm{{Projected }}$ {colorbar_label}" if units is not None and units != "": colorbar_label += _get_units_label(units) diff --git a/yt/visualization/tests/test_save.py b/yt/visualization/tests/test_save.py index 31ec9e230a..2612e8054d 100644 --- a/yt/visualization/tests/test_save.py +++ b/yt/visualization/tests/test_save.py @@ -76,8 +76,8 @@ def test_suffix_clashing(ext, simple_sliceplot, tmp_path): target = (tmp_path / "myfile").with_suffix(ext) expected_warning = re.compile( - r"Received two valid image formats '%s' \(from filename\) " - r"and 'png' \(from suffix\)\. The former is ignored\." % ext[1:] + rf"Received two valid image formats {ext.removeprefix('.')!r} " + r"\(from filename\) and 'png' \(from suffix\)\. The former is ignored\." ) with pytest.warns(UserWarning, match=expected_warning): diff --git a/yt/visualization/volume_rendering/lens.py b/yt/visualization/volume_rendering/lens.py index 0e19b7e0a1..7335cebe34 100644 --- a/yt/visualization/volume_rendering/lens.py +++ b/yt/visualization/volume_rendering/lens.py @@ -137,10 +137,11 @@ def project_to_plane(self, camera, pos, res=None): return px, py, dz def __repr__(self): - disp = ":\n\tlens_type:plane-parallel\n\tviewpoint:%s" % ( - self.viewpoint + return ( + ":\n" + "\tlens_type:plane-parallel\n" + f"\tviewpoint:{self.viewpoint}" ) - return disp class PerspectiveLens(Lens): diff --git a/yt/visualization/volume_rendering/render_source.py b/yt/visualization/volume_rendering/render_source.py index 6943d04ed7..aff769543f 100644 --- a/yt/visualization/volume_rendering/render_source.py +++ b/yt/visualization/volume_rendering/render_source.py @@ -290,7 +290,7 @@ def transfer_function(self, value): if not isinstance(value, valid_types): raise RuntimeError( "transfer_function not a valid type, " - "received object of type %s" % type(value) + f"received object of type {type(value)}" ) if isinstance(value, ProjectionTransferFunction): self.sampler_type = "projection" diff --git a/yt/visualization/volume_rendering/scene.py b/yt/visualization/volume_rendering/scene.py index e45033cb14..ca8207cfbe 100644 --- a/yt/visualization/volume_rendering/scene.py +++ b/yt/visualization/volume_rendering/scene.py @@ -141,8 +141,8 @@ def add_source(self, render_source, keyname=None): lens_str = str(self.camera.lens) if "fisheye" in lens_str or "spherical" in lens_str: raise NotImplementedError( - "Line annotation sources are not supported for %s." - % (type(self.camera.lens).__name__), + "Line annotation sources are not supported " + f"for {type(self.camera.lens).__name__}." ) if isinstance(render_source, (LineSource, PointSource)): diff --git a/yt/visualization/volume_rendering/utils.py b/yt/visualization/volume_rendering/utils.py index d275f37b41..60340aca2a 100644 --- a/yt/visualization/volume_rendering/utils.py +++ b/yt/visualization/volume_rendering/utils.py @@ -26,7 +26,7 @@ def data_source_or_all(data_source): raise RuntimeError( "The data_source is not a valid 3D data container.\n" "Expected an object of type YTSelectionContainer3D but received " - "an object of type %s." % type(data_source) + f"an object of type {type(data_source)}." ) return data_source From c17b2f28d71ce74fc7ab9014ff8f3a1c727ff90a Mon Sep 17 00:00:00 2001 From: yut23 Date: Tue, 7 May 2024 13:00:02 -0400 Subject: [PATCH 009/186] BLD: fix bleeding-edge build script --- .github/workflows/bleeding-edge.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/bleeding-edge.yaml b/.github/workflows/bleeding-edge.yaml index 0807341e48..4b7a24db26 100644 --- a/.github/workflows/bleeding-edge.yaml +++ b/.github/workflows/bleeding-edge.yaml @@ -61,6 +61,7 @@ jobs: # are not installed by pip as specified from pyproject.toml, hence we get # to use the dev version of numpy at build time. run: | + python setup.py build_clib -q python setup.py build_ext -q -j2 python -m pip install -e .[test] --no-build-isolation From 38c3f1830755c40076f6c58ce499acaa72c5d8aa Mon Sep 17 00:00:00 2001 From: yut23 Date: Tue, 7 May 2024 13:00:46 -0400 Subject: [PATCH 010/186] BLD: disable deprecated numpy API for fixed_interpolator clib --- setup.py | 2 ++ setupext.py | 13 ++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index f1d547b183..2c49b693b3 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,7 @@ create_build_ext, get_python_include_dirs, install_ccompiler, + NUMPY_MACROS, ) install_ccompiler() @@ -111,6 +112,7 @@ def has_ext_modules(self): { "sources": ["yt/utilities/lib/fixed_interpolator.cpp"], "include_dirs": clib_include_dirs, + "define_macros": NUMPY_MACROS, }, ) diff --git a/setupext.py b/setupext.py index cf5af62812..a9aafdbbf9 100644 --- a/setupext.py +++ b/setupext.py @@ -394,6 +394,13 @@ def get_python_include_dirs(): return include_dirs +NUMPY_MACROS = [ + ("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION"), + # keep in sync with runtime requirements (pyproject.toml) + ("NPY_TARGET_VERSION", "NPY_1_19_API_VERSION"), +] + + def create_build_ext(lib_exts, cythonize_aliases): class build_ext(_build_ext): # subclass setuptools extension builder to avoid importing cython and numpy @@ -425,11 +432,7 @@ def finalize_options(self): self.include_dirs.append(numpy.get_include()) self.include_dirs.append(ewah_bool_utils.get_include()) - define_macros = [ - ("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION"), - # keep in sync with runtime requirements (pyproject.toml) - ("NPY_TARGET_VERSION", "NPY_1_19_API_VERSION"), - ] + define_macros = NUMPY_MACROS if self.define is None: self.define = define_macros From 70832cc67a13bc2e454e3f27a4c976b5e3f29591 Mon Sep 17 00:00:00 2001 From: yut23 Date: Tue, 7 May 2024 14:23:24 -0400 Subject: [PATCH 011/186] Fix import sorting --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2c49b693b3..56a26ef1f0 100644 --- a/setup.py +++ b/setup.py @@ -7,13 +7,13 @@ from setuptools import Distribution, setup from setupext import ( + NUMPY_MACROS, check_CPP14_flags, check_for_openmp, check_for_pyembree, create_build_ext, get_python_include_dirs, install_ccompiler, - NUMPY_MACROS, ) install_ccompiler() From 7eea5d18320ff1c1aca05f14647c2b731b2780ff Mon Sep 17 00:00:00 2001 From: yut23 Date: Tue, 7 May 2024 16:04:55 -0400 Subject: [PATCH 012/186] Remove setup.py calls in bleeding-edge workflow --- .github/workflows/bleeding-edge.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/bleeding-edge.yaml b/.github/workflows/bleeding-edge.yaml index 4b7a24db26..3567465440 100644 --- a/.github/workflows/bleeding-edge.yaml +++ b/.github/workflows/bleeding-edge.yaml @@ -61,9 +61,7 @@ jobs: # are not installed by pip as specified from pyproject.toml, hence we get # to use the dev version of numpy at build time. run: | - python setup.py build_clib -q - python setup.py build_ext -q -j2 - python -m pip install -e .[test] --no-build-isolation + python -m pip -v install -e .[test] --no-build-isolation - run: python -m pip list From a73fc924df30cc6bf9c970f2a42ea12af62df683 Mon Sep 17 00:00:00 2001 From: yut23 Date: Tue, 7 May 2024 16:10:10 -0400 Subject: [PATCH 013/186] Add build_clib to setup.py call in `yt update` --- yt/funcs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/funcs.py b/yt/funcs.py index 94547af091..f63b5f73fc 100644 --- a/yt/funcs.py +++ b/yt/funcs.py @@ -515,7 +515,7 @@ def update_git(path): def rebuild_modules(path, f): f.write("Rebuilding modules\n\n") p = subprocess.Popen( - [sys.executable, "setup.py", "build_ext", "-i"], + [sys.executable, "setup.py", "build_clib", "build_ext", "-i"], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, From ff5b6315f0abd8cbb55772cb16bcface82aac788 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Fri, 10 May 2024 14:17:56 -0400 Subject: [PATCH 014/186] some spelling fixes --- CONTRIBUTING.rst | 2 +- doc/cheatsheet.tex | 2 +- doc/source/analyzing/fields.rst | 2 +- doc/source/developing/testing.rst | 2 +- yt/frontends/halo_catalog/io.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 24fda87f6a..033d76df9f 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -450,7 +450,7 @@ For all types of contributions, it is required that all tests pass, or that all future. (See :ref:`testing`) * At a minimum, a minimal, self-contained example demonstrating the bug should because included in the body of the Pull Request, or as part of an - indepedent issue. + independent issue. When submitting, you will be asked to make sure that your changes meet all of these requirements. They are pretty easy to meet, and we're also happy to help diff --git a/doc/cheatsheet.tex b/doc/cheatsheet.tex index 911daefe66..fd2355c1ac 100644 --- a/doc/cheatsheet.tex +++ b/doc/cheatsheet.tex @@ -319,7 +319,7 @@ \subsection{Git} \texttt{git status} \textemdash\ Show status of working tree.\\ \texttt{git diff} \textemdash\ Show changed files in the working tree. \\ \texttt{git log} \textemdash\ Show a log of changes in reverse chronological -oder.\\ +order.\\ \texttt{git revert } \textemdash\ Revert the changes in an existing commit and create a new commit with reverted changes. \\ \texttt{git add } \textemdash\ Stage changes in the working tree to diff --git a/doc/source/analyzing/fields.rst b/doc/source/analyzing/fields.rst index 73a872e83a..50529db95b 100644 --- a/doc/source/analyzing/fields.rst +++ b/doc/source/analyzing/fields.rst @@ -368,7 +368,7 @@ second sets the corresponding ``value``. Currently available format properties a .. _efields: -Energy and Momemtum Fields +Energy and Momentum Fields -------------------------- Fields in yt representing energy and momentum quantities follow a specific diff --git a/doc/source/developing/testing.rst b/doc/source/developing/testing.rst index 1bea29f1b4..c1e1ac1d6e 100644 --- a/doc/source/developing/testing.rst +++ b/doc/source/developing/testing.rst @@ -438,7 +438,7 @@ test suite, which are intended to be as modern as possible (we don't set upper limits to versions unless we need to). The ``minimal`` target is used to check that we don't break backward compatibility with old versions of upstream projects by accident. It is intended to pin strictly our minimal supported -versions. The ``test`` target specifies the tools neeed to run the tests, but +versions. The ``test`` target specifies the tools needed to run the tests, but not needed by yt itself. **Python version support.** diff --git a/yt/frontends/halo_catalog/io.py b/yt/frontends/halo_catalog/io.py index c22b43407b..97a4672853 100644 --- a/yt/frontends/halo_catalog/io.py +++ b/yt/frontends/halo_catalog/io.py @@ -105,7 +105,7 @@ def _read_particle_fields(self, dobj, ptf): _read_particle_selection = IOHandlerGadgetFOFHaloHDF5._read_particle_selection -# ignoring type in this mixing to circunvent this error from mypy +# ignoring type in this mixing to circumvent this error from mypy # Definition of "_read_particle_fields" in base class "HaloDatasetIOHandler" # is incompatible with definition in base class "IOHandlerYTHaloCatalog" # From 8cc65978298f3fa29fd2a1cf4b464c087680f921 Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Wed, 15 May 2024 13:45:42 -0500 Subject: [PATCH 015/186] Move field resolution inside conditional. --- yt/data_objects/data_containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/data_objects/data_containers.py b/yt/data_objects/data_containers.py index 26763d5972..83f4ff7594 100644 --- a/yt/data_objects/data_containers.py +++ b/yt/data_objects/data_containers.py @@ -33,8 +33,8 @@ def sanitize_weight_field(ds, field, weight): - field_object = ds._get_field_info(field) if weight is None: + field_object = ds._get_field_info(field) if field_object.sampling_type == "particle": if field_object.name[0] == "gas": ptype = ds._sph_ptypes[0] From 06a6e2d81f07ef770f02452e68e91ff1a8123bd1 Mon Sep 17 00:00:00 2001 From: Henry Jones Date: Thu, 18 Jan 2024 16:49:29 -0800 Subject: [PATCH 016/186] Renamed boxlib frontend to amrex --- .../definitions.py => amrex/__init__.py} | 0 yt/frontends/amrex/api.py | 24 + .../{boxlib => amrex}/data_structures.py | 0 .../{boxlib/misc.py => amrex/definitions.py} | 0 yt/frontends/{boxlib => amrex}/fields.py | 0 yt/frontends/{boxlib => amrex}/io.py | 0 yt/frontends/amrex/misc.py | 0 yt/frontends/amrex/tests/__init__.py | 0 yt/frontends/amrex/tests/test_outputs.py | 445 ++++++++++++++++++ yt/frontends/boxlib/__init__.py | 17 + yt/frontends/boxlib/api.py | 9 +- .../boxlib/data_structures/__init__.py | 15 + yt/frontends/boxlib/fields/__init__.py | 7 + yt/frontends/boxlib/io/__init__.py | 1 + yt/frontends/boxlib/tests/__init__.py | 1 + 15 files changed, 515 insertions(+), 4 deletions(-) rename yt/frontends/{boxlib/definitions.py => amrex/__init__.py} (100%) create mode 100644 yt/frontends/amrex/api.py rename yt/frontends/{boxlib => amrex}/data_structures.py (100%) rename yt/frontends/{boxlib/misc.py => amrex/definitions.py} (100%) rename yt/frontends/{boxlib => amrex}/fields.py (100%) rename yt/frontends/{boxlib => amrex}/io.py (100%) create mode 100644 yt/frontends/amrex/misc.py create mode 100644 yt/frontends/amrex/tests/__init__.py create mode 100644 yt/frontends/amrex/tests/test_outputs.py create mode 100644 yt/frontends/boxlib/data_structures/__init__.py create mode 100644 yt/frontends/boxlib/fields/__init__.py create mode 100644 yt/frontends/boxlib/io/__init__.py diff --git a/yt/frontends/boxlib/definitions.py b/yt/frontends/amrex/__init__.py similarity index 100% rename from yt/frontends/boxlib/definitions.py rename to yt/frontends/amrex/__init__.py diff --git a/yt/frontends/amrex/api.py b/yt/frontends/amrex/api.py new file mode 100644 index 0000000000..334cc8bb63 --- /dev/null +++ b/yt/frontends/amrex/api.py @@ -0,0 +1,24 @@ +from . import tests +from .data_structures import ( + AMReXDataset, + AMReXHierarchy, + BoxlibDataset, + BoxlibGrid, + BoxlibHierarchy, + CastroDataset, + MaestroDataset, + NyxDataset, + NyxHierarchy, + OrionDataset, + OrionHierarchy, + WarpXDataset, + WarpXHierarchy, +) +from .fields import ( + BoxlibFieldInfo, + CastroFieldInfo, + MaestroFieldInfo, + NyxFieldInfo, + WarpXFieldInfo, +) +from .io import IOHandlerBoxlib diff --git a/yt/frontends/boxlib/data_structures.py b/yt/frontends/amrex/data_structures.py similarity index 100% rename from yt/frontends/boxlib/data_structures.py rename to yt/frontends/amrex/data_structures.py diff --git a/yt/frontends/boxlib/misc.py b/yt/frontends/amrex/definitions.py similarity index 100% rename from yt/frontends/boxlib/misc.py rename to yt/frontends/amrex/definitions.py diff --git a/yt/frontends/boxlib/fields.py b/yt/frontends/amrex/fields.py similarity index 100% rename from yt/frontends/boxlib/fields.py rename to yt/frontends/amrex/fields.py diff --git a/yt/frontends/boxlib/io.py b/yt/frontends/amrex/io.py similarity index 100% rename from yt/frontends/boxlib/io.py rename to yt/frontends/amrex/io.py diff --git a/yt/frontends/amrex/misc.py b/yt/frontends/amrex/misc.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/yt/frontends/amrex/tests/__init__.py b/yt/frontends/amrex/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/yt/frontends/amrex/tests/test_outputs.py b/yt/frontends/amrex/tests/test_outputs.py new file mode 100644 index 0000000000..d3e9e5d91e --- /dev/null +++ b/yt/frontends/amrex/tests/test_outputs.py @@ -0,0 +1,445 @@ +import numpy as np +from numpy.testing import assert_allclose, assert_equal + +from yt.frontends.boxlib.api import ( + AMReXDataset, + CastroDataset, + MaestroDataset, + NyxDataset, + OrionDataset, + WarpXDataset, +) +from yt.loaders import load +from yt.testing import ( + disable_dataset_cache, + requires_file, + units_override_check, +) +from yt.utilities.answer_testing.framework import ( + GridValuesTest, + data_dir_load, + requires_ds, + small_patch_amr, +) + +# We don't do anything needing ghost zone generation right now, because these +# are non-periodic datasets. +_orion_fields = ( + ("gas", "temperature"), + ("gas", "density"), + ("gas", "velocity_magnitude"), +) +_nyx_fields = ( + ("boxlib", "Ne"), + ("boxlib", "Temp"), + ("boxlib", "particle_mass_density"), +) +_warpx_fields = (("mesh", "Ex"), ("mesh", "By"), ("mesh", "jz")) +_castro_fields = ( + ("boxlib", "Temp"), + ("gas", "density"), + ("boxlib", "particle_count"), +) + +radadvect = "RadAdvect/plt00000" + + +@requires_ds(radadvect) +def test_radadvect(): + ds = data_dir_load(radadvect) + assert_equal(str(ds), "plt00000") + for test in small_patch_amr(ds, _orion_fields): + test_radadvect.__name__ = test.description + yield test + + +rt = "RadTube/plt00500" + + +@requires_ds(rt) +def test_radtube(): + ds = data_dir_load(rt) + assert_equal(str(ds), "plt00500") + for test in small_patch_amr(ds, _orion_fields): + test_radtube.__name__ = test.description + yield test + + +star = "StarParticles/plrd01000" + + +@requires_ds(star) +def test_star(): + ds = data_dir_load(star) + assert_equal(str(ds), "plrd01000") + for test in small_patch_amr(ds, _orion_fields): + test_star.__name__ = test.description + yield test + + +LyA = "Nyx_LyA/plt00000" + + +@requires_ds(LyA) +def test_LyA(): + ds = data_dir_load(LyA) + assert_equal(str(ds), "plt00000") + for test in small_patch_amr( + ds, _nyx_fields, input_center="c", input_weight=("boxlib", "Ne") + ): + test_LyA.__name__ = test.description + yield test + + +@requires_file(LyA) +def test_nyx_particle_io(): + ds = data_dir_load(LyA) + + grid = ds.index.grids[0] + npart_grid_0 = 7908 # read directly from the header + assert_equal(grid[("all", "particle_position_x")].size, npart_grid_0) + assert_equal(grid["DM", "particle_position_y"].size, npart_grid_0) + assert_equal(grid["all", "particle_position_z"].size, npart_grid_0) + + ad = ds.all_data() + npart = 32768 # read directly from the header + assert_equal(ad[("all", "particle_velocity_x")].size, npart) + assert_equal(ad["DM", "particle_velocity_y"].size, npart) + assert_equal(ad["all", "particle_velocity_z"].size, npart) + + assert np.all(ad[("all", "particle_mass")] == ad[("all", "particle_mass")][0]) + + left_edge = ds.arr([0.0, 0.0, 0.0], "code_length") + right_edge = ds.arr([4.0, 4.0, 4.0], "code_length") + center = 0.5 * (left_edge + right_edge) + + reg = ds.region(center, left_edge, right_edge) + + assert np.all( + np.logical_and( + reg[("all", "particle_position_x")] <= right_edge[0], + reg[("all", "particle_position_x")] >= left_edge[0], + ) + ) + + assert np.all( + np.logical_and( + reg[("all", "particle_position_y")] <= right_edge[1], + reg[("all", "particle_position_y")] >= left_edge[1], + ) + ) + + assert np.all( + np.logical_and( + reg[("all", "particle_position_z")] <= right_edge[2], + reg[("all", "particle_position_z")] >= left_edge[2], + ) + ) + + +RT_particles = "RT_particles/plt00050" + + +@requires_ds(RT_particles) +def test_RT_particles(): + ds = data_dir_load(RT_particles) + assert_equal(str(ds), "plt00050") + for test in small_patch_amr(ds, _castro_fields): + test_RT_particles.__name__ = test.description + yield test + + +@requires_file(RT_particles) +def test_castro_particle_io(): + ds = data_dir_load(RT_particles) + + grid = ds.index.grids[2] + npart_grid_2 = 49 # read directly from the header + assert_equal(grid[("all", "particle_position_x")].size, npart_grid_2) + assert_equal(grid["Tracer", "particle_position_y"].size, npart_grid_2) + assert_equal(grid["all", "particle_position_y"].size, npart_grid_2) + + ad = ds.all_data() + npart = 49 # read directly from the header + assert_equal(ad[("all", "particle_velocity_x")].size, npart) + assert_equal(ad["Tracer", "particle_velocity_y"].size, npart) + assert_equal(ad["all", "particle_velocity_y"].size, npart) + + left_edge = ds.arr([0.0, 0.0, 0.0], "code_length") + right_edge = ds.arr([0.25, 1.0, 1.0], "code_length") + center = 0.5 * (left_edge + right_edge) + + reg = ds.region(center, left_edge, right_edge) + + assert np.all( + np.logical_and( + reg[("all", "particle_position_x")] <= right_edge[0], + reg[("all", "particle_position_x")] >= left_edge[0], + ) + ) + + assert np.all( + np.logical_and( + reg[("all", "particle_position_y")] <= right_edge[1], + reg[("all", "particle_position_y")] >= left_edge[1], + ) + ) + + +langmuir = "LangmuirWave/plt00020_v2" + + +@requires_ds(langmuir) +def test_langmuir(): + ds = data_dir_load(langmuir) + assert_equal(str(ds), "plt00020_v2") + for test in small_patch_amr( + ds, _warpx_fields, input_center="c", input_weight=("mesh", "Ex") + ): + test_langmuir.__name__ = test.description + yield test + + +plasma = "PlasmaAcceleration/plt00030_v2" + + +@requires_ds(plasma) +def test_plasma(): + ds = data_dir_load(plasma) + assert_equal(str(ds), "plt00030_v2") + for test in small_patch_amr( + ds, _warpx_fields, input_center="c", input_weight=("mesh", "Ex") + ): + test_plasma.__name__ = test.description + yield test + + +beam = "GaussianBeam/plt03008" + + +@requires_ds(beam) +def test_beam(): + ds = data_dir_load(beam) + assert_equal(str(ds), "plt03008") + for param in ("number of boxes", "maximum zones"): + # PR 2807 + # these parameters are only populated if the config file attached to this + # dataset is read correctly + assert param in ds.parameters + for test in small_patch_amr( + ds, _warpx_fields, input_center="c", input_weight=("mesh", "Ex") + ): + test_beam.__name__ = test.description + yield test + + +@requires_file(plasma) +def test_warpx_particle_io(): + ds = data_dir_load(plasma) + grid = ds.index.grids[0] + + # read directly from the header + npart0_grid_0 = 344 + npart1_grid_0 = 69632 + + assert_equal(grid["particle0", "particle_position_x"].size, npart0_grid_0) + assert_equal(grid["particle1", "particle_position_y"].size, npart1_grid_0) + assert_equal(grid["all", "particle_position_z"].size, npart0_grid_0 + npart1_grid_0) + + # read directly from the header + npart0 = 1360 + npart1 = 802816 + ad = ds.all_data() + assert_equal(ad["particle0", "particle_velocity_x"].size, npart0) + assert_equal(ad["particle1", "particle_velocity_y"].size, npart1) + assert_equal(ad["all", "particle_velocity_z"].size, npart0 + npart1) + + np.all(ad["particle1", "particle_mass"] == ad["particle1", "particle_mass"][0]) + np.all(ad["particle0", "particle_mass"] == ad["particle0", "particle_mass"][0]) + + left_edge = ds.arr([-7.5e-5, -7.5e-5, -7.5e-5], "code_length") + right_edge = ds.arr([2.5e-5, 2.5e-5, 2.5e-5], "code_length") + center = 0.5 * (left_edge + right_edge) + + reg = ds.region(center, left_edge, right_edge) + + assert np.all( + np.logical_and( + reg[("all", "particle_position_x")] <= right_edge[0], + reg[("all", "particle_position_x")] >= left_edge[0], + ) + ) + + assert np.all( + np.logical_and( + reg[("all", "particle_position_y")] <= right_edge[1], + reg[("all", "particle_position_y")] >= left_edge[1], + ) + ) + + assert np.all( + np.logical_and( + reg[("all", "particle_position_z")] <= right_edge[2], + reg[("all", "particle_position_z")] >= left_edge[2], + ) + ) + + +_raw_fields = [("raw", "Bx"), ("raw", "Ey"), ("raw", "jz")] + +laser = "Laser/plt00015" + + +@requires_ds(laser) +def test_raw_fields(): + for field in _raw_fields: + yield GridValuesTest(laser, field) + + +@requires_file(rt) +def test_OrionDataset(): + assert isinstance(data_dir_load(rt), OrionDataset) + + +@requires_file(LyA) +def test_NyxDataset(): + assert isinstance(data_dir_load(LyA), NyxDataset) + + +@requires_file("nyx_small/nyx_small_00000") +def test_NyxDataset_2(): + assert isinstance(data_dir_load("nyx_small/nyx_small_00000"), NyxDataset) + + +@requires_file(RT_particles) +def test_CastroDataset(): + assert isinstance(data_dir_load(RT_particles), CastroDataset) + + +@requires_file("castro_sod_x_plt00036") +def test_CastroDataset_2(): + assert isinstance(data_dir_load("castro_sod_x_plt00036"), CastroDataset) + + +@requires_file("castro_sedov_1d_cyl_plt00150") +def test_CastroDataset_3(): + assert isinstance(data_dir_load("castro_sedov_1d_cyl_plt00150"), CastroDataset) + + +@requires_file(plasma) +def test_WarpXDataset(): + assert isinstance(data_dir_load(plasma), WarpXDataset) + + +@disable_dataset_cache +@requires_file(plasma) +def test_magnetic_units(): + ds1 = load(plasma) + assert_allclose(ds1.magnetic_unit.value, 1.0) + assert str(ds1.magnetic_unit.units) == "T" + mag_unit1 = ds1.magnetic_unit.to("code_magnetic") + assert_allclose(mag_unit1.value, 1.0) + assert str(mag_unit1.units) == "code_magnetic" + ds2 = load(plasma, unit_system="cgs") + assert_allclose(ds2.magnetic_unit.value, 1.0e4) + assert str(ds2.magnetic_unit.units) == "G" + mag_unit2 = ds2.magnetic_unit.to("code_magnetic") + assert_allclose(mag_unit2.value, 1.0) + assert str(mag_unit2.units) == "code_magnetic" + + +@requires_ds(laser) +def test_WarpXDataset_2(): + assert isinstance(data_dir_load(laser), WarpXDataset) + + +@requires_file("plt.Cavity00010") +def test_AMReXDataset(): + ds = data_dir_load("plt.Cavity00010", kwargs={"cparam_filename": "inputs"}) + assert isinstance(ds, AMReXDataset) + + +@requires_file(rt) +def test_units_override(): + units_override_check(rt) + + +nyx_no_particles = "nyx_sedov_plt00086" + + +@requires_file(nyx_no_particles) +def test_nyx_no_part(): + assert isinstance(data_dir_load(nyx_no_particles), NyxDataset) + + fields = sorted( + [ + ("boxlib", "H"), + ("boxlib", "He"), + ("boxlib", "MachNumber"), + ("boxlib", "Ne"), + ("boxlib", "Rank"), + ("boxlib", "StateErr"), + ("boxlib", "Temp"), + ("boxlib", "X(H)"), + ("boxlib", "X(He)"), + ("boxlib", "density"), + ("boxlib", "divu"), + ("boxlib", "eint_E"), + ("boxlib", "eint_e"), + ("boxlib", "entropy"), + ("boxlib", "forcex"), + ("boxlib", "forcey"), + ("boxlib", "forcez"), + ("boxlib", "kineng"), + ("boxlib", "logden"), + ("boxlib", "magmom"), + ("boxlib", "magvel"), + ("boxlib", "magvort"), + ("boxlib", "pressure"), + ("boxlib", "rho_E"), + ("boxlib", "rho_H"), + ("boxlib", "rho_He"), + ("boxlib", "rho_e"), + ("boxlib", "soundspeed"), + ("boxlib", "x_velocity"), + ("boxlib", "xmom"), + ("boxlib", "y_velocity"), + ("boxlib", "ymom"), + ("boxlib", "z_velocity"), + ("boxlib", "zmom"), + ] + ) + + ds = data_dir_load(nyx_no_particles) + assert_equal(sorted(ds.field_list), fields) + + +msubch = "maestro_subCh_plt00248" + + +@requires_file(msubch) +def test_maestro_parameters(): + assert isinstance(data_dir_load(msubch), MaestroDataset) + ds = data_dir_load(msubch) + + # Check a string parameter + assert ds.parameters["plot_base_name"] == "subCh_hot_baserun_plt" + assert type(ds.parameters["plot_base_name"]) is str # noqa: E721 + + # Check boolean parameters: T or F + assert not ds.parameters["use_thermal_diffusion"] + assert type(ds.parameters["use_thermal_diffusion"]) is bool # noqa: E721 + + assert ds.parameters["do_burning"] + assert type(ds.parameters["do_burning"]) is bool # noqa: E721 + + # Check a float parameter with a decimal point + assert ds.parameters["sponge_kappa"] == float("10.00000000") + assert type(ds.parameters["sponge_kappa"]) is float # noqa: E721 + + # Check a float parameter with E exponent notation + assert ds.parameters["small_dt"] == float("0.1000000000E-09") + + # Check an int parameter + assert ds.parameters["s0_interp_type"] == 3 + assert type(ds.parameters["s0_interp_type"]) is int # noqa: E721 diff --git a/yt/frontends/boxlib/__init__.py b/yt/frontends/boxlib/__init__.py index e69de29bb2..6f14e2d512 100644 --- a/yt/frontends/boxlib/__init__.py +++ b/yt/frontends/boxlib/__init__.py @@ -0,0 +1,17 @@ +""" + +import sys + +from .. import amrex + +sys.module[__name__] = __import__('amrex') + +sys.module[__name__.data_structures] = __import__('amrex.data_structures') + +sys.module[__name__.tests] = __import__('amrex.tests') + +sys.module[__name__.fields] = __import__('amrex.fields') + +sys.module[__name__.io] = __import__('amrex.io') + +""" \ No newline at end of file diff --git a/yt/frontends/boxlib/api.py b/yt/frontends/boxlib/api.py index 334cc8bb63..baa3adc8a8 100644 --- a/yt/frontends/boxlib/api.py +++ b/yt/frontends/boxlib/api.py @@ -1,5 +1,6 @@ -from . import tests -from .data_structures import ( +from ..amrex import tests +from ..amrex import data_structures +from ..amrex.data_structures import ( AMReXDataset, AMReXHierarchy, BoxlibDataset, @@ -14,11 +15,11 @@ WarpXDataset, WarpXHierarchy, ) -from .fields import ( +from ..amrex.fields import ( BoxlibFieldInfo, CastroFieldInfo, MaestroFieldInfo, NyxFieldInfo, WarpXFieldInfo, ) -from .io import IOHandlerBoxlib +from ..amrex.io import IOHandlerBoxlib \ No newline at end of file diff --git a/yt/frontends/boxlib/data_structures/__init__.py b/yt/frontends/boxlib/data_structures/__init__.py new file mode 100644 index 0000000000..4e9b15b50c --- /dev/null +++ b/yt/frontends/boxlib/data_structures/__init__.py @@ -0,0 +1,15 @@ +from ...amrex.data_structures import ( + AMReXDataset, + AMReXHierarchy, + BoxlibDataset, + BoxlibGrid, + BoxlibHierarchy, + CastroDataset, + MaestroDataset, + NyxDataset, + NyxHierarchy, + OrionDataset, + OrionHierarchy, + WarpXDataset, + WarpXHierarchy, +) diff --git a/yt/frontends/boxlib/fields/__init__.py b/yt/frontends/boxlib/fields/__init__.py new file mode 100644 index 0000000000..7ce9100ba1 --- /dev/null +++ b/yt/frontends/boxlib/fields/__init__.py @@ -0,0 +1,7 @@ +from ...amrex.fields import ( + BoxlibFieldInfo, + CastroFieldInfo, + MaestroFieldInfo, + NyxFieldInfo, + WarpXFieldInfo, +) \ No newline at end of file diff --git a/yt/frontends/boxlib/io/__init__.py b/yt/frontends/boxlib/io/__init__.py new file mode 100644 index 0000000000..c45c52ad5d --- /dev/null +++ b/yt/frontends/boxlib/io/__init__.py @@ -0,0 +1 @@ +from ...amrex.io import IOHandlerBoxlib \ No newline at end of file diff --git a/yt/frontends/boxlib/tests/__init__.py b/yt/frontends/boxlib/tests/__init__.py index e69de29bb2..f5c9a51350 100644 --- a/yt/frontends/boxlib/tests/__init__.py +++ b/yt/frontends/boxlib/tests/__init__.py @@ -0,0 +1 @@ +from ...amrex import tests \ No newline at end of file From 4ddb8d9d27908a8fafb86f58acc07d1c5ab35369 Mon Sep 17 00:00:00 2001 From: Henry Jones Date: Tue, 5 Mar 2024 13:08:33 -0800 Subject: [PATCH 017/186] Updated inits --- yt/frontends/__init__.py | 1 + yt/frontends/boxlib/__init__.py | 17 ----------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/yt/frontends/__init__.py b/yt/frontends/__init__.py index 989c1deea5..a5b005c398 100644 --- a/yt/frontends/__init__.py +++ b/yt/frontends/__init__.py @@ -1,6 +1,7 @@ __all__ = [ "adaptahop", "ahf", + "amrex", "amrvac", "art", "artio", diff --git a/yt/frontends/boxlib/__init__.py b/yt/frontends/boxlib/__init__.py index 6f14e2d512..e69de29bb2 100644 --- a/yt/frontends/boxlib/__init__.py +++ b/yt/frontends/boxlib/__init__.py @@ -1,17 +0,0 @@ -""" - -import sys - -from .. import amrex - -sys.module[__name__] = __import__('amrex') - -sys.module[__name__.data_structures] = __import__('amrex.data_structures') - -sys.module[__name__.tests] = __import__('amrex.tests') - -sys.module[__name__.fields] = __import__('amrex.fields') - -sys.module[__name__.io] = __import__('amrex.io') - -""" \ No newline at end of file From 0abc89a61f3d94f440e6564e7a08b7e6880c45a5 Mon Sep 17 00:00:00 2001 From: Henry Jones Date: Tue, 5 Mar 2024 14:34:24 -0800 Subject: [PATCH 018/186] Fixed test_outputs import --- yt/frontends/amrex/tests/test_outputs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/frontends/amrex/tests/test_outputs.py b/yt/frontends/amrex/tests/test_outputs.py index d3e9e5d91e..bfde30d0fb 100644 --- a/yt/frontends/amrex/tests/test_outputs.py +++ b/yt/frontends/amrex/tests/test_outputs.py @@ -1,7 +1,7 @@ import numpy as np from numpy.testing import assert_allclose, assert_equal -from yt.frontends.boxlib.api import ( +from yt.frontends.amrex.api import ( AMReXDataset, CastroDataset, MaestroDataset, From 4a4208813b57ef578190b1d5701e50e22a471f34 Mon Sep 17 00:00:00 2001 From: Henry Jones Date: Fri, 8 Mar 2024 11:56:15 -0800 Subject: [PATCH 019/186] Issue deprecation warnings --- yt/frontends/boxlib/api.py | 2 +- yt/frontends/boxlib/data_structures/__init__.py | 9 +++++++++ yt/frontends/boxlib/fields/__init__.py | 11 ++++++++++- yt/frontends/boxlib/io/__init__.py | 11 ++++++++++- yt/frontends/boxlib/tests/__init__.py | 11 ++++++++++- 5 files changed, 40 insertions(+), 4 deletions(-) diff --git a/yt/frontends/boxlib/api.py b/yt/frontends/boxlib/api.py index baa3adc8a8..a4005ca658 100644 --- a/yt/frontends/boxlib/api.py +++ b/yt/frontends/boxlib/api.py @@ -22,4 +22,4 @@ NyxFieldInfo, WarpXFieldInfo, ) -from ..amrex.io import IOHandlerBoxlib \ No newline at end of file +from ..amrex.io import IOHandlerBoxlib diff --git a/yt/frontends/boxlib/data_structures/__init__.py b/yt/frontends/boxlib/data_structures/__init__.py index 4e9b15b50c..f0abb20ac5 100644 --- a/yt/frontends/boxlib/data_structures/__init__.py +++ b/yt/frontends/boxlib/data_structures/__init__.py @@ -13,3 +13,12 @@ WarpXDataset, WarpXHierarchy, ) +from yt._maintenance.deprecation import issue_deprecation_warning + +issue_deprecation_warning( + "The historic 'boxlib' frontend is \n" + "deprecated as it has been renamed 'amrex'. " + "Future work should reference the 'amrex' frontend.", + stacklevel=3, + since="TBD", +) diff --git a/yt/frontends/boxlib/fields/__init__.py b/yt/frontends/boxlib/fields/__init__.py index 7ce9100ba1..736696e857 100644 --- a/yt/frontends/boxlib/fields/__init__.py +++ b/yt/frontends/boxlib/fields/__init__.py @@ -4,4 +4,13 @@ MaestroFieldInfo, NyxFieldInfo, WarpXFieldInfo, -) \ No newline at end of file +) +from yt._maintenance.deprecation import issue_deprecation_warning + +issue_deprecation_warning( + "The historic 'boxlib' frontend is \n" + "deprecated as it has been renamed 'amrex'. " + "Future work should reference the 'amrex' frontend.", + stacklevel=3, + since="TBD", +) diff --git a/yt/frontends/boxlib/io/__init__.py b/yt/frontends/boxlib/io/__init__.py index c45c52ad5d..92ee4a19d4 100644 --- a/yt/frontends/boxlib/io/__init__.py +++ b/yt/frontends/boxlib/io/__init__.py @@ -1 +1,10 @@ -from ...amrex.io import IOHandlerBoxlib \ No newline at end of file +from ...amrex.io import IOHandlerBoxlib +from yt._maintenance.deprecation import issue_deprecation_warning + +issue_deprecation_warning( + "The historic 'boxlib' frontend is \n" + "deprecated as it has been renamed 'amrex'. " + "Future work should reference the 'amrex' frontend.", + stacklevel=3, + since="TBD", +) diff --git a/yt/frontends/boxlib/tests/__init__.py b/yt/frontends/boxlib/tests/__init__.py index f5c9a51350..8ec892c1d3 100644 --- a/yt/frontends/boxlib/tests/__init__.py +++ b/yt/frontends/boxlib/tests/__init__.py @@ -1 +1,10 @@ -from ...amrex import tests \ No newline at end of file +from ...amrex import tests +from yt._maintenance.deprecation import issue_deprecation_warning + +issue_deprecation_warning( + "The historic 'boxlib' frontend is \n" + "deprecated as it has been renamed 'amrex'. " + "Future work should reference the 'amrex' frontend.", + stacklevel=3, + since="TBD", +) From 5b8b1b42e37e257670a66f343f91340bedb81861 Mon Sep 17 00:00:00 2001 From: Henry Jones Date: Mon, 11 Mar 2024 14:14:10 -0700 Subject: [PATCH 020/186] Refactor and CI tasks --- pyproject.toml | 3 +++ tests/tests.yaml | 22 +++++++++++++++++++ yt/frontends/__init__.py | 2 +- .../boxlib/data_structures/__init__.py | 10 ++------- yt/frontends/boxlib/deprecation.py | 13 +++++++++++ yt/frontends/boxlib/fields/__init__.py | 10 ++------- yt/frontends/boxlib/io/__init__.py | 10 ++------- yt/frontends/boxlib/tests/__init__.py | 10 ++------- 8 files changed, 47 insertions(+), 33 deletions(-) create mode 100644 yt/frontends/boxlib/deprecation.py diff --git a/pyproject.toml b/pyproject.toml index 168c11aed2..0fe215c7a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,6 +89,7 @@ Fortran = ["f90nml>=1.1"] # We also normalize all target names to lower case for consistency. adaptahop = [] ahf = [] +amrex = [] amrvac = ["yt[Fortran]"] art = [] arepo = ["yt[HDF5]"] @@ -146,6 +147,7 @@ full = [ "ratarmount~=0.8.1;platform_system!='Windows' and platform_system!='Darwin'", "yt[adaptahop]", "yt[ahf]", + "yt[amrex]", "yt[amrvac]", "yt[art]", "yt[arepo]", @@ -348,6 +350,7 @@ addopts = ''' --ignore='yt/fields/tests/test_xray_fields.py' --ignore='yt/frontends/adaptahop/tests/test_outputs.py' --ignore='yt/frontends/ahf/tests/test_outputs.py' + --ignore='yt/frontends/amrex/tests/test_outputs.py' --ignore='yt/frontends/amrvac/tests/test_outputs.py' --ignore='yt/frontends/amrvac/tests/test_units_override.py' --ignore='yt/frontends/arepo/tests/test_outputs.py' diff --git a/tests/tests.yaml b/tests/tests.yaml index 5ef62862ac..b10873b901 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -3,6 +3,28 @@ answer_tests: local_art_004: # PR 3081, 3101 - yt/frontends/art/tests/test_outputs.py:test_d9p +#copied from boxlib frontend + local_amrex_012: + - yt/frontends/amrex/tests/test_outputs.py:test_radadvect + - yt/frontends/amrex/tests/test_outputs.py:test_radtube + - yt/frontends/amrex/tests/test_outputs.py:test_star + - yt/frontends/amrex/tests/test_outputs.py:test_OrionDataset + - yt/frontends/amrex/tests/test_outputs.py:test_CastroDataset + - yt/frontends/amrex/tests/test_outputs.py:test_RT_particles + - yt/frontends/amrex/tests/test_outputs.py:test_units_override + - yt/frontends/amrex/tests/test_outputs.py:test_raw_fields + + local_amrex_particles_010: + - yt/frontends/amrex/tests/test_outputs.py:test_LyA + - yt/frontends/amrex/tests/test_outputs.py:test_nyx_particle_io + - yt/frontends/amrex/tests/test_outputs.py:test_castro_particle_io + - yt/frontends/amrex/tests/test_outputs.py:test_langmuir + - yt/frontends/amrex/tests/test_outputs.py:test_plasma + - yt/frontends/amrex/tests/test_outputs.py:test_beam + - yt/frontends/amrex/tests/test_outputs.py:test_warpx_particle_io + - yt/frontends/amrex/tests/test_outputs.py:test_NyxDataset + - yt/frontends/amrex/tests/test_outputs.py:test_WarpXDataset + local_amrvac_009: # PR 2945 - yt/frontends/amrvac/tests/test_outputs.py:test_domain_size - yt/frontends/amrvac/tests/test_outputs.py:test_bw_polar_2d diff --git a/yt/frontends/__init__.py b/yt/frontends/__init__.py index a5b005c398..4a5d6a2ff7 100644 --- a/yt/frontends/__init__.py +++ b/yt/frontends/__init__.py @@ -7,7 +7,7 @@ "artio", "athena", "athena_pp", - "boxlib", + "boxlib", # the boxlib frontend is deprecated, use 'amrex' "cf_radial", "chimera", "chombo", diff --git a/yt/frontends/boxlib/data_structures/__init__.py b/yt/frontends/boxlib/data_structures/__init__.py index f0abb20ac5..5fc7e6874b 100644 --- a/yt/frontends/boxlib/data_structures/__init__.py +++ b/yt/frontends/boxlib/data_structures/__init__.py @@ -13,12 +13,6 @@ WarpXDataset, WarpXHierarchy, ) -from yt._maintenance.deprecation import issue_deprecation_warning +from ..deprecation import boxlib_deprecation -issue_deprecation_warning( - "The historic 'boxlib' frontend is \n" - "deprecated as it has been renamed 'amrex'. " - "Future work should reference the 'amrex' frontend.", - stacklevel=3, - since="TBD", -) +boxlib_deprecation() diff --git a/yt/frontends/boxlib/deprecation.py b/yt/frontends/boxlib/deprecation.py new file mode 100644 index 0000000000..bfc25f9be9 --- /dev/null +++ b/yt/frontends/boxlib/deprecation.py @@ -0,0 +1,13 @@ +from yt._maintenance.deprecation import issue_deprecation_warning, warnings + + +def boxlib_deprecation(): + with warnings.catch_warnings(): + warnings.simplefilter("default") + issue_deprecation_warning( + "The historic 'boxlib' frontend is \n" + "deprecated as it has been renamed 'amrex'. " + "Existing and future work should instead reference the 'amrex' frontend.", + stacklevel=3, + since="TBD", + ) diff --git a/yt/frontends/boxlib/fields/__init__.py b/yt/frontends/boxlib/fields/__init__.py index 736696e857..822830bfbb 100644 --- a/yt/frontends/boxlib/fields/__init__.py +++ b/yt/frontends/boxlib/fields/__init__.py @@ -5,12 +5,6 @@ NyxFieldInfo, WarpXFieldInfo, ) -from yt._maintenance.deprecation import issue_deprecation_warning +from ..deprecation import boxlib_deprecation -issue_deprecation_warning( - "The historic 'boxlib' frontend is \n" - "deprecated as it has been renamed 'amrex'. " - "Future work should reference the 'amrex' frontend.", - stacklevel=3, - since="TBD", -) +boxlib_deprecation() diff --git a/yt/frontends/boxlib/io/__init__.py b/yt/frontends/boxlib/io/__init__.py index 92ee4a19d4..620cc02113 100644 --- a/yt/frontends/boxlib/io/__init__.py +++ b/yt/frontends/boxlib/io/__init__.py @@ -1,10 +1,4 @@ from ...amrex.io import IOHandlerBoxlib -from yt._maintenance.deprecation import issue_deprecation_warning +from ..deprecation import boxlib_deprecation -issue_deprecation_warning( - "The historic 'boxlib' frontend is \n" - "deprecated as it has been renamed 'amrex'. " - "Future work should reference the 'amrex' frontend.", - stacklevel=3, - since="TBD", -) +boxlib_deprecation() diff --git a/yt/frontends/boxlib/tests/__init__.py b/yt/frontends/boxlib/tests/__init__.py index 8ec892c1d3..2e9f12a2a6 100644 --- a/yt/frontends/boxlib/tests/__init__.py +++ b/yt/frontends/boxlib/tests/__init__.py @@ -1,10 +1,4 @@ from ...amrex import tests -from yt._maintenance.deprecation import issue_deprecation_warning +from ..deprecation import boxlib_deprecation -issue_deprecation_warning( - "The historic 'boxlib' frontend is \n" - "deprecated as it has been renamed 'amrex'. " - "Future work should reference the 'amrex' frontend.", - stacklevel=3, - since="TBD", -) +boxlib_deprecation() From 07c39887921ee6d508d7a00c976bc1db32eea09a Mon Sep 17 00:00:00 2001 From: Henry Jones Date: Mon, 11 Mar 2024 14:42:23 -0700 Subject: [PATCH 021/186] Made deprecation module private --- yt/frontends/boxlib/_deprecation.py | 13 +++++++++++++ yt/frontends/boxlib/data_structures/__init__.py | 2 +- yt/frontends/boxlib/fields/__init__.py | 2 +- yt/frontends/boxlib/io/__init__.py | 2 +- yt/frontends/boxlib/tests/__init__.py | 2 +- 5 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 yt/frontends/boxlib/_deprecation.py diff --git a/yt/frontends/boxlib/_deprecation.py b/yt/frontends/boxlib/_deprecation.py new file mode 100644 index 0000000000..bfc25f9be9 --- /dev/null +++ b/yt/frontends/boxlib/_deprecation.py @@ -0,0 +1,13 @@ +from yt._maintenance.deprecation import issue_deprecation_warning, warnings + + +def boxlib_deprecation(): + with warnings.catch_warnings(): + warnings.simplefilter("default") + issue_deprecation_warning( + "The historic 'boxlib' frontend is \n" + "deprecated as it has been renamed 'amrex'. " + "Existing and future work should instead reference the 'amrex' frontend.", + stacklevel=3, + since="TBD", + ) diff --git a/yt/frontends/boxlib/data_structures/__init__.py b/yt/frontends/boxlib/data_structures/__init__.py index 5fc7e6874b..253f1c6dc3 100644 --- a/yt/frontends/boxlib/data_structures/__init__.py +++ b/yt/frontends/boxlib/data_structures/__init__.py @@ -13,6 +13,6 @@ WarpXDataset, WarpXHierarchy, ) -from ..deprecation import boxlib_deprecation +from .._deprecation import boxlib_deprecation boxlib_deprecation() diff --git a/yt/frontends/boxlib/fields/__init__.py b/yt/frontends/boxlib/fields/__init__.py index 822830bfbb..4eaa65a9b2 100644 --- a/yt/frontends/boxlib/fields/__init__.py +++ b/yt/frontends/boxlib/fields/__init__.py @@ -5,6 +5,6 @@ NyxFieldInfo, WarpXFieldInfo, ) -from ..deprecation import boxlib_deprecation +from .._deprecation import boxlib_deprecation boxlib_deprecation() diff --git a/yt/frontends/boxlib/io/__init__.py b/yt/frontends/boxlib/io/__init__.py index 620cc02113..2c74949cfe 100644 --- a/yt/frontends/boxlib/io/__init__.py +++ b/yt/frontends/boxlib/io/__init__.py @@ -1,4 +1,4 @@ from ...amrex.io import IOHandlerBoxlib -from ..deprecation import boxlib_deprecation +from .._deprecation import boxlib_deprecation boxlib_deprecation() diff --git a/yt/frontends/boxlib/tests/__init__.py b/yt/frontends/boxlib/tests/__init__.py index 2e9f12a2a6..bbbe0ab192 100644 --- a/yt/frontends/boxlib/tests/__init__.py +++ b/yt/frontends/boxlib/tests/__init__.py @@ -1,4 +1,4 @@ from ...amrex import tests -from ..deprecation import boxlib_deprecation +from .._deprecation import boxlib_deprecation boxlib_deprecation() From d2cb8857a512e5b0ffd23abbb08fb82ad1188418 Mon Sep 17 00:00:00 2001 From: Henry Jones Date: Mon, 11 Mar 2024 18:03:17 -0700 Subject: [PATCH 022/186] Set deprecation warning filter to always --- yt/frontends/boxlib/_deprecation.py | 2 +- yt/frontends/boxlib/deprecation.py | 13 ------------- 2 files changed, 1 insertion(+), 14 deletions(-) delete mode 100644 yt/frontends/boxlib/deprecation.py diff --git a/yt/frontends/boxlib/_deprecation.py b/yt/frontends/boxlib/_deprecation.py index bfc25f9be9..7dae20a305 100644 --- a/yt/frontends/boxlib/_deprecation.py +++ b/yt/frontends/boxlib/_deprecation.py @@ -3,7 +3,7 @@ def boxlib_deprecation(): with warnings.catch_warnings(): - warnings.simplefilter("default") + warnings.simplefilter("always") issue_deprecation_warning( "The historic 'boxlib' frontend is \n" "deprecated as it has been renamed 'amrex'. " diff --git a/yt/frontends/boxlib/deprecation.py b/yt/frontends/boxlib/deprecation.py deleted file mode 100644 index bfc25f9be9..0000000000 --- a/yt/frontends/boxlib/deprecation.py +++ /dev/null @@ -1,13 +0,0 @@ -from yt._maintenance.deprecation import issue_deprecation_warning, warnings - - -def boxlib_deprecation(): - with warnings.catch_warnings(): - warnings.simplefilter("default") - issue_deprecation_warning( - "The historic 'boxlib' frontend is \n" - "deprecated as it has been renamed 'amrex'. " - "Existing and future work should instead reference the 'amrex' frontend.", - stacklevel=3, - since="TBD", - ) From 7e7e4665b38e6e4a9167ae4cb6e79f265f8989d2 Mon Sep 17 00:00:00 2001 From: Henry Jones Date: Tue, 19 Mar 2024 13:21:36 -0700 Subject: [PATCH 023/186] Added import test, removed warning catch in _deprecation --- yt/frontends/boxlib/_deprecation.py | 18 ++++++++--------- .../boxlib/tests/test_boxlib_deprecation.py | 20 +++++++++++++++++++ 2 files changed, 29 insertions(+), 9 deletions(-) create mode 100644 yt/frontends/boxlib/tests/test_boxlib_deprecation.py diff --git a/yt/frontends/boxlib/_deprecation.py b/yt/frontends/boxlib/_deprecation.py index 7dae20a305..bc1c302d44 100644 --- a/yt/frontends/boxlib/_deprecation.py +++ b/yt/frontends/boxlib/_deprecation.py @@ -2,12 +2,12 @@ def boxlib_deprecation(): - with warnings.catch_warnings(): - warnings.simplefilter("always") - issue_deprecation_warning( - "The historic 'boxlib' frontend is \n" - "deprecated as it has been renamed 'amrex'. " - "Existing and future work should instead reference the 'amrex' frontend.", - stacklevel=3, - since="TBD", - ) + warnings.simplefilter("always") + issue_deprecation_warning( + "The historic 'boxlib' frontend is \n" + "deprecated as it has been renamed 'amrex'. " + "Existing and future work should instead reference the 'amrex' frontend.", + stacklevel=3, + since="TBD", + ) + warnings.resetwarnings() diff --git a/yt/frontends/boxlib/tests/test_boxlib_deprecation.py b/yt/frontends/boxlib/tests/test_boxlib_deprecation.py new file mode 100644 index 0000000000..9686bc11a5 --- /dev/null +++ b/yt/frontends/boxlib/tests/test_boxlib_deprecation.py @@ -0,0 +1,20 @@ +from yt._maintenance.deprecation import warnings + + +def test_imports(): + with warnings.catch_warnings(record=True) as w: + from yt.frontends.boxlib import ( + data_structures, + fields, + io, + ) + + assert len(w) == 3 and all( + [ + issubclass(w[0].category, DeprecationWarning), + issubclass(w[1].category, DeprecationWarning), + issubclass(w[2].category, DeprecationWarning), + ] + ) + del (data_structures, io, fields) + warnings.resetwarnings() From c5dee4b0f36cdec5b6a99189d37a1900ceaf36be Mon Sep 17 00:00:00 2001 From: Henry Jones Date: Fri, 22 Mar 2024 11:59:24 -0700 Subject: [PATCH 024/186] Make 'boxlib' tests minimal --- tests/tests.yaml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/tests.yaml b/tests/tests.yaml index b10873b901..208a78f374 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -143,23 +143,8 @@ answer_tests: local_boxlib_012: - yt/frontends/boxlib/tests/test_outputs.py:test_radadvect - - yt/frontends/boxlib/tests/test_outputs.py:test_radtube - - yt/frontends/boxlib/tests/test_outputs.py:test_star - - yt/frontends/boxlib/tests/test_outputs.py:test_OrionDataset - - yt/frontends/boxlib/tests/test_outputs.py:test_CastroDataset - - yt/frontends/boxlib/tests/test_outputs.py:test_RT_particles - - yt/frontends/boxlib/tests/test_outputs.py:test_units_override - - yt/frontends/boxlib/tests/test_outputs.py:test_raw_fields local_boxlib_particles_010: - - yt/frontends/boxlib/tests/test_outputs.py:test_LyA - - yt/frontends/boxlib/tests/test_outputs.py:test_nyx_particle_io - - yt/frontends/boxlib/tests/test_outputs.py:test_castro_particle_io - - yt/frontends/boxlib/tests/test_outputs.py:test_langmuir - - yt/frontends/boxlib/tests/test_outputs.py:test_plasma - - yt/frontends/boxlib/tests/test_outputs.py:test_beam - - yt/frontends/boxlib/tests/test_outputs.py:test_warpx_particle_io - - yt/frontends/boxlib/tests/test_outputs.py:test_NyxDataset - yt/frontends/boxlib/tests/test_outputs.py:test_WarpXDataset local_ramses_005: # PR 3856 From a63ea77ddad62ac56f2b61863e3bd5025c393f2b Mon Sep 17 00:00:00 2001 From: Henry Jones Date: Fri, 29 Mar 2024 15:54:46 -0700 Subject: [PATCH 025/186] Make test_imports() reload modules --- .../boxlib/tests/test_boxlib_deprecation.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/yt/frontends/boxlib/tests/test_boxlib_deprecation.py b/yt/frontends/boxlib/tests/test_boxlib_deprecation.py index 9686bc11a5..dd3c5ce439 100644 --- a/yt/frontends/boxlib/tests/test_boxlib_deprecation.py +++ b/yt/frontends/boxlib/tests/test_boxlib_deprecation.py @@ -1,13 +1,15 @@ +from importlib import import_module, reload + from yt._maintenance.deprecation import warnings def test_imports(): with warnings.catch_warnings(record=True) as w: - from yt.frontends.boxlib import ( - data_structures, - fields, - io, - ) + warnings.simplefilter("always") + for index, mname in enumerate(["data_structures", "fields", "io"]): + mod_name = import_module("yt.frontends.boxlib." + mname) + if len(w) != index + 1: + reload(mod_name) assert len(w) == 3 and all( [ @@ -16,5 +18,4 @@ def test_imports(): issubclass(w[2].category, DeprecationWarning), ] ) - del (data_structures, io, fields) warnings.resetwarnings() From 6de10d59ef229e79eb9f6ed06e4831fa4a36d43a Mon Sep 17 00:00:00 2001 From: Henry N Jones Date: Wed, 1 May 2024 10:26:06 -0700 Subject: [PATCH 026/186] Add catch_warnings() --- yt/frontends/boxlib/_deprecation.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/yt/frontends/boxlib/_deprecation.py b/yt/frontends/boxlib/_deprecation.py index bc1c302d44..7dae20a305 100644 --- a/yt/frontends/boxlib/_deprecation.py +++ b/yt/frontends/boxlib/_deprecation.py @@ -2,12 +2,12 @@ def boxlib_deprecation(): - warnings.simplefilter("always") - issue_deprecation_warning( - "The historic 'boxlib' frontend is \n" - "deprecated as it has been renamed 'amrex'. " - "Existing and future work should instead reference the 'amrex' frontend.", - stacklevel=3, - since="TBD", - ) - warnings.resetwarnings() + with warnings.catch_warnings(): + warnings.simplefilter("always") + issue_deprecation_warning( + "The historic 'boxlib' frontend is \n" + "deprecated as it has been renamed 'amrex'. " + "Existing and future work should instead reference the 'amrex' frontend.", + stacklevel=3, + since="TBD", + ) From f91fffb68fb1ece883463d0325ed207ae21e6573 Mon Sep 17 00:00:00 2001 From: Henry N Jones Date: Thu, 9 May 2024 14:20:46 -0700 Subject: [PATCH 027/186] Removed tests deprecation and pointed deprecation to import --- yt/frontends/boxlib/_deprecation.py | 4 ++-- yt/frontends/boxlib/tests/__init__.py | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/yt/frontends/boxlib/_deprecation.py b/yt/frontends/boxlib/_deprecation.py index 7dae20a305..8303ed96bf 100644 --- a/yt/frontends/boxlib/_deprecation.py +++ b/yt/frontends/boxlib/_deprecation.py @@ -8,6 +8,6 @@ def boxlib_deprecation(): "The historic 'boxlib' frontend is \n" "deprecated as it has been renamed 'amrex'. " "Existing and future work should instead reference the 'amrex' frontend.", - stacklevel=3, - since="TBD", + stacklevel=4, + since="4.4.0", ) diff --git a/yt/frontends/boxlib/tests/__init__.py b/yt/frontends/boxlib/tests/__init__.py index bbbe0ab192..2ce3d37993 100644 --- a/yt/frontends/boxlib/tests/__init__.py +++ b/yt/frontends/boxlib/tests/__init__.py @@ -1,4 +1 @@ from ...amrex import tests -from .._deprecation import boxlib_deprecation - -boxlib_deprecation() From 537c578f963b8acb6f42237de73e936162d1291a Mon Sep 17 00:00:00 2001 From: Henry Jones Date: Fri, 5 Apr 2024 09:59:19 -0700 Subject: [PATCH 028/186] Updated pyproject dependency --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0fe215c7a7..fd95235356 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,7 +117,7 @@ http-stream = ["requests>=2.20.0"] idefix = ["yt_idefix[HDF5]>=2.3.0"] # externally packaged frontend moab = ["yt[HDF5]"] nc4-cm1 = ["yt[netCDF4]"] -open-pmd = ["yt[HDF5]"] +open-pmd = ["yt[openpmd_api]"] owls = ["yt[HDF5]"] owls-subfind = ["yt[HDF5]"] ramses = ["yt[Fortran]"] From 9da50dbcea5f53c48f8669d0b515908c29f0c8e7 Mon Sep 17 00:00:00 2001 From: Henry N Jones Date: Wed, 15 May 2024 12:10:04 -0700 Subject: [PATCH 029/186] Fixed pyproject dependecy and removed warnings_reset --- pyproject.toml | 2 +- yt/frontends/boxlib/tests/test_boxlib_deprecation.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fd95235356..0fe215c7a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,7 +117,7 @@ http-stream = ["requests>=2.20.0"] idefix = ["yt_idefix[HDF5]>=2.3.0"] # externally packaged frontend moab = ["yt[HDF5]"] nc4-cm1 = ["yt[netCDF4]"] -open-pmd = ["yt[openpmd_api]"] +open-pmd = ["yt[HDF5]"] owls = ["yt[HDF5]"] owls-subfind = ["yt[HDF5]"] ramses = ["yt[Fortran]"] diff --git a/yt/frontends/boxlib/tests/test_boxlib_deprecation.py b/yt/frontends/boxlib/tests/test_boxlib_deprecation.py index dd3c5ce439..c54e68133a 100644 --- a/yt/frontends/boxlib/tests/test_boxlib_deprecation.py +++ b/yt/frontends/boxlib/tests/test_boxlib_deprecation.py @@ -18,4 +18,3 @@ def test_imports(): issubclass(w[2].category, DeprecationWarning), ] ) - warnings.resetwarnings() From 8511d6959dbf943d1461df607af495e2bf5a4ac7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Fri, 17 May 2024 16:02:36 +0200 Subject: [PATCH 030/186] BLD: minimally test wheels and sdist --- .github/workflows/wheels.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index 41340b78b8..e61064f6bf 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -44,6 +44,8 @@ jobs: CIBW_ARCHS_WINDOWS: auto64 CIBW_ENVIRONMENT: "LDFLAGS='-static-libstdc++'" CIBW_BUILD_VERBOSITY: 1 + CIBW_TEST_SKIP: "*-manylinux*" # https://github.com/yt-project/yt/issues/4910 + CIBW_TEST_COMMAND: python -c "import yt" - uses: actions/upload-artifact@v4 with: @@ -57,9 +59,22 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.9' + - name: Build sdist run: pipx run build --sdist + - name: Test sdist + run: | + python -m pip install "$(echo dist/*.tar.gz)" + python -m pip list + project_dir=$(pwd) + cd ../../ + python -c "import yt" + - name: Upload sdist uses: actions/upload-artifact@v4 with: From 36a897f9b89404d641c0c65b3eb8ad598df138f3 Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Tue, 21 May 2024 13:29:51 +0200 Subject: [PATCH 031/186] BUG: Make sure fields are read in the order they appear in the file (#4907) Co-authored-by: Corentin Cadiou --- yt/frontends/ramses/io_utils.pyx | 3 ++ yt/frontends/ramses/tests/test_outputs.py | 39 +++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/yt/frontends/ramses/io_utils.pyx b/yt/frontends/ramses/io_utils.pyx index 69368b086c..643099a170 100644 --- a/yt/frontends/ramses/io_utils.pyx +++ b/yt/frontends/ramses/io_utils.pyx @@ -192,6 +192,9 @@ def fill_hydro(FortranFile f, cdef int jump_len, Ncells cdef np.uint8_t[::1] mask_level = np.zeros(nlevels, dtype=np.uint8) + # First, make sure fields are in the same order + fields = sorted(fields, key=lambda f: all_fields.index(f)) + # The ordering is very important here, as we'll write directly into the memory # address the content of the files. cdef np.float64_t[::1, :, :] buffer diff --git a/yt/frontends/ramses/tests/test_outputs.py b/yt/frontends/ramses/tests/test_outputs.py index a1ed589995..064c57b78f 100644 --- a/yt/frontends/ramses/tests/test_outputs.py +++ b/yt/frontends/ramses/tests/test_outputs.py @@ -644,3 +644,42 @@ def test_print_stats(): ds.print_stats() # FIXME #3197: use `capsys` with pytest to make sure the print_stats function works as intended + + +@requires_file(output_00080) +def test_reading_order(): + # This checks the bug unvovered in #4880 + # This checks that the result of field accession doesn't + # depend on the order + + def _dummy_field(field, data): + # Note: this is a dummy field + # that doesn't really have any physical meaning + # but may trigger some bug in the field + # handling. + T = data["gas", "temperature"] + Z = data["gas", "metallicity"] + return T * 1**Z + + fields = [ + "Density", + "x-velocity", + "y-velocity", + "z-velocity", + "Pressure", + "Metallicity", + ] + ds = yt.load(output_00080, fields=fields) + + ds.add_field( + ("gas", "test"), function=_dummy_field, units=None, sampling_type="cell" + ) + + ad = ds.all_data() + v0 = ad["gas", "test"] + + ad = ds.all_data() + ad["gas", "temperature"] + v1 = ad["gas", "test"] + + np.testing.assert_allclose(v0, v1) From 2068504d5ec1cae4ace655e8f957d340b3e72c1a Mon Sep 17 00:00:00 2001 From: yut23 Date: Tue, 21 May 2024 16:46:21 -0400 Subject: [PATCH 032/186] BLD: fix segfault with manylinux2014 The cibuildwheel docs say that manylinux2014 supports all C++ standards up to C++17 (https://cibuildwheel.pypa.io/en/stable/cpp_standards/). Fixes: yt-project/yt#4910 --- .github/workflows/wheels.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index e61064f6bf..ed9a7e2585 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -42,9 +42,7 @@ jobs: CIBW_ARCHS_MACOS: auto MACOSX_DEPLOYMENT_TARGET: "10.9" # as of CIBW 2.9, this is the default value, pin it so it can't be bumped silently CIBW_ARCHS_WINDOWS: auto64 - CIBW_ENVIRONMENT: "LDFLAGS='-static-libstdc++'" CIBW_BUILD_VERBOSITY: 1 - CIBW_TEST_SKIP: "*-manylinux*" # https://github.com/yt-project/yt/issues/4910 CIBW_TEST_COMMAND: python -c "import yt" - uses: actions/upload-artifact@v4 From 83ec7d9377921e6176b5ae2408d7ed3668f57436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Fri, 17 May 2024 10:43:20 +0200 Subject: [PATCH 033/186] BLD: exclude Cython generated `.c` and `.cpp` files from build artifacts --- MANIFEST.in | 47 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 14e2df79d9..e1e1eb29d5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,9 +8,54 @@ include yt/utilities/mesh_types.yaml exclude yt/utilities/lib/cykdtree/c_kdtree.cpp prune tests prune answer-store -recursive-include yt *.py *.pyx *.pxi *.pxd *.h *.hpp README* *.txt LICENSE* *.cu +recursive-include yt *.py *.pyx *.pxi *.pxd README* *.txt LICENSE* *.cu recursive-include doc *.rst *.txt *.py *.ipynb *.png *.jpg *.css *.html recursive-include doc *.h *.c *.sh *.svgz *.pdf *.svg *.pyx + + +# start with excluding all C/C++ files +recursive-exclude yt *.h *.c *.hpp *.cpp + +# then include back every non-generated C/C++ source file +# the list can be generated by the following command +# git ls-files | grep -E '\.(h|c)(pp)?$' +include yt/frontends/artio/artio_headers/artio.c +include yt/frontends/artio/artio_headers/artio.h +include yt/frontends/artio/artio_headers/artio_endian.c +include yt/frontends/artio/artio_headers/artio_endian.h +include yt/frontends/artio/artio_headers/artio_file.c +include yt/frontends/artio/artio_headers/artio_grid.c +include yt/frontends/artio/artio_headers/artio_internal.h +include yt/frontends/artio/artio_headers/artio_mpi.c +include yt/frontends/artio/artio_headers/artio_mpi.h +include yt/frontends/artio/artio_headers/artio_parameter.c +include yt/frontends/artio/artio_headers/artio_particle.c +include yt/frontends/artio/artio_headers/artio_posix.c +include yt/frontends/artio/artio_headers/artio_selector.c +include yt/frontends/artio/artio_headers/artio_sfc.c +include yt/frontends/artio/artio_headers/cosmology.c +include yt/frontends/artio/artio_headers/cosmology.h +include yt/geometry/vectorized_ops.h +include yt/utilities/lib/_octree_raytracing.hpp +include yt/utilities/lib/cykdtree/c_kdtree.cpp +include yt/utilities/lib/cykdtree/c_kdtree.hpp +include yt/utilities/lib/cykdtree/c_utils.cpp +include yt/utilities/lib/cykdtree/c_utils.hpp +include yt/utilities/lib/cykdtree/windows/stdint.h +include yt/utilities/lib/endian_swap.h +include yt/utilities/lib/fixed_interpolator.cpp +include yt/utilities/lib/fixed_interpolator.hpp +include yt/utilities/lib/marching_cubes.h +include yt/utilities/lib/mesh_triangulation.h +include yt/utilities/lib/origami_tags.c +include yt/utilities/lib/origami_tags.h +include yt/utilities/lib/pixelization_constants.cpp +include yt/utilities/lib/pixelization_constants.hpp +include yt/utilities/lib/platform_dep.h +include yt/utilities/lib/platform_dep_math.hpp +include yt/utilities/lib/tsearch.c +include yt/utilities/lib/tsearch.h + include doc/README doc/activate doc/activate.csh doc/cheatsheet.tex exclude doc/cheatsheet.pdf include doc/extensions/README doc/Makefile From 6860676b23e4186b93d5954458265080431c21a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Tue, 28 May 2024 20:30:54 +0200 Subject: [PATCH 034/186] TST: cleanup unnecessary warning filter --- conftest.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/conftest.py b/conftest.py index 7c1ecb0959..f0786b909d 100644 --- a/conftest.py +++ b/conftest.py @@ -161,19 +161,6 @@ def pytest_configure(config): ":DeprecationWarning", ) - if find_spec("astropy") is not None: - # at the time of writing, astropy's wheels are behind numpy's latest - # version but this doesn't cause actual problems in our test suite - # last updated with astropy 5.0 + numpy 1.22 + pytest 6.2.5 - config.addinivalue_line( - "filterwarnings", - ( - "ignore:numpy.ndarray size changed, may indicate binary incompatibility. Expected " - r"(80 from C header, got 88|88 from C header, got 96|80 from C header, got 96)" - " from PyObject:RuntimeWarning" - ), - ) - if PANDAS_VERSION is not None and PANDAS_VERSION >= Version("2.2.0"): config.addinivalue_line( "filterwarnings", From accaf59712b2487c53bba6d33f8bb79b616ea0f4 Mon Sep 17 00:00:00 2001 From: "Eric T. Johnson" Date: Tue, 30 Apr 2024 20:40:51 -0400 Subject: [PATCH 035/186] BUG: avoid ResourceWarnings in the amrex frontend Use context managers where the line length allows, and manually close all other files that are opened outside a `with` statement. Fixes #4890. --- yt/frontends/amrex/data_structures.py | 77 ++++++++++++++++----------- 1 file changed, 45 insertions(+), 32 deletions(-) diff --git a/yt/frontends/amrex/data_structures.py b/yt/frontends/amrex/data_structures.py index 2725275896..59fab4c4c5 100644 --- a/yt/frontends/amrex/data_structures.py +++ b/yt/frontends/amrex/data_structures.py @@ -385,8 +385,10 @@ def _parse_index(self): default_ybounds = (0.0, np.pi) default_zbounds = (0.0, 2 * np.pi) else: + header_file.close() raise RuntimeError("Unknown BoxLib coordinate system.") if int(next(header_file)) != 0: + header_file.close() raise RuntimeError("INTERNAL ERROR! This should be a zero.") # each level is one group with ngrids on it. @@ -453,9 +455,11 @@ def _parse_index(self): go = self.grid(grid_counter + gi, int(offset), filename, self) go.Level = self.grid_levels[grid_counter + gi, :] = level self.grids.append(go) + level_header_file.close() grid_counter += ngrids # already read the filenames above... self.float_type = "float64" + header_file.close() def _cache_endianness(self, test_grid): """ @@ -543,6 +547,7 @@ def _count_grids(self): if len(line.split()) != 3: continue self.num_grids += int(line.split()[1]) + header_file.close() def _initialize_grid_arrays(self): super()._initialize_grid_arrays() @@ -698,7 +703,8 @@ def _is_valid(cls, filename, *args, cparam_filename=None, **kwargs): if cparam_filepath is None: return False - lines = [line.lower() for line in open(cparam_filepath).readlines()] + with open(cparam_filepath) as f: + lines = [line.lower() for line in f] return any(cls._subtype_keyword in line for line in lines) @classmethod @@ -734,39 +740,41 @@ def _parse_parameter_file(self): def _parse_cparams(self): if self.cparam_filename is None: return - for line in (line.split("#")[0].strip() for line in open(self.cparam_filename)): - try: - param, vals = (s.strip() for s in line.split("=")) - except ValueError: - continue - # Castro and Maestro mark overridden defaults with a "[*]" before - # the parameter name - param = param.removeprefix("[*]").strip() - if param == "amr.ref_ratio": - vals = self.refine_by = int(vals[0]) - elif param == "Prob.lo_bc": - vals = tuple(p == "1" for p in vals.split()) - assert len(vals) == self.dimensionality - periodicity = [False, False, False] # default to non periodic - periodicity[: self.dimensionality] = vals # fill in ndim parsed values - self._periodicity = tuple(periodicity) - elif param == "castro.use_comoving": - vals = self.cosmological_simulation = int(vals) - else: + with open(self.cparam_filename) as param_file: + for line in (line.split("#")[0].strip() for line in param_file): try: - vals = _guess_pcast(vals) - except (IndexError, ValueError): - # hitting an empty string or a comment - vals = None - self.parameters[param] = vals + param, vals = (s.strip() for s in line.split("=")) + except ValueError: + continue + # Castro and Maestro mark overridden defaults with a "[*]" + # before the parameter name + param = param.removeprefix("[*]").strip() + if param == "amr.ref_ratio": + vals = self.refine_by = int(vals[0]) + elif param == "Prob.lo_bc": + vals = tuple(p == "1" for p in vals.split()) + assert len(vals) == self.dimensionality + # default to non periodic + periodicity = [False, False, False] + # fill in ndim parsed values + periodicity[: self.dimensionality] = vals + self._periodicity = tuple(periodicity) + elif param == "castro.use_comoving": + vals = self.cosmological_simulation = int(vals) + else: + try: + vals = _guess_pcast(vals) + except (IndexError, ValueError): + # hitting an empty string or a comment + vals = None + self.parameters[param] = vals if getattr(self, "cosmological_simulation", 0) == 1: self.omega_lambda = self.parameters["comoving_OmL"] self.omega_matter = self.parameters["comoving_OmM"] self.hubble_constant = self.parameters["comoving_h"] - a_file = open(os.path.join(self.output_dir, "comoving_a")) - line = a_file.readline().strip() - a_file.close() + with open(os.path.join(self.output_dir, "comoving_a")) as a_file: + line = a_file.readline().strip() self.current_redshift = 1 / float(line) - 1 else: self.current_redshift = 0.0 @@ -783,7 +791,8 @@ def _parse_fparams(self): """ if self.fparam_filename is None: return - for line in (l for l in open(self.fparam_filename) if "=" in l): + param_file = open(self.fparam_filename) + for line in (l for l in param_file if "=" in l): param, vals = (v.strip() for v in line.split("=")) # Now, there are a couple different types of parameters. # Some will be where you only have floating point values, others @@ -798,6 +807,7 @@ def _parse_fparams(self): if len(vals) == 1: vals = vals[0] self.parameters[param] = vals + param_file.close() def _parse_header_file(self): """ @@ -845,6 +855,7 @@ def _parse_header_file(self): float(rf) / self.refine_by == int(float(rf) / self.refine_by) for rf in ref_factors ): + header_file.close() raise RuntimeError base_log = np.log2(self.refine_by) self.level_offsets = [0] # level 0 has to have 0 offset @@ -888,6 +899,7 @@ def _parse_header_file(self): try: geom_str = known_types[coordinate_type] except KeyError as err: + header_file.close() raise ValueError(f"Unknown BoxLib coord_type `{coordinate_type}`.") from err else: self.geometry = Geometry(geom_str) @@ -897,6 +909,8 @@ def _parse_header_file(self): dre[2] = 2.0 * np.pi self.domain_right_edge = dre + header_file.close() + def _set_code_unit_attributes(self): setdefaultattr(self, "length_unit", self.quan(1.0, "cm")) setdefaultattr(self, "mass_unit", self.quan(1.0, "g")) @@ -1289,9 +1303,8 @@ def _parse_parameter_file(self): # Read in the `comoving_a` file and parse the value. We should fix this # in the new Nyx output format... - a_file = open(os.path.join(self.output_dir, "comoving_a")) - a_string = a_file.readline().strip() - a_file.close() + with open(os.path.join(self.output_dir, "comoving_a")) as a_file: + a_string = a_file.readline().strip() # Set the scale factor and redshift self.cosmological_scale_factor = float(a_string) From 6051ad8f25daf6f90f18f9c064d345500efde88c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Thu, 30 May 2024 09:35:00 +0200 Subject: [PATCH 036/186] CLN: drop unused and duplicated parsing logic in AMRex frontend --- yt/frontends/amrex/fields.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/yt/frontends/amrex/fields.py b/yt/frontends/amrex/fields.py index 8ce65068a7..cf193326c5 100644 --- a/yt/frontends/amrex/fields.py +++ b/yt/frontends/amrex/fields.py @@ -1,5 +1,4 @@ import re -import string import numpy as np @@ -369,14 +368,6 @@ def setup_fluid_fields(self): function=func, units=self.ds.unit_system["density"], ) - # We know this will either have one letter, or two. - if field[3] in string.ascii_letters: - element, weight = field[2:4], field[4:-1] - else: - element, weight = field[2:3], field[3:-1] # NOQA - - # Here we can, later, add number density - # right now element and weight inferred above are unused class MaestroFieldInfo(FieldInfoContainer): @@ -491,20 +482,6 @@ def setup_fluid_fields(self): display_name=rf"\rho {tex_label}", ) - # Most of the time our species will be of the form - # element name + atomic weight (e.g. C12), but - # sometimes we make up descriptive names (e.g. ash) - if any(char.isdigit() for char in field): - # We know this will either have one letter, or two. - if field[3] in string.ascii_letters: - element, weight = field[2:4], field[4:-1] - else: - element, weight = field[2:3], field[3:-1] # NOQA - weight = int(weight) - - # Here we can, later, add number density using 'element' and - # 'weight' inferred above - elif field.startswith("omegadot("): nice_name, tex_label = _nice_species_name(field) display_name = rf"\dot{{\omega}}\left[{tex_label}\right]" From ba928a32d3266309379cf96f2a52de205f643627 Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Thu, 30 May 2024 15:09:40 -0500 Subject: [PATCH 037/186] add check for Sub_conventions for CFradial --- yt/frontends/cf_radial/data_structures.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt/frontends/cf_radial/data_structures.py b/yt/frontends/cf_radial/data_structures.py index 436517b300..25592e6a2d 100644 --- a/yt/frontends/cf_radial/data_structures.py +++ b/yt/frontends/cf_radial/data_structures.py @@ -311,10 +311,10 @@ def _is_valid(cls, filename: str, *args, **kwargs) -> bool: with nc4_file.open_ds(keepweakref=True) as ds: con = "Conventions" # the attribute to check for file conventions cons = "" # the value of the Conventions attribute - for c in [con, con.lower()]: + for c in [con, con.lower(), "Sub_" + con.lower()]: if hasattr(ds, c): cons += getattr(ds, c) - is_cfrad = "CF/Radial" in cons + is_cfrad = "CF/Radial" in cons or "CF-Radial" in cons except (OSError, AttributeError, ImportError): return False From 4cb2db2093184080874525f16c06b36520039d86 Mon Sep 17 00:00:00 2001 From: chavlin Date: Fri, 31 May 2024 10:30:29 -0500 Subject: [PATCH 038/186] add comment on string attributes in cf_radial _is_valid --- yt/frontends/cf_radial/data_structures.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/yt/frontends/cf_radial/data_structures.py b/yt/frontends/cf_radial/data_structures.py index 25592e6a2d..1c1309510d 100644 --- a/yt/frontends/cf_radial/data_structures.py +++ b/yt/frontends/cf_radial/data_structures.py @@ -310,6 +310,9 @@ def _is_valid(cls, filename: str, *args, **kwargs) -> bool: nc4_file = NetCDF4FileHandler(filename) with nc4_file.open_ds(keepweakref=True) as ds: con = "Conventions" # the attribute to check for file conventions + # note that the attributes here are potentially space- or + # comma-delimited strings, so we concatenate a single string + # to search for a substring. cons = "" # the value of the Conventions attribute for c in [con, con.lower(), "Sub_" + con.lower()]: if hasattr(ds, c): From 70ecfc8d943f475e154cb58fd4880d5571071411 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Jun 2024 05:15:33 +0000 Subject: [PATCH 039/186] Bump pypa/cibuildwheel in /.github/workflows in the actions group Bumps the actions group in /.github/workflows with 1 update: [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel). Updates `pypa/cibuildwheel` from 2.17.0 to 2.18.1 - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.17.0...v2.18.1) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/wheels.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index ed9a7e2585..6270377f02 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -32,7 +32,7 @@ jobs: uses: actions/checkout@v4 - name: Build wheels for CPython - uses: pypa/cibuildwheel@v2.17.0 + uses: pypa/cibuildwheel@v2.18.1 with: output-dir: dist env: From 70b7cee2d9f63802a8d55519c82b6f55fa5c8f81 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 3 Jun 2024 17:09:43 +0000 Subject: [PATCH 040/186] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.3 → v0.4.7](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.3...v0.4.7) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 08cf87f28b..4c52974b86 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,7 +36,7 @@ repos: additional_dependencies: [black==24.3.0] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.3 + rev: v0.4.7 hooks: - id: ruff-format types_or: [ python, pyi, jupyter ] From 1abaab71050b1ecfe9038e85ba6cddd754acc3f3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 3 Jun 2024 17:10:00 +0000 Subject: [PATCH 041/186] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- yt/data_objects/data_containers.py | 2 +- yt/loaders.py | 2 +- yt/visualization/_handlers.py | 2 +- yt/visualization/base_plot_types.py | 6 +++--- yt/visualization/plot_modifications.py | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/yt/data_objects/data_containers.py b/yt/data_objects/data_containers.py index 83f4ff7594..80b2283f8d 100644 --- a/yt/data_objects/data_containers.py +++ b/yt/data_objects/data_containers.py @@ -87,7 +87,7 @@ def __init__(self, ds: Optional["Dataset"], field_parameters) -> None: # constructor, in which case it will override the default. # This code ensures it is never not set. - self.ds: "Dataset" + self.ds: Dataset if ds is not None: self.ds = ds else: diff --git a/yt/loaders.py b/yt/loaders.py index 1c05b877ee..54d52e5b83 100644 --- a/yt/loaders.py +++ b/yt/loaders.py @@ -120,7 +120,7 @@ def load( for entrypoint in external_frontends: entrypoint.load() - candidates: list[type["Dataset"]] = [] + candidates: list[type[Dataset]] = [] for cls in output_type_registry.values(): if cls._is_valid(fn, *args, **kwargs): candidates.append(cls) diff --git a/yt/visualization/_handlers.py b/yt/visualization/_handlers.py index 659b1eb6d3..fd4b381a76 100644 --- a/yt/visualization/_handlers.py +++ b/yt/visualization/_handlers.py @@ -431,7 +431,7 @@ def __init__( self._draw_minorticks = draw_minorticks self._cmap: Optional[Colormap] = None self._set_cmap(cmap) - self._background_color: Optional["ColorType"] = background_color + self._background_color: Optional[ColorType] = background_color @property def draw_cbar(self) -> bool: diff --git a/yt/visualization/base_plot_types.py b/yt/visualization/base_plot_types.py index 263a563f69..377e316be9 100644 --- a/yt/visualization/base_plot_types.py +++ b/yt/visualization/base_plot_types.py @@ -247,7 +247,7 @@ def __init__( ): """Initialize ImagePlotMPL class object""" - self._transform: Optional["Transform"] + self._transform: Optional[Transform] setdefaultattr(self, "_transform", None) self.colorbar_handler = colorbar_handler @@ -332,7 +332,7 @@ def _init_image(self, data, extent, aspect, *, alpha: AlphaT = None): self._set_axes() def _set_axes(self) -> None: - fmt_kwargs: "FormatKwargs" = { + fmt_kwargs: FormatKwargs = { "style": "scientific", "scilimits": (-2, 3), "useMathText": True, @@ -343,7 +343,7 @@ def _set_axes(self) -> None: self.cax.tick_params(which="both", direction="in") self.cb = self.figure.colorbar(self.image, self.cax) - cb_axis: "Axis" + cb_axis: Axis if self.cb.orientation == "vertical": cb_axis = self.cb.ax.yaxis else: diff --git a/yt/visualization/plot_modifications.py b/yt/visualization/plot_modifications.py index ca69ff6549..d16ad9a454 100644 --- a/yt/visualization/plot_modifications.py +++ b/yt/visualization/plot_modifications.py @@ -483,7 +483,7 @@ def __call__(self, plot) -> "BaseQuiverCallback": ) else: assert_never(geometry) - qcb: "BaseQuiverCallback" + qcb: BaseQuiverCallback if plot._type_name == "CuttingPlane": qcb = CuttingQuiverCallback( (ftype, "cutting_plane_velocity_x"), @@ -609,7 +609,7 @@ def __call__(self, plot) -> "BaseQuiverCallback": ftype = plot.data._current_fluid_type # Instantiation of these is cheap geometry: Geometry = plot.data.ds.geometry - qcb: "BaseQuiverCallback" + qcb: BaseQuiverCallback if plot._type_name == "CuttingPlane": if geometry is Geometry.CARTESIAN: pass From 5ce793a830029bbab22d54fd170b2c202e497153 Mon Sep 17 00:00:00 2001 From: Simon Guichandut Date: Fri, 17 May 2024 15:11:19 -0400 Subject: [PATCH 042/186] ENH: add support for molecular fields (AMRex) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Robert --- nose_ignores.txt | 1 + tests/tests.yaml | 1 + yt/frontends/amrex/fields.py | 113 +++++++++++++----- .../amrex/tests/test_field_parsing.py | 51 ++++++++ 4 files changed, 137 insertions(+), 29 deletions(-) create mode 100644 yt/frontends/amrex/tests/test_field_parsing.py diff --git a/nose_ignores.txt b/nose_ignores.txt index 09ad4a1259..a60796b570 100644 --- a/nose_ignores.txt +++ b/nose_ignores.txt @@ -43,3 +43,4 @@ --ignore-file=test_alt_ray_tracers\.py --ignore-file=test_minimal_representation\.py --ignore-file=test_set_log_level\.py +--ignore-file=test_field_parsing\.py diff --git a/tests/tests.yaml b/tests/tests.yaml index 208a78f374..7a8a1826a3 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -187,6 +187,7 @@ other_tests: - "--ignore-file=test_ewah_write_load\\.py" - "--ignore-file=test_external_frontends\\.py" - "--ignore-file=test_field_access_pytest\\.py" + - "--ignore-file=test_field_parsing\\.py" - "--ignore-file=test_file_sanitizer\\.py" - "--ignore-file=test_firefly\\.py" - "--ignore-file=test_geometries\\.py" diff --git a/yt/frontends/amrex/fields.py b/yt/frontends/amrex/fields.py index cf193326c5..fbde2d2bb0 100644 --- a/yt/frontends/amrex/fields.py +++ b/yt/frontends/amrex/fields.py @@ -1,4 +1,5 @@ import re +import sys import numpy as np @@ -7,12 +8,15 @@ from yt.units import YTQuantity from yt.utilities.physical_constants import amu_cgs, boltzmann_constant_cgs, c +if sys.version_info >= (3, 10): + from typing import TypeAlias +else: + from typing_extensions import TypeAlias + rho_units = "code_mass / code_length**3" mom_units = "code_mass / (code_time * code_length**2)" eden_units = "code_mass / (code_time**2 * code_length)" # erg / cm^3 -spec_finder = re.compile(r".*\((\D*)(\d*)\).*") - def _thermal_energy_density(field, data): # What we've got here is UEINT: @@ -357,16 +361,22 @@ def setup_fluid_fields(self): for _, field in self.ds.field_list: if field.startswith("X("): # We have a fraction - nice_name, tex_label = _nice_species_name(field) - self.alias( - ("gas", f"{nice_name}_fraction"), ("boxlib", field), units="" + sub = Substance(field) + # Overwrite field to use nicer tex_label display_name + self.add_output_field( + ("boxlib", field), + sampling_type="cell", + units="", + display_name=rf"X\left({sub.to_tex()}\right)", ) - func = _create_density_func(("gas", f"{nice_name}_fraction")) + self.alias(("gas", f"{sub}_fraction"), ("boxlib", field), units="") + func = _create_density_func(("gas", f"{sub}_fraction")) self.add_field( - name=("gas", f"{nice_name}_density"), + name=("gas", f"{sub}_density"), sampling_type="cell", function=func, units=self.ds.unit_system["density"], + display_name=rf"\rho {sub.to_tex()}", ) @@ -462,29 +472,27 @@ def setup_fluid_fields(self): for _, field in self.ds.field_list: if field.startswith("X("): # We have a mass fraction - nice_name, tex_label = _nice_species_name(field) + sub = Substance(field) # Overwrite field to use nicer tex_label display_name self.add_output_field( ("boxlib", field), sampling_type="cell", units="", - display_name=tex_label, - ) - self.alias( - ("gas", f"{nice_name}_fraction"), ("boxlib", field), units="" + display_name=rf"X\left({sub.to_tex()}\right)", ) - func = _create_density_func(("gas", f"{nice_name}_fraction")) + self.alias(("gas", f"{sub}_fraction"), ("boxlib", field), units="") + func = _create_density_func(("gas", f"{sub}_fraction")) self.add_field( - name=("gas", f"{nice_name}_density"), + name=("gas", f"{sub}_density"), sampling_type="cell", function=func, units=unit_system["density"], - display_name=rf"\rho {tex_label}", + display_name=rf"\rho {sub.to_tex()}", ) elif field.startswith("omegadot("): - nice_name, tex_label = _nice_species_name(field) - display_name = rf"\dot{{\omega}}\left[{tex_label}\right]" + sub = Substance(field) + display_name = rf"\dot{{\omega}}\left[{sub.to_tex()}\right]" # Overwrite field to use nicer tex_label'ed display_name self.add_output_field( ("boxlib", field), @@ -493,23 +501,70 @@ def setup_fluid_fields(self): display_name=display_name, ) self.alias( - ("gas", f"{nice_name}_creation_rate"), + ("gas", f"{sub}_creation_rate"), ("boxlib", field), units=unit_system["frequency"], ) -def _nice_species_name(field): - spec_match = spec_finder.search(field) - nice_name = "".join(spec_match.groups()) - # if the species field is a descriptive name, then the match - # on the integer will be blank - # modify the tex string in this case to remove spurious tex spacing - lab = r"X\left(^{%s}%s\right)" - if spec_match.groups()[-1] == "": - lab = r"X\left(%s%s\right)" - tex_label = lab % spec_match.groups()[::-1] - return nice_name, tex_label +substance_expr_re = re.compile(r"\(([a-zA-Z][a-zA-Z0-9]*)\)") +substance_elements_re = re.compile(r"(?P[a-zA-Z]+)(?P\d*)") +SubstanceSpec: TypeAlias = list[tuple[str, int]] + + +class Substance: + def __init__(self, data: str) -> None: + if (m := substance_expr_re.search(data)) is None: + raise ValueError(f"{data!r} doesn't match expected regular expression") + sub_str = m.group() + constituents = substance_elements_re.findall(sub_str) + + # 0 is used as a sentinel value to mark descriptive names + default_value = 1 if len(constituents) > 1 else 0 + self._spec: SubstanceSpec = [ + (name, int(count or default_value)) for (name, count) in constituents + ] + + def get_spec(self) -> SubstanceSpec: + return self._spec.copy() + + def is_isotope(self) -> bool: + return len(self._spec) == 1 and self._spec[0][1] > 0 + + def is_molecule(self) -> bool: + return len(self._spec) != 1 + + def is_descriptive_name(self) -> bool: + return len(self._spec) == 1 and self._spec[0][1] == 0 + + def __str__(self) -> str: + return "".join( + f"{element}{count if count > 1 else ''}" for element, count in self._spec + ) + + def _to_tex_isotope(self) -> str: + element, count = self._spec[0] + return rf"^{{{count}}}{element}" + + def _to_tex_molecule(self) -> str: + return "".join( + rf"{element}_{{{count if count>1 else ''}}}" + for element, count in self._spec + ) + + def _to_tex_descriptive(self) -> str: + return str(self) + + def to_tex(self) -> str: + if self.is_isotope(): + return self._to_tex_isotope() + elif self.is_molecule(): + return self._to_tex_molecule() + elif self.is_descriptive_name(): + return self._to_tex_descriptive() + else: + # should only be reachable in case of a regular expression defect + raise RuntimeError def _create_density_func(field_name): diff --git a/yt/frontends/amrex/tests/test_field_parsing.py b/yt/frontends/amrex/tests/test_field_parsing.py new file mode 100644 index 0000000000..ec3fbe779f --- /dev/null +++ b/yt/frontends/amrex/tests/test_field_parsing.py @@ -0,0 +1,51 @@ +import pytest + +from yt.frontends.amrex.fields import Substance + + +@pytest.mark.parametrize( + "data, expected", + [ + pytest.param("X(He5)", [("He", 5)], id="isotope_1"), + pytest.param("X(C12)", [("C", 12)], id="isotope_2"), + pytest.param("X(A1B2C3)", [("A", 1), ("B", 2), ("C", 3)], id="molecule_1"), + pytest.param("X(C12H24)", [("C", 12), ("H", 24)], id="molecule_2"), + pytest.param("X(H2O)", [("H", 2), ("O", 1)], id="molecule_3"), + pytest.param("X(ash)", [("ash", 0)], id="descriptive_name"), + ], +) +def test_Substance_spec(data, expected): + assert Substance(data)._spec == expected + + +@pytest.mark.parametrize( + "data, expected_type", + [ + pytest.param("X(He5)", "isotope", id="isotope_1"), + pytest.param("X(C12)", "isotope", id="isotope_2"), + pytest.param("X(A1B2C3)", "molecule", id="molecule_1"), + pytest.param("X(C12H24)", "molecule", id="molecule_2"), + pytest.param("X(H2O)", "molecule", id="molecule_3"), + pytest.param("X(ash)", "descriptive_name", id="descriptive_name"), + ], +) +def test_Substance_type(data, expected_type): + sub = Substance(data) + assert getattr(sub, f"is_{expected_type}")() + + +@pytest.mark.parametrize( + "data, expected_str, expected_tex", + [ + pytest.param("X(He5)", "He5", "^{5}He", id="isotope_1"), + pytest.param("X(C12)", "C12", "^{12}C", id="isotope_2"), + pytest.param("X(A1B2C3)", "AB2C3", "A_{}B_{2}C_{3}", id="molecule_1"), + pytest.param("X(C12H24)", "C12H24", "C_{12}H_{24}", id="molecule_2"), + pytest.param("X(H2O)", "H2O", "H_{2}O_{}", id="molecule_2"), + pytest.param("X(ash)", "ash", "ash", id="descriptive_name"), + ], +) +def test_Substance_to_str(data, expected_str, expected_tex): + sub = Substance(data) + assert str(sub) == expected_str + assert sub.to_tex() == expected_tex From a0c45d534de3dbac766e48ae12c258bcdbd0ed55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Tue, 4 Jun 2024 09:15:07 +0200 Subject: [PATCH 043/186] BUG: fix deprecated implicit casting of arrays with ndim>0 to int scalar --- yt/frontends/amrex/data_structures.py | 2 +- yt/frontends/athena/data_structures.py | 2 +- yt/frontends/chombo/data_structures.py | 2 +- yt/frontends/gdf/data_structures.py | 2 +- yt/frontends/stream/data_structures.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/yt/frontends/amrex/data_structures.py b/yt/frontends/amrex/data_structures.py index 59fab4c4c5..8c6e2756be 100644 --- a/yt/frontends/amrex/data_structures.py +++ b/yt/frontends/amrex/data_structures.py @@ -517,7 +517,7 @@ def _reconstruct_parent_child(self): get_box_grids_level( self.grid_left_edge[i, :], self.grid_right_edge[i, :], - self.grid_levels[i] + 1, + self.grid_levels[i].item() + 1, self.grid_left_edge, self.grid_right_edge, self.grid_levels, diff --git a/yt/frontends/athena/data_structures.py b/yt/frontends/athena/data_structures.py index 3378ec1b11..c0e1a92cf2 100644 --- a/yt/frontends/athena/data_structures.py +++ b/yt/frontends/athena/data_structures.py @@ -432,7 +432,7 @@ def _reconstruct_parent_child(self): get_box_grids_level( self.grid_left_edge[i, :], self.grid_right_edge[i, :], - self.grid_levels[i] + 1, + self.grid_levels[i].item() + 1, self.grid_left_edge, self.grid_right_edge, self.grid_levels, diff --git a/yt/frontends/chombo/data_structures.py b/yt/frontends/chombo/data_structures.py index 8bffd1ab1a..df93c64cda 100644 --- a/yt/frontends/chombo/data_structures.py +++ b/yt/frontends/chombo/data_structures.py @@ -219,7 +219,7 @@ def _reconstruct_parent_child(self): get_box_grids_level( self.grid_left_edge[i, :], self.grid_right_edge[i, :], - self.grid_levels[i] + 1, + self.grid_levels[i].item() + 1, self.grid_left_edge, self.grid_right_edge, self.grid_levels, diff --git a/yt/frontends/gdf/data_structures.py b/yt/frontends/gdf/data_structures.py index 1bb7a88b66..0817c2aa1b 100644 --- a/yt/frontends/gdf/data_structures.py +++ b/yt/frontends/gdf/data_structures.py @@ -115,7 +115,7 @@ def _populate_grid_objects(self): get_box_grids_level( self.grid_left_edge[gi, :], self.grid_right_edge[gi, :], - self.grid_levels[gi], + self.grid_levels[gi].item(), self.grid_left_edge, self.grid_right_edge, self.grid_levels, diff --git a/yt/frontends/stream/data_structures.py b/yt/frontends/stream/data_structures.py index f1d4db1b13..68b6f18ac4 100644 --- a/yt/frontends/stream/data_structures.py +++ b/yt/frontends/stream/data_structures.py @@ -240,7 +240,7 @@ def _reconstruct_parent_child(self): get_box_grids_level( self.grid_left_edge[i, :], self.grid_right_edge[i, :], - self.grid_levels[i] + 1, + self.grid_levels[i].item() + 1, self.grid_left_edge, self.grid_right_edge, self.grid_levels, From d49a5a75095e25c1746611449244774c2ac13d2a Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Thu, 6 Jun 2024 14:24:02 -0500 Subject: [PATCH 044/186] tipsy on demand aux opening for ascii --- yt/frontends/tipsy/io.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yt/frontends/tipsy/io.py b/yt/frontends/tipsy/io.py index 5d7b795fb2..d7296484a0 100644 --- a/yt/frontends/tipsy/io.py +++ b/yt/frontends/tipsy/io.py @@ -181,11 +181,11 @@ def aux_fh(afield): ) ) else: - aux_fh[afield].seek(0) + aux_fh(afield).seek(0) sh = aux_fields_offsets[afield][ptype] if tp[ptype] > 0: aux = np.genfromtxt( - aux_fh[afield], skip_header=sh, max_rows=count + aux_fh(afield), skip_header=sh, max_rows=count ) if aux.ndim < 1: aux = np.array([aux]) @@ -212,7 +212,7 @@ def aux_fh(afield): # close all file handles f.close() - for fh in list(_aux_fh.values()): + for fh in _aux_fh.values(): fh.close() return return_data From 748014da39a33fccec5bb473c521a16229f3a2d9 Mon Sep 17 00:00:00 2001 From: "Eric T. Johnson" Date: Fri, 7 Jun 2024 02:14:21 -0400 Subject: [PATCH 045/186] BLD: run pytest on wheels and sdist (#4913) --- .github/workflows/wheels.yaml | 7 +- pyproject.toml | 164 +++++++++++++++++----------------- 2 files changed, 86 insertions(+), 85 deletions(-) diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index 6270377f02..83340ba70d 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -43,7 +43,8 @@ jobs: MACOSX_DEPLOYMENT_TARGET: "10.9" # as of CIBW 2.9, this is the default value, pin it so it can't be bumped silently CIBW_ARCHS_WINDOWS: auto64 CIBW_BUILD_VERBOSITY: 1 - CIBW_TEST_COMMAND: python -c "import yt" + CIBW_TEST_EXTRAS: test + CIBW_TEST_COMMAND: pytest -c {project}/pyproject.toml --rootdir . --color=yes --pyargs yt - uses: actions/upload-artifact@v4 with: @@ -67,11 +68,11 @@ jobs: - name: Test sdist run: | - python -m pip install "$(echo dist/*.tar.gz)" + python -m pip install "$(echo dist/*.tar.gz)[test]" python -m pip list project_dir=$(pwd) cd ../../ - python -c "import yt" + pytest -c $project_dir/pyproject.toml --rootdir . --color=yes --pyargs yt - name: Upload sdist uses: actions/upload-artifact@v4 diff --git a/pyproject.toml b/pyproject.toml index 0fe215c7a7..6fe32ca438 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -335,88 +335,88 @@ addopts = ''' -s -v -rsfE - --ignore-glob='*_nose.py' - --ignore='yt/data_objects/level_sets/tests/test_clump_finding.py' - --ignore='yt/data_objects/tests/test_connected_sets.py' - --ignore='yt/data_objects/tests/test_data_containers.py' - --ignore='yt/data_objects/tests/test_dataset_access.py' - --ignore='yt/data_objects/tests/test_disks.py' - --ignore='yt/data_objects/tests/test_particle_filter.py' - --ignore='yt/data_objects/tests/test_particle_trajectories.py' - --ignore='yt/data_objects/tests/test_pickling.py' - --ignore='yt/data_objects/tests/test_regions.py' - --ignore='yt/fields/tests/test_particle_fields.py' - --ignore='yt/fields/tests/test_vector_fields.py' - --ignore='yt/fields/tests/test_xray_fields.py' - --ignore='yt/frontends/adaptahop/tests/test_outputs.py' - --ignore='yt/frontends/ahf/tests/test_outputs.py' - --ignore='yt/frontends/amrex/tests/test_outputs.py' - --ignore='yt/frontends/amrvac/tests/test_outputs.py' - --ignore='yt/frontends/amrvac/tests/test_units_override.py' - --ignore='yt/frontends/arepo/tests/test_outputs.py' - --ignore='yt/frontends/art/tests/test_outputs.py' - --ignore='yt/frontends/artio/tests/test_outputs.py' - --ignore='yt/frontends/athena/tests/test_outputs.py' - --ignore='yt/frontends/athena_pp/tests/test_outputs.py' - --ignore='yt/frontends/boxlib/tests/test_outputs.py' - --ignore='yt/frontends/cf_radial/tests/test_outputs.py' - --ignore='yt/frontends/chimera/tests/test_outputs.py' - --ignore='yt/frontends/cholla/tests/test_outputs.py' - --ignore='yt/frontends/chombo/tests/test_outputs.py' - --ignore='yt/frontends/eagle/tests/test_outputs.py' - --ignore='yt/frontends/enzo/tests/test_outputs.py' - --ignore='yt/frontends/enzo_e/tests/test_outputs.py' - --ignore='yt/frontends/exodus_ii/tests/test_outputs.py' - --ignore='yt/frontends/fits/tests/test_outputs.py' - --ignore='yt/frontends/flash/tests/test_outputs.py' - --ignore='yt/frontends/gadget/tests/test_outputs.py' - --ignore='yt/frontends/gadget_fof/tests/test_outputs.py' - --ignore='yt/frontends/gamer/tests/test_outputs.py' - --ignore='yt/frontends/gdf/tests/test_outputs.py' - --ignore='yt/frontends/gdf/tests/test_outputs_nose.py' - --ignore='yt/frontends/gizmo/tests/test_outputs.py' - --ignore='yt/frontends/halo_catalog/tests/test_outputs.py' - --ignore='yt/frontends/moab/tests/test_c5.py' - --ignore='yt/frontends/nc4_cm1/tests/test_outputs.py' - --ignore='yt/frontends/open_pmd/tests/test_outputs.py' - --ignore='yt/frontends/owls/tests/test_outputs.py' - --ignore='yt/frontends/owls_subfind/tests/test_outputs.py' - --ignore='yt/frontends/ramses/tests/test_outputs.py' - --ignore='yt/frontends/rockstar/tests/test_outputs.py' - --ignore='yt/frontends/tipsy/tests/test_outputs.py' - --ignore='yt/frontends/ytdata/tests/test_old_outputs.py' - --ignore='yt/frontends/ytdata/tests/test_outputs.py' - --ignore='yt/frontends/ytdata/tests/test_unit.py' - --ignore='yt/geometry/coordinates/tests/test_axial_pixelization.py' - --ignore='yt/geometry/coordinates/tests/test_cylindrical_coordinates.py' - --ignore='yt/geometry/coordinates/tests/test_spherical_coordinates.py' - --ignore='yt/tests/test_funcs.py' - --ignore='yt/utilities/lib/cykdtree/tests/__init__.py' - --ignore='yt/utilities/lib/cykdtree/tests/test_kdtree.py' - --ignore='yt/utilities/lib/cykdtree/tests/test_plot.py' - --ignore='yt/utilities/lib/cykdtree/tests/test_utils.py' - --ignore='yt/utilities/tests/test_cosmology.py' - --ignore='yt/visualization/tests/test_callbacks.py' - --ignore='yt/visualization/tests/test_color_maps.py' - --ignore='yt/visualization/tests/test_geo_projections.py' - --ignore='yt/visualization/tests/test_image_writer.py' - --ignore='yt/visualization/tests/test_line_plots.py' - --ignore='yt/visualization/tests/test_mesh_slices.py' - --ignore='yt/visualization/tests/test_norm_api_custom_norm.py' - --ignore='yt/visualization/tests/test_norm_api_inf_zlim.py' - --ignore='yt/visualization/tests/test_norm_api_lineplot.py' - --ignore='yt/visualization/tests/test_norm_api_particleplot.py' - --ignore='yt/visualization/tests/test_norm_api_phaseplot_set_colorbar_explicit.py' - --ignore='yt/visualization/tests/test_norm_api_phaseplot_set_colorbar_implicit.py' - --ignore='yt/visualization/tests/test_norm_api_profileplot.py' - --ignore='yt/visualization/tests/test_norm_api_set_background_color.py' - --ignore='yt/visualization/tests/test_particle_plot.py' - --ignore='yt/visualization/tests/test_plot_modifications.py' - --ignore='yt/visualization/tests/test_plotwindow.py' - --ignore='yt/visualization/tests/test_profile_plots.py' - --ignore='yt/visualization/tests/test_raw_field_slices.py' - --ignore='yt/visualization/volume_rendering/tests/test_mesh_render.py' - --ignore='yt/visualization/volume_rendering/tests/test_vr_orientation.py' + --ignore-glob='/*_nose.py' + --ignore-glob='/*/yt/data_objects/level_sets/tests/test_clump_finding.py' + --ignore-glob='/*/yt/data_objects/tests/test_connected_sets.py' + --ignore-glob='/*/yt/data_objects/tests/test_data_containers.py' + --ignore-glob='/*/yt/data_objects/tests/test_dataset_access.py' + --ignore-glob='/*/yt/data_objects/tests/test_disks.py' + --ignore-glob='/*/yt/data_objects/tests/test_particle_filter.py' + --ignore-glob='/*/yt/data_objects/tests/test_particle_trajectories.py' + --ignore-glob='/*/yt/data_objects/tests/test_pickling.py' + --ignore-glob='/*/yt/data_objects/tests/test_regions.py' + --ignore-glob='/*/yt/fields/tests/test_particle_fields.py' + --ignore-glob='/*/yt/fields/tests/test_vector_fields.py' + --ignore-glob='/*/yt/fields/tests/test_xray_fields.py' + --ignore-glob='/*/yt/frontends/adaptahop/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/ahf/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/amrex/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/amrvac/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/amrvac/tests/test_units_override.py' + --ignore-glob='/*/yt/frontends/arepo/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/art/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/artio/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/athena/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/athena_pp/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/boxlib/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/cf_radial/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/chimera/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/cholla/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/chombo/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/eagle/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/enzo/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/enzo_e/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/exodus_ii/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/fits/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/flash/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/gadget/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/gadget_fof/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/gamer/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/gdf/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/gdf/tests/test_outputs_nose.py' + --ignore-glob='/*/yt/frontends/gizmo/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/halo_catalog/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/moab/tests/test_c5.py' + --ignore-glob='/*/yt/frontends/nc4_cm1/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/open_pmd/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/owls/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/owls_subfind/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/ramses/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/rockstar/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/tipsy/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/ytdata/tests/test_old_outputs.py' + --ignore-glob='/*/yt/frontends/ytdata/tests/test_outputs.py' + --ignore-glob='/*/yt/frontends/ytdata/tests/test_unit.py' + --ignore-glob='/*/yt/geometry/coordinates/tests/test_axial_pixelization.py' + --ignore-glob='/*/yt/geometry/coordinates/tests/test_cylindrical_coordinates.py' + --ignore-glob='/*/yt/geometry/coordinates/tests/test_spherical_coordinates.py' + --ignore-glob='/*/yt/tests/test_funcs.py' + --ignore-glob='/*/yt/utilities/lib/cykdtree/tests/__init__.py' + --ignore-glob='/*/yt/utilities/lib/cykdtree/tests/test_kdtree.py' + --ignore-glob='/*/yt/utilities/lib/cykdtree/tests/test_plot.py' + --ignore-glob='/*/yt/utilities/lib/cykdtree/tests/test_utils.py' + --ignore-glob='/*/yt/utilities/tests/test_cosmology.py' + --ignore-glob='/*/yt/visualization/tests/test_callbacks.py' + --ignore-glob='/*/yt/visualization/tests/test_color_maps.py' + --ignore-glob='/*/yt/visualization/tests/test_geo_projections.py' + --ignore-glob='/*/yt/visualization/tests/test_image_writer.py' + --ignore-glob='/*/yt/visualization/tests/test_line_plots.py' + --ignore-glob='/*/yt/visualization/tests/test_mesh_slices.py' + --ignore-glob='/*/yt/visualization/tests/test_norm_api_custom_norm.py' + --ignore-glob='/*/yt/visualization/tests/test_norm_api_inf_zlim.py' + --ignore-glob='/*/yt/visualization/tests/test_norm_api_lineplot.py' + --ignore-glob='/*/yt/visualization/tests/test_norm_api_particleplot.py' + --ignore-glob='/*/yt/visualization/tests/test_norm_api_phaseplot_set_colorbar_explicit.py' + --ignore-glob='/*/yt/visualization/tests/test_norm_api_phaseplot_set_colorbar_implicit.py' + --ignore-glob='/*/yt/visualization/tests/test_norm_api_profileplot.py' + --ignore-glob='/*/yt/visualization/tests/test_norm_api_set_background_color.py' + --ignore-glob='/*/yt/visualization/tests/test_particle_plot.py' + --ignore-glob='/*/yt/visualization/tests/test_plot_modifications.py' + --ignore-glob='/*/yt/visualization/tests/test_plotwindow.py' + --ignore-glob='/*/yt/visualization/tests/test_profile_plots.py' + --ignore-glob='/*/yt/visualization/tests/test_raw_field_slices.py' + --ignore-glob='/*/yt/visualization/volume_rendering/tests/test_mesh_render.py' + --ignore-glob='/*/yt/visualization/volume_rendering/tests/test_vr_orientation.py' ''' From 5bacc84f7d8d23249492f69037f100debe23c34e Mon Sep 17 00:00:00 2001 From: "Eric T. Johnson" Date: Wed, 12 Jun 2024 16:00:00 -0400 Subject: [PATCH 046/186] DOC: fix typo in the debugging signal names --- doc/source/developing/debugdrive.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/developing/debugdrive.rst b/doc/source/developing/debugdrive.rst index 3ed352a2fa..e8fa885437 100644 --- a/doc/source/developing/debugdrive.rst +++ b/doc/source/developing/debugdrive.rst @@ -66,7 +66,7 @@ will induce the requested behavior. SIGUSR1 This will cause the python code to print a stack trace, showing exactly where in the function stack it is currently executing. - SIGUSR1 + SIGUSR2 This will cause the python code to insert an IPython session wherever it currently is, with all local variables in the local namespace. It should allow you to change the state variables. From e9ba8aca6c262085afc7926541881136bd4c426c Mon Sep 17 00:00:00 2001 From: Chris Havlin Date: Fri, 14 Jun 2024 02:54:13 -0500 Subject: [PATCH 047/186] DOC: replace boxlib with amrex references in the docs (#4925) * replace boxlib with amrex references in the docs * add ipykernel to doc reqs * fix typo in docs * pin ipykernel>=6.29.4 for docs --- doc/source/developing/creating_frontend.rst | 9 ++++---- doc/source/examining/loading_data.rst | 4 ++-- doc/source/reference/api/api.rst | 24 ++++++++++----------- pyproject.toml | 1 + 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/doc/source/developing/creating_frontend.rst b/doc/source/developing/creating_frontend.rst index 2678d3abec..c11d1feba0 100644 --- a/doc/source/developing/creating_frontend.rst +++ b/doc/source/developing/creating_frontend.rst @@ -56,7 +56,7 @@ called ``_is_valid()`` that lets the ``yt.load`` method help identify an input file as belonging to *this* particular ``Dataset`` subclass (see :ref:`data-format-detection`). For the most part, the examples of -``yt.frontends.boxlib.data_structures.OrionDataset`` and +``yt.frontends.amrex.data_structures.OrionDataset`` and ``yt.frontends.enzo.data_structures.EnzoDataset`` should be followed, but ``yt.frontends.chombo.data_structures.ChomboDataset``, as a slightly newer addition, can also be used as an instructive example. @@ -64,7 +64,8 @@ slightly newer addition, can also be used as an instructive example. A new set of fields must be added in the file ``fields.py`` in your new directory. For the most part this means subclassing ``FieldInfoContainer`` and adding the necessary fields specific to -your code. Here is a snippet from the base BoxLib field container: +your code. Here is a snippet from the base BoxLib field container (defined in +``yt.frontends.amrex.fields``): .. code-block:: python @@ -273,7 +274,7 @@ that is needed: Even one of the more complex grid objects, -``yt.frontends.boxlib.BoxlibGrid``, is still relatively simple. +``yt.frontends.amrex.BoxlibGrid``, is still relatively simple. Data Reading Functions ---------------------- @@ -310,7 +311,7 @@ At a minimum, one should also override the following methods If your dataset has particle information, you'll want to override the ``_read_particle_coords()`` and ``read_particle_fields()`` methods as well. Each code is going to read data from disk in a different -fashion, but the ``yt.frontends.boxlib.io.IOHandlerBoxlib`` is a +fashion, but the ``yt.frontends.amrex.io.IOHandlerBoxlib`` is a decent place to start. And that just about covers it. Please feel free to email diff --git a/doc/source/examining/loading_data.rst b/doc/source/examining/loading_data.rst index 605614b2de..0415712367 100644 --- a/doc/source/examining/loading_data.rst +++ b/doc/source/examining/loading_data.rst @@ -558,8 +558,8 @@ using a ``parameters`` dict, accepting the following keys: AMReX / BoxLib Data ------------------- -AMReX and BoxLib share a frontend (currently named ``boxlib``), since -the file format nearly identical. yt has been tested with AMReX/BoxLib +AMReX and BoxLib share a frontend, since +the file format is nearly identical. yt has been tested with AMReX/BoxLib data generated by Orion, Nyx, Maestro, Castro, IAMR, and WarpX. Currently it is cared for by a combination of Andrew Myers, Matthew Turk, and Mike Zingale. diff --git a/doc/source/reference/api/api.rst b/doc/source/reference/api/api.rst index 61b63f2c15..3cc192c578 100644 --- a/doc/source/reference/api/api.rst +++ b/doc/source/reference/api/api.rst @@ -217,18 +217,18 @@ AMReX/Boxlib .. autosummary:: - ~yt.frontends.boxlib.data_structures.BoxlibGrid - ~yt.frontends.boxlib.data_structures.BoxlibHierarchy - ~yt.frontends.boxlib.data_structures.BoxlibDataset - ~yt.frontends.boxlib.data_structures.CastroDataset - ~yt.frontends.boxlib.data_structures.MaestroDataset - ~yt.frontends.boxlib.data_structures.NyxHierarchy - ~yt.frontends.boxlib.data_structures.NyxDataset - ~yt.frontends.boxlib.data_structures.OrionHierarchy - ~yt.frontends.boxlib.data_structures.OrionDataset - ~yt.frontends.boxlib.fields.BoxlibFieldInfo - ~yt.frontends.boxlib.io.IOHandlerBoxlib - ~yt.frontends.boxlib.io.IOHandlerOrion + ~yt.frontends.amrex.data_structures.BoxlibGrid + ~yt.frontends.amrex.data_structures.BoxlibHierarchy + ~yt.frontends.amrex.data_structures.BoxlibDataset + ~yt.frontends.amrex.data_structures.CastroDataset + ~yt.frontends.amrex.data_structures.MaestroDataset + ~yt.frontends.amrex.data_structures.NyxHierarchy + ~yt.frontends.amrex.data_structures.NyxDataset + ~yt.frontends.amrex.data_structures.OrionHierarchy + ~yt.frontends.amrex.data_structures.OrionDataset + ~yt.frontends.amrex.fields.BoxlibFieldInfo + ~yt.frontends.amrex.io.IOHandlerBoxlib + ~yt.frontends.amrex.io.IOHandlerOrion CfRadial ^^^^^^^^ diff --git a/pyproject.toml b/pyproject.toml index 6fe32ca438..24e2982c5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -191,6 +191,7 @@ full = [ doc = [ "alabaster>=0.7.13", "bottle>=0.12.25", + "ipykernel>=6.29.4", "jinja2<3.1.0", # see https://github.com/readthedocs/readthedocs.org/issues/9037 "jupyter-client>=8.3.1", "nbsphinx>=0.9.3", From 4f17ed14a5b88cbff6ad17d81f86851a4b980692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sun, 16 Jun 2024 12:01:07 +0200 Subject: [PATCH 048/186] BUG: ensure type safety in a call to np.logspace --- yt/frontends/art/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/frontends/art/io.py b/yt/frontends/art/io.py index 137cd096b0..1047928ce7 100644 --- a/yt/frontends/art/io.py +++ b/yt/frontends/art/io.py @@ -635,7 +635,7 @@ def b2t(tb, n=1e2, logger=None, **kwargs): return a2t(b2a(tb)) if len(tb) < n: n = len(tb) - tbs = -1.0 * np.logspace(np.log10(-tb.min()), np.log10(-tb.max()), n) + tbs = -1.0 * np.logspace(np.log10(-tb.min()), np.log10(-tb.max()), int(n)) ages = [] for i, tbi in enumerate(tbs): ages += (a2t(b2a(tbi)),) From 0045fc79da934529f0bce33e1d79a720ca789b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sun, 16 Jun 2024 16:05:05 +0200 Subject: [PATCH 049/186] BLD: require numpy 2.0.0 (stable) at build time --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 24e2982c5b..9afadd0483 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ requires = [ # for the upper pin in Cython # see https://github.com/yt-project/yt/issues/4044 "Cython>=3.0.3, <3.1", - "numpy>=2.0.0rc1", + "numpy>=2.0.0", "ewah-bool-utils>=1.2.0", ] build-backend = "setuptools.build_meta:__legacy__" From 103cba557fce6731ce8500b479a11be58bff6425 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sun, 16 Jun 2024 21:10:47 +0200 Subject: [PATCH 050/186] TST: bump enzo answer tests following change in numpy 2.0 --- tests/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests.yaml b/tests/tests.yaml index 7a8a1826a3..4a3786b359 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -69,7 +69,7 @@ answer_tests: - yt/frontends/chombo/tests/test_outputs.py:test_zp - yt/frontends/chombo/tests/test_outputs.py:test_kho - local_enzo_009: # PR 3856 + local_enzo_010: # PR 4930 - yt/frontends/enzo/tests/test_outputs.py:test_moving7 - yt/frontends/enzo/tests/test_outputs.py:test_galaxy0030 - yt/frontends/enzo/tests/test_outputs.py:test_toro1d From 175cd2d13b65c765da7e195131183e7eab4c76fa Mon Sep 17 00:00:00 2001 From: Chris Havlin Date: Sat, 22 Jun 2024 09:49:40 +0200 Subject: [PATCH 051/186] BUG: avoid inconsistent unit conversions in _divergence field --- yt/fields/vector_operations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/fields/vector_operations.py b/yt/fields/vector_operations.py index 99e0544539..58eeba0a17 100644 --- a/yt/fields/vector_operations.py +++ b/yt/fields/vector_operations.py @@ -362,7 +362,7 @@ def _divergence(field, data): ds = div_fac * just_one(data["index", "dz"]) f += data[zn[0], f"relative_{zn[1]}"][1:-1, 1:-1, sl_right] / ds f -= data[zn[0], f"relative_{zn[1]}"][1:-1, 1:-1, sl_left] / ds - new_field = data.ds.arr(np.zeros(data[xn].shape, dtype=np.float64), f.units) + new_field = data.ds.arr(np.zeros(data[xn].shape, dtype="f8"), str(f.units)) new_field[1:-1, 1:-1, 1:-1] = f return new_field From 47a57dcc02e1331b2aa31a30804af0c8b9962f20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sat, 22 Jun 2024 10:04:26 +0200 Subject: [PATCH 052/186] TYP: ignore a spurious mypy warning --- yt/_maintenance/numpy2_compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/_maintenance/numpy2_compat.py b/yt/_maintenance/numpy2_compat.py index 5f53beb91b..3cc1f01aa1 100644 --- a/yt/_maintenance/numpy2_compat.py +++ b/yt/_maintenance/numpy2_compat.py @@ -9,4 +9,4 @@ if NUMPY_VERSION >= Version("2.0.0dev0"): from numpy import trapezoid as trapezoid # type: ignore [attr-defined] else: - from numpy import trapz as trapezoid # noqa: F401 + from numpy import trapz as trapezoid # type: ignore [attr-defined] # noqa: F401 From 73a167207c06b160d44770bc70704894ca6ce7d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sat, 22 Jun 2024 10:39:40 +0200 Subject: [PATCH 053/186] TST: bump enzo answer tests again --- tests/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests.yaml b/tests/tests.yaml index 4a3786b359..d0f12ad1e5 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -69,7 +69,7 @@ answer_tests: - yt/frontends/chombo/tests/test_outputs.py:test_zp - yt/frontends/chombo/tests/test_outputs.py:test_kho - local_enzo_010: # PR 4930 + local_enzo_011: # PR 4930 - yt/frontends/enzo/tests/test_outputs.py:test_moving7 - yt/frontends/enzo/tests/test_outputs.py:test_galaxy0030 - yt/frontends/enzo/tests/test_outputs.py:test_toro1d From 7f0f4b9a1bae7dfdacf219cf723761e0388cacc9 Mon Sep 17 00:00:00 2001 From: smsutherland <56368041+smsutherland@users.noreply.github.com> Date: Tue, 25 Jun 2024 05:02:30 -0400 Subject: [PATCH 054/186] ENH: fix compatibility issues with newest versions of SWIFT (#4921) --- yt/frontends/swift/data_structures.py | 32 ++++++++++++++++++++++----- yt/frontends/swift/fields.py | 27 ++++++++++++++++++++++ yt/frontends/swift/io.py | 8 ++++++- 3 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 yt/frontends/swift/fields.py diff --git a/yt/frontends/swift/data_structures.py b/yt/frontends/swift/data_structures.py index 95bf9a8371..39d881c73c 100644 --- a/yt/frontends/swift/data_structures.py +++ b/yt/frontends/swift/data_structures.py @@ -2,11 +2,12 @@ from yt.data_objects.static_output import ParticleFile from yt.frontends.sph.data_structures import SPHDataset, SPHParticleIndex -from yt.frontends.sph.fields import SPHFieldInfo from yt.funcs import only_on_root from yt.utilities.logger import ytLogger as mylog from yt.utilities.on_demand_imports import _h5py as h5py +from .fields import SwiftFieldInfo + class SwiftParticleFile(ParticleFile): pass @@ -15,7 +16,7 @@ class SwiftParticleFile(ParticleFile): class SwiftDataset(SPHDataset): _load_requirements = ["h5py"] _index_class = SPHParticleIndex - _field_info_class = SPHFieldInfo + _field_info_class = SwiftFieldInfo _file_class = SwiftParticleFile _particle_mass_name = "Masses" @@ -97,7 +98,15 @@ def _parse_parameter_file(self): # Read from the HDF5 file, this gives us all the info we need. The rest # of this function is just parsing. header = self._get_info_attributes("Header") - runtime_parameters = self._get_info_attributes("RuntimePars") + # RuntimePars were removed from snapshots at SWIFT commit 6271388 + # between SWIFT versions 0.8.5 and 0.9.0 + with h5py.File(self.filename, mode="r") as handle: + has_runtime_pars = "RuntimePars" in handle.keys() + + if has_runtime_pars: + runtime_parameters = self._get_info_attributes("RuntimePars") + else: + runtime_parameters = {} policy = self._get_info_attributes("Policy") # These are the parameterfile parameters from *.yml at runtime @@ -113,7 +122,10 @@ def _parse_parameter_file(self): self.dimensionality = int(header["Dimension"]) # SWIFT is either all periodic, or not periodic at all - periodic = int(runtime_parameters["PeriodicBoundariesOn"]) + if has_runtime_pars: + periodic = int(runtime_parameters["PeriodicBoundariesOn"]) + else: + periodic = int(parameters["InitialConditions:periodic"]) if periodic: self._periodicity = [True] * self.dimensionality @@ -131,7 +143,14 @@ def _parse_parameter_file(self): self.current_redshift = float(header["Redshift"]) # These won't be present if self.cosmological_simulation is false self.omega_lambda = float(parameters["Cosmology:Omega_lambda"]) - self.omega_matter = float(parameters["Cosmology:Omega_m"]) + # Cosmology:Omega_m parameter deprecated at SWIFT commit d2783c2 + # Between SWIFT versions 0.9.0 and 1.0.0 + if "Cosmology:Omega_cdm" in parameters: + self.omega_matter = float(parameters["Cosmology:Omega_b"]) + float( + parameters["Cosmology:Omega_cdm"] + ) + else: + self.omega_matter = float(parameters["Cosmology:Omega_m"]) # This is "little h" self.hubble_constant = float(parameters["Cosmology:h"]) except KeyError: @@ -155,9 +174,10 @@ def _parse_parameter_file(self): # Store the un-parsed information should people want it. self.parameters = { "header": header, - "runtime_parameters": runtime_parameters, "policy": policy, "parameters": parameters, + # NOTE: runtime_parameters may be empty + "runtime_parameters": runtime_parameters, "hydro": hydro, "subgrid": subgrid, } diff --git a/yt/frontends/swift/fields.py b/yt/frontends/swift/fields.py new file mode 100644 index 0000000000..19ec84822c --- /dev/null +++ b/yt/frontends/swift/fields.py @@ -0,0 +1,27 @@ +from yt.frontends.sph.fields import SPHFieldInfo + + +class SwiftFieldInfo(SPHFieldInfo): + def __init__(self, ds, field_list, slice_info=None): + self.known_particle_fields += ( + ( + "InternalEnergies", + ("code_specific_energy", ["specific_thermal_energy"], None), + ), + ("Densities", ("code_mass / code_length**3", ["density"], None)), + ("SmoothingLengths", ("code_length", ["smoothing_length"], None)), + ) + super().__init__(ds, field_list, slice_info) + + def setup_particle_fields(self, ptype, *args, **kwargs): + super().setup_particle_fields(ptype, *args, **kwargs) + + if ptype in ("PartType0", "Gas"): + self.setup_gas_particle_fields(ptype) + + def setup_gas_particle_fields(self, ptype): + self.alias((ptype, "temperature"), (ptype, "Temperatures")) + self.alias(("gas", "temperature"), (ptype, "Temperatures")) + + for ax in ("x", "y", "z"): + self.alias((ptype, ax), (ptype, "particle_position_" + ax)) diff --git a/yt/frontends/swift/io.py b/yt/frontends/swift/io.py index 2cd07d01d9..439176e480 100644 --- a/yt/frontends/swift/io.py +++ b/yt/frontends/swift/io.py @@ -65,8 +65,14 @@ def _get_smoothing_length(self, sub_file, pdtype=None, pshape=None): with h5py.File(sub_file.filename, mode="r") as f: pcount = f["/Header"].attrs["NumPart_ThisFile"][ind].astype("int64") pcount = np.clip(pcount - si, 0, ei - si) + keys = f[ptype].keys() + # SWIFT commit a94cc81 changed from "SmoothingLength" to "SmoothingLengths" + # between SWIFT versions 0.8.2 and 0.8.3 + if "SmoothingLengths" in keys: + hsml = f[ptype]["SmoothingLengths"][si:ei, ...] + else: + hsml = f[ptype]["SmoothingLength"][si:ei, ...] # we upscale to float64 - hsml = f[ptype]["SmoothingLength"][si:ei, ...] hsml = hsml.astype("float64", copy=False) return hsml From c4ce79fa5a04e4ae4c6b84f800aac9583eeb407c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Tue, 7 May 2024 08:44:10 +0200 Subject: [PATCH 055/186] TST: migrate test_disks.py from nose to pytest --- nose_ignores.txt | 1 + pyproject.toml | 1 - yt/data_objects/tests/test_disks.py | 57 +++++++++++++++-------------- 3 files changed, 31 insertions(+), 28 deletions(-) diff --git a/nose_ignores.txt b/nose_ignores.txt index a60796b570..115b917d59 100644 --- a/nose_ignores.txt +++ b/nose_ignores.txt @@ -44,3 +44,4 @@ --ignore-file=test_minimal_representation\.py --ignore-file=test_set_log_level\.py --ignore-file=test_field_parsing\.py +--ignore-file=test_disks\.py diff --git a/pyproject.toml b/pyproject.toml index 37df5e4b76..50d2bd55f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -341,7 +341,6 @@ addopts = ''' --ignore-glob='/*/yt/data_objects/tests/test_connected_sets.py' --ignore-glob='/*/yt/data_objects/tests/test_data_containers.py' --ignore-glob='/*/yt/data_objects/tests/test_dataset_access.py' - --ignore-glob='/*/yt/data_objects/tests/test_disks.py' --ignore-glob='/*/yt/data_objects/tests/test_particle_filter.py' --ignore-glob='/*/yt/data_objects/tests/test_particle_trajectories.py' --ignore-glob='/*/yt/data_objects/tests/test_pickling.py' diff --git a/yt/data_objects/tests/test_disks.py b/yt/data_objects/tests/test_disks.py index 1e2abebe50..8d87ed1d07 100644 --- a/yt/data_objects/tests/test_disks.py +++ b/yt/data_objects/tests/test_disks.py @@ -1,5 +1,4 @@ -from nose.tools import assert_raises -from numpy.testing import assert_equal +import pytest from yt import YTQuantity from yt.testing import fake_random_ds @@ -10,31 +9,37 @@ def test_bad_disk_input(): ds = fake_random_ds(16) # Test invalid 3d array - with assert_raises(TypeError) as ex: + with pytest.raises( + TypeError, + match=r"^Expected an array of size \(3,\), received 'list' of length 4$", + ): ds.disk(ds.domain_center, [0, 0, 1, 1], (10, "kpc"), (20, "kpc")) - desired = "Expected an array of size (3,), received 'list' of length 4" - assert_equal(str(ex.exception), desired) # Test invalid float - with assert_raises(TypeError) as ex: + with pytest.raises( + TypeError, + match=( + r"^Expected a numeric value \(or size-1 array\), " + r"received 'unyt.array.unyt_array' of length 3$" + ), + ): ds.disk(ds.domain_center, [0, 0, 1], ds.domain_center, (20, "kpc")) - desired = ( - "Expected a numeric value (or size-1 array)," - " received 'unyt.array.unyt_array' of length 3" - ) - assert_equal(str(ex.exception), desired) # Test invalid float - with assert_raises(TypeError) as ex: + with pytest.raises( + TypeError, + match=( + r"^Expected a numeric value \(or tuple of format \(float, String\)\), " + r"received an inconsistent tuple '\(10, 10\)'.$" + ), + ): ds.disk(ds.domain_center, [0, 0, 1], (10, 10), (20, "kpc")) - desired = ( - "Expected a numeric value (or tuple of format (float, String))," - " received an inconsistent tuple '(10, 10)'." - ) - assert_equal(str(ex.exception), desired) # Test invalid iterable - with assert_raises(TypeError) as ex: + with pytest.raises( + TypeError, + match=r"^Expected an iterable object, received 'unyt\.array\.unyt_quantity'$", + ): ds.disk( ds.domain_center, [0, 0, 1], @@ -42,18 +47,16 @@ def test_bad_disk_input(): (20, "kpc"), fields=YTQuantity(1, "kpc"), ) - desired = "Expected an iterable object, received 'unyt.array.unyt_quantity'" - assert_equal(str(ex.exception), desired) # Test invalid object - with assert_raises(TypeError) as ex: + with pytest.raises( + TypeError, + match=( + r"^Expected an object of 'yt\.data_objects\.static_output\.Dataset' type, " + r"received 'yt\.data_objects\.selection_objects\.region\.YTRegion'$" + ), + ): ds.disk(ds.domain_center, [0, 0, 1], (10, "kpc"), (20, "kpc"), ds=ds.all_data()) - desired = ( - "Expected an object of 'yt.data_objects.static_output.Dataset' " - "type, received " - "'yt.data_objects.selection_objects.region.YTRegion'" - ) - assert_equal(str(ex.exception), desired) # Test valid disk ds.disk(ds.domain_center, [0, 0, 1], (10, "kpc"), (20, "kpc")) From be99dc527f56e0fdc99da1784fe9406c861f5ea9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 05:35:03 +0000 Subject: [PATCH 056/186] Bump the actions group in /.github/workflows with 2 updates Bumps the actions group in /.github/workflows with 2 updates: [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) and [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `pypa/cibuildwheel` from 2.18.1 to 2.19.1 - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.18.1...v2.19.1) Updates `pypa/gh-action-pypi-publish` from 1.8.14 to 1.9.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.14...v1.9.0) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/wheels.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index 83340ba70d..25d6700171 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -32,7 +32,7 @@ jobs: uses: actions/checkout@v4 - name: Build wheels for CPython - uses: pypa/cibuildwheel@v2.18.1 + uses: pypa/cibuildwheel@v2.19.1 with: output-dir: dist env: @@ -134,7 +134,7 @@ jobs: merge-multiple: true - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.8.14 + uses: pypa/gh-action-pypi-publish@v1.9.0 with: user: __token__ password: ${{ secrets.pypi_token }} From 9fd7c1eebadef9263bcf6cec56047e2cece08912 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 17:17:05 +0000 Subject: [PATCH 057/186] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/adamchainz/blacken-docs: 1.16.0 → 1.18.0](https://github.com/adamchainz/blacken-docs/compare/1.16.0...1.18.0) - [github.com/astral-sh/ruff-pre-commit: v0.4.7 → v0.5.0](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.7...v0.5.0) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4c52974b86..679e525f61 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,13 +30,13 @@ repos: # TODO: replace this with ruff when it supports embedded python blocks # see https://github.com/astral-sh/ruff/issues/8237 - repo: https://github.com/adamchainz/blacken-docs - rev: 1.16.0 + rev: 1.18.0 hooks: - id: blacken-docs additional_dependencies: [black==24.3.0] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.7 + rev: v0.5.0 hooks: - id: ruff-format types_or: [ python, pyi, jupyter ] From 3327feedf220da4c6ce20913d02c0d2726f9c778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Mon, 1 Jul 2024 19:46:13 +0200 Subject: [PATCH 058/186] STY: apply manual fixes for ruff 0.5 (rule E721) --- yt/frontends/amrex/data_structures.py | 2 +- yt/utilities/configuration_tree.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/yt/frontends/amrex/data_structures.py b/yt/frontends/amrex/data_structures.py index 8c6e2756be..a669d9ee41 100644 --- a/yt/frontends/amrex/data_structures.py +++ b/yt/frontends/amrex/data_structures.py @@ -1350,7 +1350,7 @@ def _guess_pcast(vals): pcast = float else: pcast = int - if pcast == bool: + if pcast is bool: vals = [value == "T" for value in vals.split()] else: vals = [pcast(value) for value in vals.split()] diff --git a/yt/utilities/configuration_tree.py b/yt/utilities/configuration_tree.py index 06a561941a..c3b87e86c3 100644 --- a/yt/utilities/configuration_tree.py +++ b/yt/utilities/configuration_tree.py @@ -171,7 +171,7 @@ def value(self): @value.setter def value(self, new_value): - if type(self.value) == type(new_value): + if type(self.value) is type(new_value): self._value = new_value else: tree = self.get_tree() From 88eccc2481cf0455af8fd93820442c41bdbdf8b6 Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Tue, 2 Jul 2024 13:17:06 +0100 Subject: [PATCH 059/186] Do not fail on records with weird lengths Some simulations (like NewHorizon and Horizon-AGN) have weird record lengths that aren't multiple of 1, 2, 4 or 8. This allows to read the dataset nonetheless --- yt/frontends/ramses/particle_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/frontends/ramses/particle_handlers.py b/yt/frontends/ramses/particle_handlers.py index 42d2fb9c11..c8ee5fc416 100644 --- a/yt/frontends/ramses/particle_handlers.py +++ b/yt/frontends/ramses/particle_handlers.py @@ -266,7 +266,7 @@ def build_iterator(): if record_len != exp_len: # Guess record vtype from length nbytes = record_len // hvals["npart"] - vtype = _default_dtypes[nbytes] + vtype = _default_dtypes.get(nbytes, "d") mylog.warning( "Supposed that `%s` has type %s given record size", From 89a8503320922b4867e15dd42cb5070502c7f53a Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Tue, 2 Jul 2024 16:46:36 +0200 Subject: [PATCH 060/186] Add note --- yt/frontends/ramses/particle_handlers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/yt/frontends/ramses/particle_handlers.py b/yt/frontends/ramses/particle_handlers.py index c8ee5fc416..7cfe3ffad1 100644 --- a/yt/frontends/ramses/particle_handlers.py +++ b/yt/frontends/ramses/particle_handlers.py @@ -266,6 +266,9 @@ def build_iterator(): if record_len != exp_len: # Guess record vtype from length nbytes = record_len // hvals["npart"] + # NOTE: in some simulations (e.g. New Horizon), the record length is not + # a multiple of 1, 2, 4 or 8. In this case, fallback onto assuming + # double precision. vtype = _default_dtypes.get(nbytes, "d") mylog.warning( From 7ea1dd14b543a66b90bb4e1736843a5822d3e917 Mon Sep 17 00:00:00 2001 From: "Jordi De Jonghe, PhD" Date: Wed, 3 Jul 2024 10:46:03 +0100 Subject: [PATCH 061/186] ENH: [ARMVAC] implement variable He abundance (#4937) --- yt/frontends/amrvac/data_structures.py | 32 +++++++++++++++----------- yt/frontends/amrvac/io.py | 3 +++ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/yt/frontends/amrvac/data_structures.py b/yt/frontends/amrvac/data_structures.py index 227555b62f..a5bb9c7584 100644 --- a/yt/frontends/amrvac/data_structures.py +++ b/yt/frontends/amrvac/data_structures.py @@ -203,20 +203,8 @@ def __init__( # note: geometry_override and parfiles are specific to this frontend self._geometry_override = geometry_override - super().__init__( - filename, - dataset_type, - units_override=units_override, - unit_system=unit_system, - default_species_fields=default_species_fields, - ) + self._parfiles = [] - self._parfiles = parfiles - - namelist = None - namelist_gamma = None - c_adiab = None - e_is_internal = None if parfiles is not None: parfiles = list(always_iterable(parfiles)) ppf = Path(parfiles[0]) @@ -228,7 +216,22 @@ def __init__( filename, ) parfiles = [Path(ytcfg["yt", "test_data_dir"]) / pf for pf in parfiles] + self._parfiles = parfiles + + super().__init__( + filename, + dataset_type, + units_override=units_override, + unit_system=unit_system, + default_species_fields=default_species_fields, + ) + namelist = None + namelist_gamma = None + c_adiab = None + e_is_internal = None + + if parfiles is not None: namelist = read_amrvac_namelist(parfiles) if "hd_list" in namelist: c_adiab = namelist["hd_list"].get("hd_adiab", 1.0) @@ -373,10 +376,11 @@ def _set_code_unit_attributes(self): # note: yt sets hydrogen mass equal to proton mass, amrvac doesn't. mp_cgs = self.quan(1.672621898e-24, "g") # This value is taken from AstroPy - He_abundance = 0.1 # hardcoded parameter in AMRVAC # get self.length_unit if overrides are supplied, otherwise use default length_unit = getattr(self, "length_unit", self.quan(1, "cm")) + namelist = read_amrvac_namelist(self._parfiles) + He_abundance = namelist.get("mhd_list", {}).get("he_abundance", 0.1) # 1. calculations for mass, density, numberdensity if "mass_unit" in self.units_override: diff --git a/yt/frontends/amrvac/io.py b/yt/frontends/amrvac/io.py index 1be3111a34..a65bdfe6b3 100644 --- a/yt/frontends/amrvac/io.py +++ b/yt/frontends/amrvac/io.py @@ -42,6 +42,9 @@ def read_amrvac_namelist(parfiles): for nml in namelists: unified_namelist.patch(nml) + if "filelist" not in unified_namelist: + return unified_namelist + # accumulate `&filelist:base_filename` base_filename = "".join( nml.get("filelist", {}).get("base_filename", "") for nml in namelists From c86e78b0f4c3d6c69b77857f313efe1a0648b0ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Mon, 8 Jul 2024 11:22:24 +0200 Subject: [PATCH 062/186] TST: add missing @requires_module decorators in particle deposition tests --- yt/geometry/tests/test_particle_deposit.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/yt/geometry/tests/test_particle_deposit.py b/yt/geometry/tests/test_particle_deposit.py index c922e88ed9..fca774c273 100644 --- a/yt/geometry/tests/test_particle_deposit.py +++ b/yt/geometry/tests/test_particle_deposit.py @@ -2,7 +2,7 @@ import yt from yt.loaders import load -from yt.testing import fake_random_ds, requires_file +from yt.testing import fake_random_ds, requires_file, requires_module from yt.utilities.exceptions import YTBoundsDefinitionError @@ -34,6 +34,7 @@ def test_one_zone_octree_deposit(): assert sp["deposit", "io_cic"].shape == (1,) +@requires_module("h5py") @requires_file(RAMSES) @requires_file(ISOGAL) def test_mesh_sampling(): @@ -52,6 +53,7 @@ def test_mesh_sampling(): assert_array_less(-dist, dx) +@requires_module("h5py") @requires_file(RAMSES) @requires_file(ISOGAL) def test_mesh_sampling_for_filtered_particles(): From 61de62d6ad0ee968796ef6ef49fb5a21d9998c5a Mon Sep 17 00:00:00 2001 From: "Eric T. Johnson" Date: Mon, 8 Jul 2024 15:22:19 -0400 Subject: [PATCH 063/186] BUG: fix loading of AMReX datasets with `~` in the filename --- yt/frontends/amrex/data_structures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/frontends/amrex/data_structures.py b/yt/frontends/amrex/data_structures.py index a669d9ee41..ec5d851020 100644 --- a/yt/frontends/amrex/data_structures.py +++ b/yt/frontends/amrex/data_structures.py @@ -652,7 +652,7 @@ def __init__( cparam_filename = cparam_filename or self.__class__._default_cparam_filename self.cparam_filename = self._lookup_cparam_filepath( - output_dir, cparam_filename=cparam_filename + self.output_dir, cparam_filename=cparam_filename ) self.fparam_filename = self._localize_check(fparam_filename) self.storage_filename = storage_filename From 2cbfab909bb082e7fb81c488b2b7b175cc0e6fac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sun, 7 Jul 2024 09:40:22 +0200 Subject: [PATCH 064/186] RFC: explicitly set strict flag in `zip` calls (enforce B905) --- .pre-commit-config.yaml | 7 ++++++- yt/data_objects/analyzer_objects.py | 6 +++++- .../construction_data_containers.py | 19 ++++++++++++----- yt/data_objects/data_containers.py | 8 +++++-- yt/data_objects/derived_quantities.py | 11 +++++++--- .../level_sets/tests/test_clump_finding.py | 8 +++++-- yt/data_objects/profiles.py | 17 ++++++++++----- yt/data_objects/region_expression.py | 6 +++++- yt/data_objects/tests/test_covering_grid.py | 8 ++++++- .../tests/test_derived_quantities.py | 11 ++++++++-- .../tests/test_sph_data_objects.py | 9 +++++++- yt/fields/derived_field.py | 6 +++++- yt/fields/magnetic_field.py | 5 ++++- yt/fields/particle_fields.py | 12 +++++++++-- yt/fields/vector_operations.py | 5 ++++- yt/frontends/amrex/data_structures.py | 8 ++++++- yt/frontends/amrvac/data_structures.py | 8 ++++++- yt/frontends/amrvac/datfile_utils.py | 6 +++++- yt/frontends/art/data_structures.py | 14 ++++++++++--- yt/frontends/art/io.py | 12 +++++++---- yt/frontends/artio/data_structures.py | 8 +++++-- yt/frontends/artio/definitions.py | 8 ++++++- yt/frontends/athena/data_structures.py | 8 ++++++- yt/frontends/athena/io.py | 8 ++++++- yt/frontends/athena_pp/data_structures.py | 8 ++++++- yt/frontends/chombo/io.py | 8 ++++++- yt/frontends/enzo/answer_testing_support.py | 6 +++++- yt/frontends/enzo/data_structures.py | 6 +++++- yt/frontends/enzo_e/data_structures.py | 8 ++++++- yt/frontends/enzo_e/misc.py | 9 ++++++-- yt/frontends/exodus_ii/data_structures.py | 7 ++++++- yt/frontends/exodus_ii/io.py | 7 ++++++- yt/frontends/fits/fields.py | 8 +++++++ yt/frontends/flash/data_structures.py | 16 +++++++++++--- yt/frontends/gadget/io.py | 6 +++++- yt/frontends/gadget/testing.py | 7 ++++++- .../halo_catalog/tests/test_outputs.py | 9 ++++++-- yt/frontends/ramses/tests/test_hilbert.py | 9 ++++++-- yt/frontends/stream/data_structures.py | 6 +++++- yt/frontends/stream/definitions.py | 6 +++++- yt/frontends/tipsy/data_structures.py | 8 ++++++- yt/frontends/tipsy/io.py | 6 +++++- yt/frontends/ytdata/data_structures.py | 6 +++++- yt/geometry/tests/test_grid_container.py | 6 +++++- yt/loaders.py | 11 ++++++---- yt/testing.py | 13 +++++++----- yt/utilities/answer_testing/framework.py | 21 ++++++++++++++++--- yt/utilities/fortran_utils.py | 8 +++++-- yt/utilities/lib/tests/test_fill_region.py | 7 ++++++- yt/utilities/sdf.py | 6 +++++- yt/utilities/tests/test_chemical_formulas.py | 7 ++++++- yt/utilities/tests/test_interpolators.py | 17 ++++++++++++--- yt/visualization/_commons.py | 4 +--- yt/visualization/fits_image.py | 14 +++++++++---- yt/visualization/plot_modifications.py | 6 ++++-- yt/visualization/plot_window.py | 4 +--- .../tests/test_offaxisprojection.py | 6 +++++- yt/visualization/tests/test_particle_plot.py | 21 +++++++++++++++---- yt/visualization/tests/test_plotwindow.py | 19 ++++++++++++++--- .../volume_rendering/old_camera.py | 10 ++++++--- .../volume_rendering/transfer_functions.py | 11 +++++++--- 61 files changed, 441 insertions(+), 114 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 679e525f61..37b92c3559 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,7 +42,12 @@ repos: types_or: [ python, pyi, jupyter ] - id: ruff types_or: [ python, pyi, jupyter ] - args: [--fix, "--show-fixes"] + args: [ + --fix, + --show-fixes, + # the following line can be removed after support for Python 3.9 is dropped + --extend-select=B905, # zip-without-explicit-strict + ] - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 diff --git a/yt/data_objects/analyzer_objects.py b/yt/data_objects/analyzer_objects.py index 97610f8273..b186ac1610 100644 --- a/yt/data_objects/analyzer_objects.py +++ b/yt/data_objects/analyzer_objects.py @@ -1,7 +1,11 @@ import inspect +import sys from yt.utilities.object_registries import analysis_task_registry +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + class AnalysisTask: def __init_subclass__(cls, *args, **kwargs): @@ -14,7 +18,7 @@ def __init__(self, *args, **kwargs): # does not override if len(args) + len(kwargs) != len(self._params): raise RuntimeError - self.__dict__.update(zip(self._params, args)) + self.__dict__.update(zip(self._params, args, strict=False)) self.__dict__.update(kwargs) def __repr__(self): diff --git a/yt/data_objects/construction_data_containers.py b/yt/data_objects/construction_data_containers.py index 03c10f92c1..854f5033d2 100644 --- a/yt/data_objects/construction_data_containers.py +++ b/yt/data_objects/construction_data_containers.py @@ -1,6 +1,7 @@ import fileinput import io import os +import sys import warnings import zipfile from functools import partial, wraps @@ -63,6 +64,9 @@ ) from yt.visualization.color_maps import get_colormap_lut +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + class YTStreamline(YTSelectionContainer1D): """ @@ -150,7 +154,9 @@ def _get_cut_mask(self, grid): mask = np.zeros(points_in_grid.sum(), dtype="int64") dts = np.zeros(points_in_grid.sum(), dtype="float64") ts = np.zeros(points_in_grid.sum(), dtype="float64") - for mi, (i, pos) in enumerate(zip(pids, self.positions[points_in_grid])): + for mi, (i, pos) in enumerate( + zip(pids, self.positions[points_in_grid], strict=True) + ): if not points_in_grid[i]: continue ci = ((pos - grid.LeftEdge) / grid.dds).astype("int64") @@ -784,7 +790,10 @@ def to_xarray(self, fields=None): def icoords(self): ic = np.indices(self.ActiveDimensions).astype("int64") return np.column_stack( - [i.ravel() + gi for i, gi in zip(ic, self.get_global_startindex())] + [ + i.ravel() + gi + for i, gi in zip(ic, self.get_global_startindex(), strict=True) + ] ) @property @@ -1122,7 +1131,7 @@ def _fill_fields(self, fields): if self.comm.size > 1: for i in range(len(fields)): output_fields[i] = self.comm.mpi_allreduce(output_fields[i], op="sum") - for field, v in zip(fields, output_fields): + for field, v in zip(fields, output_fields, strict=True): fi = self.ds._get_field_info(field) self[field] = self.ds.arr(v, fi.units) @@ -1221,7 +1230,7 @@ def write_to_gdf(self, gdf_path, fields, nprocs=1, field_units=None, **kwargs): data[field] = (self[field].in_units(units).v, units) le = self.left_edge.v re = self.right_edge.v - bbox = np.array([[l, r] for l, r in zip(le, re)]) + bbox = np.array([[l, r] for l, r in zip(le, re, strict=True)]) ds = load_uniform_grid( data, self.ActiveDimensions, @@ -1526,7 +1535,7 @@ def _fill_fields(self, fields): stacklevel=1, ) mylog.debug("Caught %d runtime errors.", runtime_errors_count) - for field, v in zip(fields, ls.fields): + for field, v in zip(fields, ls.fields, strict=True): if self.level > 0: v = v[1:-1, 1:-1, 1:-1] fi = self.ds._get_field_info(field) diff --git a/yt/data_objects/data_containers.py b/yt/data_objects/data_containers.py index 80b2283f8d..dfc6ebd487 100644 --- a/yt/data_objects/data_containers.py +++ b/yt/data_objects/data_containers.py @@ -1,4 +1,5 @@ import abc +import sys import weakref from collections import defaultdict from contextlib import contextmanager @@ -28,6 +29,9 @@ from yt.utilities.on_demand_imports import _firefly as firefly from yt.utilities.parameter_file_storage import ParameterFileStore +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + if TYPE_CHECKING: from yt.data_objects.static_output import Dataset @@ -797,7 +801,7 @@ def create_firefly_object( # tuples containing some sort of special "any" ParticleGroup unambiguous_fields_to_include = [] unambiguous_fields_units = [] - for field, field_unit in zip(fields_to_include, fields_units): + for field, field_unit in zip(fields_to_include, fields_units, strict=True): if isinstance(field, tuple): # skip tuples, they'll be checked with _determine_fields unambiguous_fields_to_include.append(field) @@ -849,7 +853,7 @@ def create_firefly_object( field_names = [] ## explicitly go after the fields we want - for field, units in zip(fields_to_include, fields_units): + for field, units in zip(fields_to_include, fields_units, strict=True): ## Only interested in fields with the current particle type, ## whether that means general fields or field tuples ftype, fname = field diff --git a/yt/data_objects/derived_quantities.py b/yt/data_objects/derived_quantities.py index 0ca5eaabb0..8bc41c0474 100644 --- a/yt/data_objects/derived_quantities.py +++ b/yt/data_objects/derived_quantities.py @@ -1,3 +1,5 @@ +import sys + import numpy as np from yt.funcs import camelcase_to_underscore, iter_fields @@ -11,6 +13,9 @@ from yt.utilities.physical_constants import gravitational_constant_cgs from yt.utilities.physical_ratios import HUGE +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + def get_position_fields(field, data): axis_names = [data.ds.coordinates.axis_name[num] for num in [0, 1, 2]] @@ -415,7 +420,7 @@ def __call__(self, fields, weight): fields = list(iter_fields(fields)) units = [self.data_source.ds._get_field_info(field).units for field in fields] rv = super().__call__(fields, weight) - rv = [self.data_source.ds.arr(v, u) for v, u in zip(rv, units)] + rv = [self.data_source.ds.arr(v, u) for v, u in zip(rv, units, strict=True)] if len(rv) == 1: rv = rv[0] return rv @@ -431,7 +436,7 @@ def process_chunk(self, data, fields, weight): my_var2s = [ (data[weight].d * (data[field].d - my_mean) ** 2).sum(dtype=np.float64) / my_weight - for field, my_mean in zip(fields, my_means) + for field, my_mean in zip(fields, my_means, strict=True) ] return my_means + my_var2s + [my_weight] @@ -623,7 +628,7 @@ def reduce_intermediate(self, values): # The values get turned into arrays here. return [ self.data_source.ds.arr([mis.min(), mas.max()]) - for mis, mas in zip(values[::2], values[1::2]) + for mis, mas in zip(values[::2], values[1::2], strict=True) ] diff --git a/yt/data_objects/level_sets/tests/test_clump_finding.py b/yt/data_objects/level_sets/tests/test_clump_finding.py index e44bd87033..bd7059f8c3 100644 --- a/yt/data_objects/level_sets/tests/test_clump_finding.py +++ b/yt/data_objects/level_sets/tests/test_clump_finding.py @@ -1,5 +1,6 @@ import os import shutil +import sys import tempfile import numpy as np @@ -12,6 +13,9 @@ from yt.testing import requires_file, requires_module from yt.utilities.answer_testing.framework import data_dir_load +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + def test_clump_finding(): n_c = 8 @@ -132,7 +136,7 @@ def test_clump_tree_save(): it2 = np.argsort(mt2).astype("int64") assert_array_equal(mt1[it1], mt2[it2]) - for i1, i2 in zip(it1, it2): + for i1, i2 in zip(it1, it2, strict=True): ct1 = t1[i1] ct2 = t2[i2] assert_array_equal(ct1["gas", "density"], ct2["grid", "density"]) @@ -191,5 +195,5 @@ def _also_density(field, data): leaf_clumps_1 = master_clump_1.leaves leaf_clumps_2 = master_clump_2.leaves - for c1, c2 in zip(leaf_clumps_1, leaf_clumps_2): + for c1, c2 in zip(leaf_clumps_1, leaf_clumps_2, strict=True): assert_array_equal(c1["gas", "density"], c2["gas", "density"]) diff --git a/yt/data_objects/profiles.py b/yt/data_objects/profiles.py index f954620543..c5fc6ef6a5 100644 --- a/yt/data_objects/profiles.py +++ b/yt/data_objects/profiles.py @@ -1,3 +1,5 @@ +import sys + import numpy as np from more_itertools import collapse @@ -23,6 +25,9 @@ parallel_objects, ) +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + def _sanitize_min_max_units(amin, amax, finfo, registry): # returns a copy of amin and amax, converted to finfo's output units @@ -217,7 +222,7 @@ def _filter(self, bin_fields): # cut_points is set to be everything initially, but # we also want to apply a filtering based on min/max pfilter = np.ones(bin_fields[0].shape, dtype="bool") - for (mi, ma), data in zip(self.bounds, bin_fields): + for (mi, ma), data in zip(self.bounds, bin_fields, strict=True): pfilter &= data > mi pfilter &= data < ma return pfilter, [data[pfilter] for data in bin_fields] @@ -1313,7 +1318,9 @@ def create_profile( data_source.ds.field_info[f].sampling_type == "local" for f in bin_fields + fields ] - is_local_or_pfield = [pf or lf for (pf, lf) in zip(is_pfield, is_local)] + is_local_or_pfield = [ + pf or lf for (pf, lf) in zip(is_pfield, is_local, strict=True) + ] if not all(is_local_or_pfield): raise YTIllDefinedProfile( bin_fields, data_source._determine_fields(fields), wf, is_pfield @@ -1370,7 +1377,7 @@ def create_profile( if extrema is None or not any(collapse(extrema.values())): ex = [ data_source.quantities["Extrema"](f, non_zero=l) - for f, l in zip(bin_fields, logs) + for f, l in zip(bin_fields, logs, strict=True) ] # pad extrema by epsilon so cells at bin edges are not excluded for i, (mi, ma) in enumerate(ex): @@ -1456,7 +1463,7 @@ def create_profile( o_bins.append(field_obin) args = [data_source] - for f, n, (mi, ma), l in zip(bin_fields, n_bins, ex, logs): + for f, n, (mi, ma), l in zip(bin_fields, n_bins, ex, logs, strict=True): if mi <= 0 and l: raise YTIllDefinedBounds(mi, ma) args += [f, n, mi, ma, l] @@ -1464,7 +1471,7 @@ def create_profile( if cls is ParticleProfile: kwargs["deposition"] = deposition if override_bins is not None: - for o_bin, ax in zip(o_bins, ["x", "y", "z"]): + for o_bin, ax in zip(o_bins, ["x", "y", "z"], strict=False): kwargs[f"override_bins_{ax}"] = o_bin obj = cls(*args, **kwargs) obj.accumulation = accumulation diff --git a/yt/data_objects/region_expression.py b/yt/data_objects/region_expression.py index 821e291bd4..c6082ff93e 100644 --- a/yt/data_objects/region_expression.py +++ b/yt/data_objects/region_expression.py @@ -1,3 +1,4 @@ +import sys import weakref from functools import cached_property @@ -12,6 +13,9 @@ from .data_containers import _get_ipython_key_completion +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + class RegionExpression: def __init__(self, ds): @@ -166,7 +170,7 @@ def _create_region(self, bounds_tuple): if d is not None: d = int(d) dims[ax] = d - center = [(cl + cr) / 2.0 for cl, cr in zip(left_edge, right_edge)] + center = [(cl + cr) / 2.0 for cl, cr in zip(left_edge, right_edge, strict=True)] if None not in dims: return self.ds.arbitrary_grid(left_edge, right_edge, dims) return self.ds.region(center, left_edge, right_edge) diff --git a/yt/data_objects/tests/test_covering_grid.py b/yt/data_objects/tests/test_covering_grid.py index 376b7d3b0d..ed59776409 100644 --- a/yt/data_objects/tests/test_covering_grid.py +++ b/yt/data_objects/tests/test_covering_grid.py @@ -1,3 +1,5 @@ +import sys + import numpy as np from numpy.testing import assert_almost_equal, assert_array_equal, assert_equal @@ -11,6 +13,10 @@ ) from yt.units import kpc +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + + # cylindrical data for covering_grid test cyl_2d = "WDMerger_hdf5_chk_1000/WDMerger_hdf5_chk_1000.hdf5" cyl_3d = "MHD_Cyl3d_hdf5_plt_cnt_0100/MHD_Cyl3d_hdf5_plt_cnt_0100.hdf5" @@ -354,7 +360,7 @@ def test_arbitrary_grid_edge(): [1.0, 1.0, 1.0] * kpc, ] - for le, re, le_ans, re_ans in zip(ledge, redge, ledge_ans, redge_ans): + for le, re, le_ans, re_ans in zip(ledge, redge, ledge_ans, redge_ans, strict=True): ag = ds.arbitrary_grid(left_edge=le, right_edge=re, dims=dims) assert np.array_equal(ag.left_edge, le_ans) assert np.array_equal(ag.right_edge, re_ans) diff --git a/yt/data_objects/tests/test_derived_quantities.py b/yt/data_objects/tests/test_derived_quantities.py index 730ad06c60..60bc9223e4 100644 --- a/yt/data_objects/tests/test_derived_quantities.py +++ b/yt/data_objects/tests/test_derived_quantities.py @@ -1,3 +1,5 @@ +import sys + import numpy as np from numpy.testing import assert_almost_equal, assert_equal @@ -11,6 +13,9 @@ requires_file, ) +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + def setup_module(): from yt.config import ytcfg @@ -183,10 +188,12 @@ def test_in_memory_sph_derived_quantities(): assert_equal(com, [1 / 7, (1 + 2) / 7, (1 + 2 + 3) / 7]) ex = ad.quantities.extrema([("io", "x"), ("io", "y"), ("io", "z")]) - for fex, ans in zip(ex, [[0, 1], [0, 2], [0, 3]]): + for fex, ans in zip(ex, [[0, 1], [0, 2], [0, 3]], strict=True): assert_equal(fex, ans) - for d, v, l in zip("xyz", [1, 2, 3], [[1, 0, 0], [0, 2, 0], [0, 0, 3]]): + for d, v, l in zip( + "xyz", [1, 2, 3], [[1, 0, 0], [0, 2, 0], [0, 0, 3]], strict=False + ): max_d, x, y, z = ad.quantities.max_location(("io", d)) assert_equal(max_d, v) assert_equal([x, y, z], l) diff --git a/yt/data_objects/tests/test_sph_data_objects.py b/yt/data_objects/tests/test_sph_data_objects.py index 990759af57..ebe350cc24 100644 --- a/yt/data_objects/tests/test_sph_data_objects.py +++ b/yt/data_objects/tests/test_sph_data_objects.py @@ -1,3 +1,5 @@ +import sys + import numpy as np from numpy.testing import assert_allclose, assert_almost_equal, assert_equal @@ -5,6 +7,11 @@ from yt.loaders import load from yt.testing import fake_sph_grid_ds, fake_sph_orientation_ds, requires_file +if sys.version_info >= (3, 10): + pass +else: + from yt._maintenance.backports import zip + def test_point(): ds = fake_sph_orientation_ds() @@ -92,7 +99,7 @@ def test_periodic_region(): for y in coords: for z in coords: center = np.array([x, y, z]) - for n, w in zip((8, 27), (1.0, 2.0)): + for n, w in zip((8, 27), (1.0, 2.0), strict=True): le = center - 0.5 * w re = center + 0.5 * w box = ds.box(le, re) diff --git a/yt/fields/derived_field.py b/yt/fields/derived_field.py index 7e90277b2d..cfb1c4d59b 100644 --- a/yt/fields/derived_field.py +++ b/yt/fields/derived_field.py @@ -1,6 +1,7 @@ import contextlib import inspect import re +import sys from collections.abc import Iterable from typing import Optional, Union @@ -25,6 +26,9 @@ NeedsProperty, ) +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + def TranslationFunc(field_name): def _TranslationFunc(field, data): @@ -255,7 +259,7 @@ def _get_needed_parameters(self, fd): else: params.extend(val.parameters) values.extend([fd.get_field_parameter(fp) for fp in val.parameters]) - return dict(zip(params, values)), permute_params + return dict(zip(params, values, strict=True)), permute_params _unit_registry = None diff --git a/yt/fields/magnetic_field.py b/yt/fields/magnetic_field.py index d9e7cb5792..8eab02ff3d 100644 --- a/yt/fields/magnetic_field.py +++ b/yt/fields/magnetic_field.py @@ -10,6 +10,9 @@ from .field_plugin_registry import register_field_plugin +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + if sys.version_info >= (3, 11): from typing import assert_never else: @@ -330,7 +333,7 @@ def _mag_field(field, data): return _mag_field - for ax, fd in zip(registry.ds.coordinates.axis_order, ds_fields): + for ax, fd in zip(registry.ds.coordinates.axis_order, ds_fields, strict=False): registry.add_field( (ftype, f"magnetic_field_{ax}"), sampling_type=sampling_type, diff --git a/yt/fields/particle_fields.py b/yt/fields/particle_fields.py index 7db9bfabea..c38c2fdf75 100644 --- a/yt/fields/particle_fields.py +++ b/yt/fields/particle_fields.py @@ -1,3 +1,5 @@ +import sys + import numpy as np from yt.fields.derived_field import ValidateParameter, ValidateSpatial @@ -24,6 +26,12 @@ from .field_functions import get_radius from .vector_operations import create_magnitude_field +if sys.version_info >= (3, 10): + pass +else: + from yt._maintenance.backports import zip + + sph_whitelist_fields = ( "density", "temperature", @@ -184,7 +192,7 @@ def _deposit_field(field, data): return _deposit_field for ax in "xyz": - for method, name in zip(("cic", "sum"), ("cic", "nn")): + for method, name in zip(("cic", "sum"), ("cic", "nn"), strict=True): function = _get_density_weighted_deposit_field( f"particle_velocity_{ax}", "code_velocity", method ) @@ -197,7 +205,7 @@ def _deposit_field(field, data): validators=[ValidateSpatial(0)], ) - for method, name in zip(("cic", "sum"), ("cic", "nn")): + for method, name in zip(("cic", "sum"), ("cic", "nn"), strict=True): function = _get_density_weighted_deposit_field("age", "code_time", method) registry.add_field( ("deposit", ("%s_" + name + "_age") % (ptype)), diff --git a/yt/fields/vector_operations.py b/yt/fields/vector_operations.py index 58eeba0a17..860cf076ea 100644 --- a/yt/fields/vector_operations.py +++ b/yt/fields/vector_operations.py @@ -19,6 +19,9 @@ from .derived_field import NeedsParameter, ValidateParameter, ValidateSpatial +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + if sys.version_info >= (3, 11): from typing import assert_never else: @@ -676,7 +679,7 @@ def atleast_4d(array): ) i_i, j_i, k_i = np.mgrid[0:3, 0:3, 0:3] - for i, j, k in zip(i_i.ravel(), j_i.ravel(), k_i.ravel()): + for i, j, k in zip(i_i.ravel(), j_i.ravel(), k_i.ravel(), strict=True): sl = ( slice(i, nx - (2 - i)), slice(j, ny - (2 - j)), diff --git a/yt/frontends/amrex/data_structures.py b/yt/frontends/amrex/data_structures.py index ec5d851020..e2a201d5ec 100644 --- a/yt/frontends/amrex/data_structures.py +++ b/yt/frontends/amrex/data_structures.py @@ -1,6 +1,7 @@ import glob import os import re +import sys from collections import namedtuple from functools import cached_property from stat import ST_CTIME @@ -25,6 +26,11 @@ WarpXFieldInfo, ) +if sys.version_info >= (3, 10): + pass +else: + from yt._maintenance.backports import zip + # This is what we use to find scientific notation that might include d's # instead of e's. _scinot_finder = re.compile(r"[-+]?[0-9]*\.?[0-9]+([eEdD][-+]?[0-9]+)?") @@ -833,7 +839,7 @@ def _parse_header_file(self): # in a slightly hidden variable. self._max_level = int(header_file.readline()) - for side, init in zip(["left", "right"], [np.zeros, np.ones]): + for side, init in zip(["left", "right"], [np.zeros, np.ones], strict=True): domain_edge = init(3, dtype="float64") domain_edge[: self.dimensionality] = header_file.readline().split() setattr(self, f"domain_{side}_edge", domain_edge) diff --git a/yt/frontends/amrvac/data_structures.py b/yt/frontends/amrvac/data_structures.py index a5bb9c7584..5b8d22ba54 100644 --- a/yt/frontends/amrvac/data_structures.py +++ b/yt/frontends/amrvac/data_structures.py @@ -7,6 +7,7 @@ import os import struct +import sys import warnings import weakref from pathlib import Path @@ -26,6 +27,9 @@ from .fields import AMRVACFieldInfo from .io import read_amrvac_namelist +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + def _parse_geometry(geometry_tag: str) -> Geometry: """Translate AMRVAC's geometry tag to yt's format. @@ -142,7 +146,9 @@ def _parse_index(self): dim = self.dataset.dimensionality self.grids = np.empty(self.num_grids, dtype="object") - for igrid, (ytlevel, morton_index) in enumerate(zip(ytlevels, morton_indices)): + for igrid, (ytlevel, morton_index) in enumerate( + zip(ytlevels, morton_indices, strict=True) + ): dx = dx0 / self.dataset.refine_by**ytlevel left_edge = xmin + (morton_index - 1) * block_nx * dx diff --git a/yt/frontends/amrvac/datfile_utils.py b/yt/frontends/amrvac/datfile_utils.py index 379be74f93..21436fd2d5 100644 --- a/yt/frontends/amrvac/datfile_utils.py +++ b/yt/frontends/amrvac/datfile_utils.py @@ -1,7 +1,11 @@ import struct +import sys import numpy as np +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + # Size of basic types (in bytes) SIZE_LOGICAL = 4 SIZE_INT = 4 @@ -98,7 +102,7 @@ def get_header(istream): ] # Store the values corresponding to the names - for val, name in zip(vals, names): + for val, name in zip(vals, names, strict=True): h[name] = val return h diff --git a/yt/frontends/art/data_structures.py b/yt/frontends/art/data_structures.py index 0bfe58bda9..d13a2f77e7 100644 --- a/yt/frontends/art/data_structures.py +++ b/yt/frontends/art/data_structures.py @@ -1,6 +1,7 @@ import glob import os import struct +import sys import weakref import numpy as np @@ -33,6 +34,9 @@ from yt.geometry.oct_geometry_handler import OctreeIndex from yt.geometry.particle_geometry_handler import ParticleIndex +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + class ARTIndex(OctreeIndex): def __init__(self, ds, dataset_type="art"): @@ -328,7 +332,9 @@ def _parse_parameter_file(self): mylog.info("Discovered %i species of particles", len(ls_nonzero)) info_str = "Particle populations: " + "%9i " * len(ls_nonzero) mylog.info(info_str, *ls_nonzero) - self._particle_type_counts = dict(zip(self.particle_types_raw, ls_nonzero)) + self._particle_type_counts = dict( + zip(self.particle_types_raw, ls_nonzero, strict=True) + ) for k, v in particle_header_vals.items(): if k in self.parameters.keys(): if not self.parameters[k] == v: @@ -396,7 +402,9 @@ def __init__(self, ds, io, filename, file_id): super().__init__(ds, io, filename, file_id, range=None) self.total_particles = {} for ptype, count in zip( - ds.particle_types_raw, ds.parameters["total_particles"] + ds.particle_types_raw, + ds.parameters["total_particles"], + strict=True, ): self.total_particles[ptype] = count with open(filename, "rb") as f: @@ -754,7 +762,7 @@ def fill(self, content, ftfields, selector): content, self.domain.level_child_offsets, self.domain.level_count ) ns = (self.domain.ds.domain_dimensions.prod() // 8, 8) - for field, fi in zip(fields, field_idxs): + for field, fi in zip(fields, field_idxs, strict=True): source[field] = np.empty(ns, dtype="float64", order="C") dt = data[fi, :].reshape(self.domain.ds.domain_dimensions, order="F") for i in range(2): diff --git a/yt/frontends/art/io.py b/yt/frontends/art/io.py index 1047928ce7..13813677ab 100644 --- a/yt/frontends/art/io.py +++ b/yt/frontends/art/io.py @@ -1,5 +1,6 @@ import os import os.path +import sys from collections import defaultdict from functools import partial @@ -16,6 +17,9 @@ from yt.utilities.io_handler import BaseIOHandler from yt.utilities.logger import ytLogger as mylog +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + class IOHandlerART(BaseIOHandler): _dataset_type = "art" @@ -125,7 +129,7 @@ def _get_field(self, field): if fname.startswith("particle_mass"): a = 0 data = np.zeros(npa, dtype="f8") - for ptb, size, m in zip(pbool, sizes, self.ws): + for ptb, size, m in zip(pbool, sizes, self.ws, strict=True): if ptb: data[a : a + size] = m a += size @@ -135,7 +139,7 @@ def _get_field(self, field): elif fname == "particle_type": a = 0 data = np.zeros(npa, dtype="int64") - for i, (ptb, size) in enumerate(zip(pbool, sizes)): + for i, (ptb, size) in enumerate(zip(pbool, sizes, strict=True)): if ptb: data[a : a + size] = i a += size @@ -218,7 +222,7 @@ def _get_field(self, field): if fname.startswith("particle_mass"): a = 0 data = np.zeros(npa, dtype="f8") - for ptb, size, m in zip(pbool, sizes, self.ws): + for ptb, size, m in zip(pbool, sizes, self.ws, strict=True): if ptb: data[a : a + size] = m a += size @@ -228,7 +232,7 @@ def _get_field(self, field): elif fname == "particle_type": a = 0 data = np.zeros(npa, dtype="int64") - for i, (ptb, size) in enumerate(zip(pbool, sizes)): + for i, (ptb, size) in enumerate(zip(pbool, sizes, strict=True)): if ptb: data[a : a + size] = i a += size diff --git a/yt/frontends/artio/data_structures.py b/yt/frontends/artio/data_structures.py index 878b86c1ef..e9acba4e68 100644 --- a/yt/frontends/artio/data_structures.py +++ b/yt/frontends/artio/data_structures.py @@ -1,4 +1,5 @@ import os +import sys import weakref from collections import defaultdict from typing import Optional @@ -21,6 +22,9 @@ from yt.geometry.geometry_handler import Index, YTDataChunk from yt.utilities.exceptions import YTParticleDepositionNotImplemented +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + class ARTIOOctreeSubset(OctreeSubset): _domain_offset = 0 @@ -72,7 +76,7 @@ def fill(self, fields, selector): self.oct_handler.fill_sfc( levels, cell_inds, file_inds, domain_counts, field_indices, tr ) - tr = dict(zip(fields, tr)) + tr = dict(zip(fields, tr, strict=True)) return tr def fill_particles(self, fields): @@ -118,7 +122,7 @@ def fill(self, fields, selector): ] tr = self.oct_handler.fill_sfc(selector, field_indices) self.data_size = tr[0].size - tr = dict(zip(fields, tr)) + tr = dict(zip(fields, tr, strict=True)) return tr def deposit(self, positions, fields=None, method=None, kernel_name="cubic"): diff --git a/yt/frontends/artio/definitions.py b/yt/frontends/artio/definitions.py index e4b39266be..b054c5d8d3 100644 --- a/yt/frontends/artio/definitions.py +++ b/yt/frontends/artio/definitions.py @@ -1,3 +1,9 @@ +import sys + +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + + yt_to_art = { "Density": "HVAR_GAS_DENSITY", "TotalEnergy": "HVAR_GAS_ENERGY", @@ -27,7 +33,7 @@ "stars": "STAR", "nbody": "N-BODY", } -art_to_yt = dict(zip(yt_to_art.values(), yt_to_art.keys())) +art_to_yt = dict(zip(yt_to_art.values(), yt_to_art.keys(), strict=True)) class ARTIOconstants: diff --git a/yt/frontends/athena/data_structures.py b/yt/frontends/athena/data_structures.py index c0e1a92cf2..57e8058a6b 100644 --- a/yt/frontends/athena/data_structures.py +++ b/yt/frontends/athena/data_structures.py @@ -1,5 +1,6 @@ import os import re +import sys import weakref import numpy as np @@ -17,6 +18,9 @@ from .fields import AthenaFieldInfo +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + def chk23(strin): return strin.encode("utf-8") @@ -359,7 +363,9 @@ def _parse_index(self): gre_orig = self.ds.arr( np.round(gle_orig + dx * gdims[i], decimals=12), "code_length" ) - bbox = np.array([[le, re] for le, re in zip(gle_orig, gre_orig)]) + bbox = np.array( + [[le, re] for le, re in zip(gle_orig, gre_orig, strict=True)] + ) psize = get_psize(self.ds.domain_dimensions, self.ds.nprocs) gle, gre, shapes, slices, _ = decompose_array(gdims[i], psize, bbox) gle_all += gle diff --git a/yt/frontends/athena/io.py b/yt/frontends/athena/io.py index d002928736..d4e2de7ddd 100644 --- a/yt/frontends/athena/io.py +++ b/yt/frontends/athena/io.py @@ -1,3 +1,5 @@ +import sys + import numpy as np from yt.funcs import mylog @@ -5,6 +7,10 @@ from .data_structures import chk23 +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + + float_size = {"float": np.dtype(">f4").itemsize, "double": np.dtype(">f8").itemsize} axis_list = ["_x", "_y", "_z"] @@ -19,7 +25,7 @@ class IOHandlerAthena(BaseIOHandler): def _field_dict(self, fhandle): keys = fhandle["field_types"].keys() val = fhandle["field_types"].keys() - return dict(zip(keys, val)) + return dict(zip(keys, val, strict=True)) def _read_field_names(self, grid): pass diff --git a/yt/frontends/athena_pp/data_structures.py b/yt/frontends/athena_pp/data_structures.py index 37f6881205..8c7bffd1fa 100644 --- a/yt/frontends/athena_pp/data_structures.py +++ b/yt/frontends/athena_pp/data_structures.py @@ -1,4 +1,5 @@ import os +import sys import weakref import numpy as np @@ -15,6 +16,9 @@ from .fields import AthenaPPFieldInfo +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + geom_map = { "cartesian": "cartesian", "cylindrical": "cylindrical", @@ -215,7 +219,9 @@ def _parse_parameter_file(self): self._field_map = {} k = 0 for dname, num_var in zip( - self._handle.attrs["DatasetNames"], self._handle.attrs["NumVariables"] + self._handle.attrs["DatasetNames"], + self._handle.attrs["NumVariables"], + strict=True, ): for j in range(num_var): fname = self._handle.attrs["VariableNames"][k].decode("ascii", "ignore") diff --git a/yt/frontends/chombo/io.py b/yt/frontends/chombo/io.py index a6f11d9a72..e6a95e4dae 100644 --- a/yt/frontends/chombo/io.py +++ b/yt/frontends/chombo/io.py @@ -1,4 +1,5 @@ import re +import sys import numpy as np @@ -6,6 +7,9 @@ from yt.utilities.io_handler import BaseIOHandler from yt.utilities.logger import ytLogger as mylog +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + class IOHandlerChomboHDF5(BaseIOHandler): _dataset_type = "chombo_hdf5" @@ -101,7 +105,9 @@ def _read_data(self, grid, field): stop = start + boxsize data = lev[self._data_string][start:stop] data_no_ghost = data.reshape(shape, order="F") - ghost_slice = tuple(slice(g, d + g, None) for g, d in zip(self.ghost, dims)) + ghost_slice = tuple( + slice(g, d + g, None) for g, d in zip(self.ghost, dims, strict=True) + ) ghost_slice = ghost_slice[0 : self.dim] return data_no_ghost[ghost_slice] diff --git a/yt/frontends/enzo/answer_testing_support.py b/yt/frontends/enzo/answer_testing_support.py index bba9dba062..bba14f7db1 100644 --- a/yt/frontends/enzo/answer_testing_support.py +++ b/yt/frontends/enzo/answer_testing_support.py @@ -1,4 +1,5 @@ import os +import sys from functools import wraps import numpy as np @@ -15,6 +16,9 @@ temp_cwd, ) +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + class AssertWrapper: """ @@ -102,7 +106,7 @@ def __call__(self): position = ad["index", "x"] for k in self.fields: field = ad[k].d - for xmin, xmax in zip(self.left_edges, self.right_edges): + for xmin, xmax in zip(self.left_edges, self.right_edges, strict=True): mask = (position >= xmin) * (position <= xmax) exact_field = np.interp(position[mask].ndview, exact["pos"], exact[k]) myname = f"ShockTubeTest_{k}" diff --git a/yt/frontends/enzo/data_structures.py b/yt/frontends/enzo/data_structures.py index 16f36b3305..ec28a10c54 100644 --- a/yt/frontends/enzo/data_structures.py +++ b/yt/frontends/enzo/data_structures.py @@ -1,6 +1,7 @@ import os import re import string +import sys import time import weakref from collections import defaultdict @@ -21,6 +22,9 @@ from .fields import EnzoFieldInfo +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + class EnzoGrid(AMRGridPatch): """ @@ -342,7 +346,7 @@ def _rebuild_top_grids(self, level=0): mylog.info("Finished rebuilding") def _populate_grid_objects(self): - for g, f in zip(self.grids, self.filenames): + for g, f in zip(self.grids, self.filenames, strict=True): g._prepare_grid() g._setup_dx() g.set_filename(f[0]) diff --git a/yt/frontends/enzo_e/data_structures.py b/yt/frontends/enzo_e/data_structures.py index a77c245d0e..5832e09e62 100644 --- a/yt/frontends/enzo_e/data_structures.py +++ b/yt/frontends/enzo_e/data_structures.py @@ -1,4 +1,5 @@ import os +import sys from functools import cached_property import numpy as np @@ -23,6 +24,11 @@ from yt.utilities.logger import ytLogger as mylog from yt.utilities.on_demand_imports import _h5py as h5py, _libconf as libconf +if sys.version_info >= (3, 10): + pass +else: + from yt._maintenance.backports import zip + class EnzoEGrid(AMRGridPatch): """ @@ -475,7 +481,7 @@ def _set_code_unit_attributes(self): setdefaultattr(self, "velocity_unit", self.quan(k["uvel"], "cm/s")) else: p = self.parameters - for d, u in zip(("length", "time"), ("cm", "s")): + for d, u in zip(("length", "time"), ("cm", "s"), strict=True): val = nested_dict_get(p, ("Units", d), default=1) setdefaultattr(self, f"{d}_unit", self.quan(val, u)) mass = nested_dict_get(p, ("Units", "mass")) diff --git a/yt/frontends/enzo_e/misc.py b/yt/frontends/enzo_e/misc.py index a1528b6963..c95de312cb 100644 --- a/yt/frontends/enzo_e/misc.py +++ b/yt/frontends/enzo_e/misc.py @@ -1,6 +1,11 @@ +import sys + import numpy as np from more_itertools import always_iterable +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + def bdecode(block): """ @@ -89,7 +94,7 @@ def get_root_block_id(block, min_dim=3): def get_child_index(anc_id, desc_id): cid = "" - for aind, dind in zip(anc_id.split("_"), desc_id.split("_")): + for aind, dind in zip(anc_id.split("_"), desc_id.split("_"), strict=True): cid += dind[len(aind)] cid = int(cid, 2) return cid @@ -100,7 +105,7 @@ def is_parent(anc_block, desc_block): if (len(desc_block.replace(":", "")) - len(anc_block.replace(":", ""))) / dim != 1: return False - for aind, dind in zip(anc_block.split("_"), desc_block.split("_")): + for aind, dind in zip(anc_block.split("_"), desc_block.split("_"), strict=True): if not dind.startswith(aind): return False return True diff --git a/yt/frontends/exodus_ii/data_structures.py b/yt/frontends/exodus_ii/data_structures.py index 52ef9a0c66..7ff0846557 100644 --- a/yt/frontends/exodus_ii/data_structures.py +++ b/yt/frontends/exodus_ii/data_structures.py @@ -1,3 +1,5 @@ +import sys + import numpy as np from yt.data_objects.index_subobjects.unstructured_mesh import UnstructuredMesh @@ -11,6 +13,9 @@ from .fields import ExodusIIFieldInfo from .util import get_num_pseudo_dims, load_info_records, sanitize_string +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + class ExodusIIUnstructuredMesh(UnstructuredMesh): _index_offset = 1 @@ -220,7 +225,7 @@ def _read_glo_var(self): return with self._handle.open_ds() as ds: values = ds.variables["vals_glo_var"][:].transpose() - for name, value in zip(names, values): + for name, value in zip(names, values, strict=True): self.parameters[name] = value def _load_info_records(self): diff --git a/yt/frontends/exodus_ii/io.py b/yt/frontends/exodus_ii/io.py index b9dbb81e68..ce8b32210a 100644 --- a/yt/frontends/exodus_ii/io.py +++ b/yt/frontends/exodus_ii/io.py @@ -1,8 +1,13 @@ +import sys + import numpy as np from yt.utilities.file_handler import NetCDF4FileHandler from yt.utilities.io_handler import BaseIOHandler +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + class IOHandlerExodusII(BaseIOHandler): _particle_reader = False @@ -69,7 +74,7 @@ def _read_fluid_selection(self, chunks, selector, fields, size): ind += g.select(selector, data, rv[field], ind) # caches if fname in self.elem_fields: field_ind = self.elem_fields.index(fname) - for g, mesh_id in zip(objs, mesh_ids): + for g, mesh_id in zip(objs, mesh_ids, strict=True): fdata = ds.variables[ "vals_elem_var%deb%s" % (field_ind + 1, mesh_id) ][:] diff --git a/yt/frontends/fits/fields.py b/yt/frontends/fits/fields.py index bc09189ade..171961845b 100644 --- a/yt/frontends/fits/fields.py +++ b/yt/frontends/fits/fields.py @@ -1,6 +1,13 @@ +import sys + from yt._typing import KnownFieldsT from yt.fields.field_info_container import FieldInfoContainer +if sys.version_info >= (3, 10): + pass +else: + from yt._maintenance.backports import zip + class FITSFieldInfo(FieldInfoContainer): known_other_fields = () @@ -73,6 +80,7 @@ def _world_f(field, data): for (i, axis), name in zip( enumerate([self.ds.lon_axis, self.ds.lat_axis]), [self.ds.lon_name, self.ds.lat_name], + strict=True, ): unit = str(wcs_2d.wcs.cunit[i]) if unit.lower() == "deg": diff --git a/yt/frontends/flash/data_structures.py b/yt/frontends/flash/data_structures.py index abc8d0d654..6456e22984 100644 --- a/yt/frontends/flash/data_structures.py +++ b/yt/frontends/flash/data_structures.py @@ -1,4 +1,5 @@ import os +import sys import weakref import numpy as np @@ -15,6 +16,9 @@ from .fields import FLASHFieldInfo +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + class FLASHGrid(AMRGridPatch): _id_offset = 1 @@ -274,7 +278,9 @@ def _find_parameter(self, ptype, pname, scalar=False): if nn not in self._handle: raise KeyError(nn) for tpname, pval in zip( - self._handle[nn][:, "name"], self._handle[nn][:, "value"] + self._handle[nn][:, "name"], + self._handle[nn][:, "value"], + strict=True, ): if tpname.decode("ascii", "ignore").strip() == pname: if hasattr(pval, "decode"): @@ -306,7 +312,9 @@ def _parse_parameter_file(self): if hn not in self._handle: continue for varname, val in zip( - self._handle[hn][:, "name"], self._handle[hn][:, "value"] + self._handle[hn][:, "name"], + self._handle[hn][:, "value"], + strict=True, ): vn = varname.strip() if hn.startswith("string"): @@ -333,7 +341,9 @@ def _parse_parameter_file(self): ) else: zipover = zip( - self._handle[hn][:, "name"], self._handle[hn][:, "value"] + self._handle[hn][:, "name"], + self._handle[hn][:, "value"], + strict=True, ) for varname, val in zipover: vn = varname.strip() diff --git a/yt/frontends/gadget/io.py b/yt/frontends/gadget/io.py index bdf3c263cd..cc10d34842 100644 --- a/yt/frontends/gadget/io.py +++ b/yt/frontends/gadget/io.py @@ -1,4 +1,5 @@ import os +import sys from collections import defaultdict from functools import cached_property @@ -12,6 +13,9 @@ from .definitions import SNAP_FORMAT_2_OFFSET, gadget_hdf5_ptypes +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + class IOHandlerGadgetHDF5(IOHandlerSPH): _dataset_type = "gadget_hdf5" @@ -509,7 +513,7 @@ def _calculate_field_offsets( pos = offset fs = self._field_size offsets = {} - pcount = dict(zip(self._ptypes, pcount)) + pcount = dict(zip(self._ptypes, pcount, strict=True)) for field in self._fields: if field == "ParticleIDs" and self.ds.long_ids: diff --git a/yt/frontends/gadget/testing.py b/yt/frontends/gadget/testing.py index 349b901769..f37aa01ae9 100644 --- a/yt/frontends/gadget/testing.py +++ b/yt/frontends/gadget/testing.py @@ -1,9 +1,14 @@ +import sys + import numpy as np from .data_structures import GadgetBinaryHeader, GadgetDataset from .definitions import gadget_field_specs, gadget_ptype_specs from .io import IOHandlerGadgetBinary +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + vector_fields = dict(IOHandlerGadgetBinary._vector_fields) block_ids = { @@ -72,7 +77,7 @@ def fake_gadget_binary( header["HubbleParam"] = 1 write_block(fp, header, endian, fmt, "HEAD") - npart = dict(zip(ptype_spec, npart)) + npart = dict(zip(ptype_spec, npart, strict=True)) for fs in field_spec: # Parse field name and particle type if isinstance(fs, str): diff --git a/yt/frontends/halo_catalog/tests/test_outputs.py b/yt/frontends/halo_catalog/tests/test_outputs.py index 21ecd9dec0..3e56217ddb 100644 --- a/yt/frontends/halo_catalog/tests/test_outputs.py +++ b/yt/frontends/halo_catalog/tests/test_outputs.py @@ -1,3 +1,5 @@ +import sys + import numpy as np from numpy.testing import assert_array_equal, assert_equal @@ -8,6 +10,9 @@ from yt.units.yt_array import YTArray, YTQuantity from yt.utilities.answer_testing.framework import data_dir_load +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + def fake_halo_catalog(data): filename = "catalog.0.h5" @@ -38,7 +43,7 @@ def test_halo_catalog(self): units = ["g"] + ["cm"] * 3 data = { field: YTArray(rs.random_sample(n_halos), unit) - for field, unit in zip(fields, units) + for field, unit in zip(fields, units, strict=True) } fn = fake_halo_catalog(data) @@ -61,7 +66,7 @@ def test_halo_catalog_boundary_particles(self): units = ["g"] + ["cm"] * 3 data = { field: YTArray(rs.random_sample(n_halos), unit) - for field, unit in zip(fields, units) + for field, unit in zip(fields, units, strict=True) } data["particle_position_x"][0] = 1.0 diff --git a/yt/frontends/ramses/tests/test_hilbert.py b/yt/frontends/ramses/tests/test_hilbert.py index 55387c7e14..c4a04cc91e 100644 --- a/yt/frontends/ramses/tests/test_hilbert.py +++ b/yt/frontends/ramses/tests/test_hilbert.py @@ -1,3 +1,5 @@ +import sys + import numpy as np from numpy.testing import assert_equal @@ -5,6 +7,9 @@ from yt.frontends.ramses.hilbert import get_cpu_list_cuboid, hilbert3d from yt.testing import requires_file +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + def test_hilbert3d(): # 8 different cases, checked against RAMSES' own implementation @@ -20,7 +25,7 @@ def test_hilbert3d(): ] outputs = [0, 1, 7, 6, 3, 2, 4, 5] - for i, o in zip(inputs, outputs): + for i, o in zip(inputs, outputs, strict=True): assert_equal(hilbert3d(i, 3).item(), o) @@ -48,7 +53,7 @@ def test_get_cpu_list(): + [ds.hilbert_indices[ds.parameters["ncpu"]][1]], dtype="float64", ) - for i, o in zip(inputs, outputs): + for i, o in zip(inputs, outputs, strict=True): bbox = i ls = list(get_cpu_list_cuboid(ds, bbox, bound_keys=bound_keys)) assert len(ls) > 0 diff --git a/yt/frontends/stream/data_structures.py b/yt/frontends/stream/data_structures.py index 68b6f18ac4..3f077377a4 100644 --- a/yt/frontends/stream/data_structures.py +++ b/yt/frontends/stream/data_structures.py @@ -1,4 +1,5 @@ import os +import sys import time import uuid import weakref @@ -46,6 +47,9 @@ from .definitions import process_data, set_particle_types from .fields import StreamFieldInfo +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + class StreamGrid(AMRGridPatch): """ @@ -422,7 +426,7 @@ def _set_code_unit_attributes(self): "magnetic_unit", ) cgs_units = ("cm", "g", "s", "cm/s", "gauss") - for unit, attr, cgs_unit in zip(base_units, attrs, cgs_units): + for unit, attr, cgs_unit in zip(base_units, attrs, cgs_units, strict=True): if isinstance(unit, str): if unit == "code_magnetic": # If no magnetic unit was explicitly specified diff --git a/yt/frontends/stream/definitions.py b/yt/frontends/stream/definitions.py index 63657231ab..ae0df1f87e 100644 --- a/yt/frontends/stream/definitions.py +++ b/yt/frontends/stream/definitions.py @@ -1,3 +1,4 @@ +import sys from collections import defaultdict import numpy as np @@ -13,6 +14,9 @@ from .fields import StreamFieldInfo +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + def assign_particle_data(ds, pdata, bbox): """ @@ -135,7 +139,7 @@ def assign_particle_data(ds, pdata, bbox): else: grid_pdata = [pdata] - for pd, gi in zip(grid_pdata, sorted(ds.stream_handler.fields)): + for pd, gi in zip(grid_pdata, sorted(ds.stream_handler.fields), strict=True): ds.stream_handler.fields[gi].update(pd) ds.stream_handler.particle_types.update(set_particle_types(pd)) npart = ds.stream_handler.fields[gi].pop("number_of_particles", 0) diff --git a/yt/frontends/tipsy/data_structures.py b/yt/frontends/tipsy/data_structures.py index d27c7f38d5..7050bd4ac9 100644 --- a/yt/frontends/tipsy/data_structures.py +++ b/yt/frontends/tipsy/data_structures.py @@ -1,6 +1,7 @@ import glob import os import struct +import sys import numpy as np @@ -12,6 +13,9 @@ from .fields import TipsyFieldInfo +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + class TipsyFile(ParticleFile): def __init__(self, ds, io, filename, file_id, range=None): @@ -122,7 +126,9 @@ def _parse_parameter_file(self): hvals = { a: c for (a, b), c in zip( - self._header_spec, struct.unpack(hh, f.read(struct.calcsize(hh))) + self._header_spec, + struct.unpack(hh, f.read(struct.calcsize(hh))), + strict=True, ) } self.parameters.update(hvals) diff --git a/yt/frontends/tipsy/io.py b/yt/frontends/tipsy/io.py index d7296484a0..29809570aa 100644 --- a/yt/frontends/tipsy/io.py +++ b/yt/frontends/tipsy/io.py @@ -1,6 +1,7 @@ import glob import os import struct +import sys import numpy as np @@ -9,6 +10,9 @@ from yt.utilities.lib.particle_kdtree_tools import generate_smoothing_length from yt.utilities.logger import ytLogger as mylog +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + class IOHandlerTipsyBinary(IOHandlerSPH): _dataset_type = "tipsy" @@ -325,7 +329,7 @@ def _count_particles(self, data_file): if None not in (si, ei): np.clip(pcount - si, 0, ei - si, out=pcount) ptypes = ["Gas", "Stars", "DarkMatter"] - npart = dict(zip(ptypes, pcount)) + npart = dict(zip(ptypes, pcount, strict=True)) return npart @classmethod diff --git a/yt/frontends/ytdata/data_structures.py b/yt/frontends/ytdata/data_structures.py index 6e8e081915..46009b4a61 100644 --- a/yt/frontends/ytdata/data_structures.py +++ b/yt/frontends/ytdata/data_structures.py @@ -1,4 +1,5 @@ import os +import sys import weakref from collections import defaultdict from functools import cached_property @@ -33,6 +34,9 @@ from .fields import YTDataContainerFieldInfo, YTGridFieldInfo +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + _grid_data_containers = ["arbitrary_grid", "covering_grid", "smoothed_covering_grid"] _set_attrs = {"periodicity": "_periodicity"} @@ -146,7 +150,7 @@ def _set_code_unit_attributes(self): ) cgs_units = ("cm", "g", "s", "cm/s", "gauss") base_units = np.ones(len(attrs)) - for unit, attr, cgs_unit in zip(base_units, attrs, cgs_units): + for unit, attr, cgs_unit in zip(base_units, attrs, cgs_units, strict=True): if attr in self.parameters and isinstance( self.parameters[attr], YTQuantity ): diff --git a/yt/geometry/tests/test_grid_container.py b/yt/geometry/tests/test_grid_container.py index 795d299cf5..23a45725ea 100644 --- a/yt/geometry/tests/test_grid_container.py +++ b/yt/geometry/tests/test_grid_container.py @@ -1,10 +1,14 @@ import random +import sys import numpy as np from numpy.testing import assert_equal, assert_raises from yt.loaders import load_amr_grids +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + def setup_test_ds(): """Prepare setup specific environment""" @@ -102,7 +106,7 @@ def test_find_points(): grid_inds = np.zeros((num_points), dtype="int64") - for ind, ixx, iyy, izz in zip(range(num_points), randx, randy, randz): + for ind, ixx, iyy, izz in zip(range(num_points), randx, randy, randz, strict=True): pos = np.array([ixx, iyy, izz]) pt_level = -1 diff --git a/yt/loaders.py b/yt/loaders.py index 54d52e5b83..7fc2b6370c 100644 --- a/yt/loaders.py +++ b/yt/loaders.py @@ -44,6 +44,9 @@ parallel_root_only_then_broadcast, ) +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + if TYPE_CHECKING: from multiprocessing.connection import Connection @@ -798,7 +801,7 @@ def load_particles( le, re = data_source.get_bbox() le = le.to_value("code_length") re = re.to_value("code_length") - bbox = list(zip(le, re)) + bbox = list(zip(le, re, strict=True)) if bbox is None: bbox = np.array([[0.0, 1.0], [0.0, 1.0], [0.0, 1.0]], "float64") else: @@ -1382,10 +1385,10 @@ def load_unstructured_mesh( node_data = list(always_iterable(node_data, base_type=dict)) or [{}] * num_meshes data = [{} for i in range(num_meshes)] # type: ignore [var-annotated] - for elem_dict, data_dict in zip(elem_data, data): + for elem_dict, data_dict in zip(elem_data, data, strict=True): for field, values in elem_dict.items(): data_dict[field] = values - for node_dict, data_dict in zip(node_data, data): + for node_dict, data_dict in zip(node_data, data, strict=True): for field, values in node_dict.items(): data_dict[field] = values @@ -1918,7 +1921,7 @@ def _reader(grid, field_name): grid_data = [] psize = get_psize(np.array(shape), nchunks) left_edges, right_edges, shapes, _, _ = decompose_array(shape, psize, bbox) - for le, re, s in zip(left_edges, right_edges, shapes): + for le, re, s in zip(left_edges, right_edges, shapes, strict=True): data = {_: reader for _ in fields} data.update({"left_edge": le, "right_edge": re, "dimensions": s, "level": 0}) grid_data.append(data) diff --git a/yt/testing.py b/yt/testing.py index 2205439ea1..699d93f509 100644 --- a/yt/testing.py +++ b/yt/testing.py @@ -23,6 +23,9 @@ from yt.loaders import load from yt.units.yt_array import YTArray, YTQuantity +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + ANSWER_TEST_TAG = "answer_test" @@ -270,14 +273,14 @@ def fake_random_ds( else: offsets.append(0.0) data = {} - for field, offset, u in zip(fields, offsets, units): + for field, offset, u in zip(fields, offsets, units, strict=True): v = (prng.random_sample(ndims) - offset) * peak_value if field[0] == "all": v = v.ravel() data[field] = (v, u) if particles: if particle_fields is not None: - for field, unit in zip(particle_fields, particle_field_units): + for field, unit in zip(particle_fields, particle_field_units, strict=True): if field in ("particle_position", "particle_velocity"): data["io", field] = (prng.random_sample((int(particles), 3)), unit) else: @@ -347,7 +350,7 @@ def fake_amr_ds( "right_edge": right_edge, "dimensions": dims, } - for f, u in zip(fields, units): + for f, u in zip(fields, units, strict=True): gdata[f] = (prng.random_sample(dims), u) if particles: for i, f in enumerate(f"particle_position_{ax}" for ax in "xyz"): @@ -412,7 +415,7 @@ def fake_particle_ds( else: offsets.append(0.0) data = data if data else {} - for field, offset, u in zip(fields, offsets, units): + for field, offset, u in zip(fields, offsets, units, strict=True): if field in data: v = data[field] continue @@ -856,7 +859,7 @@ def expand_keywords(keywords, full=False): keys = sorted(keywords) list_of_kwarg_dicts = np.array( [ - dict(zip(keys, prod)) + dict(zip(keys, prod, strict=True)) for prod in it.product(*(keywords[key] for key in keys)) ] ) diff --git a/yt/utilities/answer_testing/framework.py b/yt/utilities/answer_testing/framework.py index 37358573dd..19569db5eb 100644 --- a/yt/utilities/answer_testing/framework.py +++ b/yt/utilities/answer_testing/framework.py @@ -47,6 +47,10 @@ profile_plotter as profile_plotter, ) +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + + mylog = logging.getLogger("nose.plugins.answer-testing") run_big_data = False @@ -769,9 +773,17 @@ def run(self): return result def compare(self, new_result, old_result): - for newp, oldp in zip(new_result["parents"], old_result["parents"]): + for newp, oldp in zip( + new_result["parents"], + old_result["parents"], + strict=True, + ): assert newp == oldp - for newc, oldc in zip(new_result["children"], old_result["children"]): + for newc, oldc in zip( + new_result["children"], + old_result["children"], + strict=True, + ): assert newc == oldc @@ -840,7 +852,10 @@ def compare_image_lists(new_result, old_result, decimals): line.strip() for line in results.split("\n") if line.endswith(".png") ] for fn, img, padded in zip( - tempfiles, (expected, actual), (expected_p, actual_p) + tempfiles, + (expected, actual), + (expected_p, actual_p), + strict=True, ): # padded images are convenient for comparison # but what we really want to store and upload diff --git a/yt/utilities/fortran_utils.py b/yt/utilities/fortran_utils.py index 32956767e8..b26092598f 100644 --- a/yt/utilities/fortran_utils.py +++ b/yt/utilities/fortran_utils.py @@ -1,9 +1,13 @@ import io import os import struct +import sys import numpy as np +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + def read_attrs(f, attrs, endian="="): r"""This function accepts a file pointer and reads from that file pointer @@ -77,7 +81,7 @@ def read_attrs(f, attrs, endian="="): len(a), len(v), ) - for k, val in zip(a, v): + for k, val in zip(a, v, strict=True): vv[k] = val else: vv[a] = v @@ -145,7 +149,7 @@ def read_cattrs(f, attrs, endian="="): len(v), ) - for k, val in zip(a, v): + for k, val in zip(a, v, strict=True): vv[k] = val else: vv[a] = v diff --git a/yt/utilities/lib/tests/test_fill_region.py b/yt/utilities/lib/tests/test_fill_region.py index 28e04c3e2c..967987c0f8 100644 --- a/yt/utilities/lib/tests/test_fill_region.py +++ b/yt/utilities/lib/tests/test_fill_region.py @@ -1,8 +1,13 @@ +import sys + import numpy as np from numpy.testing import assert_equal from yt.utilities.lib.misc_utilities import fill_region +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + NDIM = 32 @@ -38,5 +43,5 @@ def test_fill_region(): np.array([2, 2, 2], dtype="i8"), ) for r in range(level + 1): - for o, i in zip(output_fields, v): + for o, i in zip(output_fields, v, strict=True): assert_equal(o[r::rf, r::rf, r::rf], i) diff --git a/yt/utilities/sdf.py b/yt/utilities/sdf.py index 2ee44c1716..47083281f8 100644 --- a/yt/utilities/sdf.py +++ b/yt/utilities/sdf.py @@ -1,4 +1,5 @@ import os +import sys from collections import UserDict from io import StringIO @@ -6,6 +7,9 @@ from yt.funcs import mylog +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + def get_thingking_deps(): try: @@ -1175,7 +1179,7 @@ def get_key_data(self, key, fields): def iter_slice_data(self, slice_dim, slice_index, fields): mask, offsets, lengths = self.get_slice_chunks(slice_dim, slice_index) - for off, l in zip(offsets, lengths): + for off, l in zip(offsets, lengths, strict=True): data = {} chunk = slice(off, off + l) for field in fields: diff --git a/yt/utilities/tests/test_chemical_formulas.py b/yt/utilities/tests/test_chemical_formulas.py index 1d2fa54afd..7b5b933063 100644 --- a/yt/utilities/tests/test_chemical_formulas.py +++ b/yt/utilities/tests/test_chemical_formulas.py @@ -1,8 +1,13 @@ +import sys + from numpy.testing import assert_allclose, assert_equal from yt.utilities.chemical_formulas import ChemicalFormula, compute_mu from yt.utilities.periodic_table import periodic_table +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + _molecules = ( ("H2O_p1", (("H", 2), ("O", 1)), 1), ("H2O_m1", (("H", 2), ("O", 1)), -1), @@ -19,7 +24,7 @@ def test_formulas(): w = sum(n * periodic_table[e].weight for e, n in components) assert_equal(f.charge, charge) assert_equal(f.weight, w) - for (n, c1), (e, c2) in zip(components, f.elements): + for (n, c1), (e, c2) in zip(components, f.elements, strict=True): assert_equal(n, e.symbol) assert_equal(c1, c2) diff --git a/yt/utilities/tests/test_interpolators.py b/yt/utilities/tests/test_interpolators.py index e55796e806..aa2fff5466 100644 --- a/yt/utilities/tests/test_interpolators.py +++ b/yt/utilities/tests/test_interpolators.py @@ -1,3 +1,5 @@ +import sys + import numpy as np from numpy.testing import assert_array_almost_equal, assert_array_equal @@ -5,6 +7,9 @@ from yt.testing import fake_random_ds from yt.utilities.lib.interpolators import ghost_zone_interpolate +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + def test_linear_interpolator_1d(): random_data = np.random.random(64) @@ -26,7 +31,7 @@ def test_linear_interpolator_1d(): def test_linear_interpolator_2d(): random_data = np.random.random((64, 64)) # evenly spaced bins - fv = dict(zip("xyz", np.mgrid[0.0:1.0:64j, 0.0:1.0:64j])) + fv = dict(zip("xy", np.mgrid[0.0:1.0:64j, 0.0:1.0:64j], strict=True)) bfi = lin.BilinearFieldInterpolator(random_data, (0.0, 1.0, 0.0, 1.0), "xy", True) assert_array_equal(bfi(fv), random_data) @@ -45,7 +50,7 @@ def test_linear_interpolator_2d(): def test_linear_interpolator_3d(): random_data = np.random.random((64, 64, 64)) # evenly spaced bins - fv = dict(zip("xyz", np.mgrid[0.0:1.0:64j, 0.0:1.0:64j, 0.0:1.0:64j])) + fv = dict(zip("xyz", np.mgrid[0.0:1.0:64j, 0.0:1.0:64j, 0.0:1.0:64j], strict=True)) tfi = lin.TrilinearFieldInterpolator( random_data, (0.0, 1.0, 0.0, 1.0, 0.0, 1.0), "xyz", True ) @@ -70,7 +75,13 @@ def test_linear_interpolator_3d(): def test_linear_interpolator_4d(): random_data = np.random.random((64, 64, 64, 64)) # evenly spaced bins - fv = dict(zip("xyzw", np.mgrid[0.0:1.0:64j, 0.0:1.0:64j, 0.0:1.0:64j, 0.0:1.0:64j])) + fv = dict( + zip( + "xyzw", + np.mgrid[0.0:1.0:64j, 0.0:1.0:64j, 0.0:1.0:64j, 0.0:1.0:64j], + strict=True, + ) + ) tfi = lin.QuadrilinearFieldInterpolator( random_data, (0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0), "xyzw", True ) diff --git a/yt/visualization/_commons.py b/yt/visualization/_commons.py index 99d4b919a2..3deaa5bfc8 100644 --- a/yt/visualization/_commons.py +++ b/yt/visualization/_commons.py @@ -10,9 +10,7 @@ from yt.config import ytcfg -if sys.version_info >= (3, 10): - pass -else: +if sys.version_info < (3, 10): from yt._maintenance.backports import zip if TYPE_CHECKING: diff --git a/yt/visualization/fits_image.py b/yt/visualization/fits_image.py index 85a098fa30..c942033c88 100644 --- a/yt/visualization/fits_image.py +++ b/yt/visualization/fits_image.py @@ -25,6 +25,9 @@ ) from yt.visualization.volume_rendering.off_axis_projection import off_axis_projection +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + class UnitfulHDU: def __init__(self, hdu): @@ -250,7 +253,7 @@ def __init__( self.fields[i] = f"{ftype}_{fname}" for is_first, _is_last, (i, (name, field)) in mark_ends( - enumerate(zip(self.fields, fields)) + enumerate(zip(self.fields, fields, strict=True)) ): if name not in exclude_fields: this_img = img_data[field] @@ -345,10 +348,13 @@ def __init__( width = [width] * self.dimensionality if isinstance(width[0], YTQuantity): cdelt = [ - wh.to_value(wcs_unit) / n for wh, n in zip(width, self.shape) + wh.to_value(wcs_unit) / n + for wh, n in zip(width, self.shape, strict=True) ] else: - cdelt = [float(wh) / n for wh, n in zip(width, self.shape)] + cdelt = [ + float(wh) / n for wh, n in zip(width, self.shape, strict=True) + ] center = img_ctr[: self.dimensionality] w.wcs.crpix = 0.5 * (np.array(self.shape) + 1) w.wcs.crval = center @@ -390,7 +396,7 @@ def _set_units(self, ds, base_units): "magnetic_unit", ) cgs_units = ("cm", "g", "s", "cm/s", "gauss") - for unit, attr, cgs_unit in zip(base_units, attrs, cgs_units): + for unit, attr, cgs_unit in zip(base_units, attrs, cgs_units, strict=True): if unit is None: if ds is not None: u = getattr(ds, attr, None) diff --git a/yt/visualization/plot_modifications.py b/yt/visualization/plot_modifications.py index d16ad9a454..1b4ced9e31 100644 --- a/yt/visualization/plot_modifications.py +++ b/yt/visualization/plot_modifications.py @@ -51,6 +51,8 @@ else: from typing_extensions import TypeGuard + from yt._maintenance.backports import zip + if sys.version_info >= (3, 11): from typing import assert_never else: @@ -745,7 +747,7 @@ def __call__(self, plot): # do the transformation. Also check for the exact bounds of the transform # which can cause issues with projections. tform_bnds = plot._transform.x_limits + plot._transform.y_limits - if any(b.d == tb for b, tb in zip(bounds, tform_bnds)): + if any(b.d == tb for b, tb in zip(bounds, tform_bnds, strict=True)): # note: cartopy will also raise its own warning, but it is useful to add this # warning as well since the only way to avoid the exact bounds is to change the # extent of the plot. @@ -1167,7 +1169,7 @@ def __call__(self, plot): GRE = GRE[new_indices] block_ids = np.array(block_ids)[new_indices] - for px_off, py_off in zip(pxs.ravel(), pys.ravel()): + for px_off, py_off in zip(pxs.ravel(), pys.ravel(), strict=True): pxo = px_off * DW[px_index] pyo = py_off * DW[py_index] left_edge_x = np.array((GLE[:, px_index] + pxo - x0) * dx) + xx0 diff --git a/yt/visualization/plot_window.py b/yt/visualization/plot_window.py index 4534d7325c..c67a3c6125 100644 --- a/yt/visualization/plot_window.py +++ b/yt/visualization/plot_window.py @@ -55,9 +55,7 @@ invalidate_plot, ) -if sys.version_info >= (3, 10): - pass -else: +if sys.version_info < (3, 10): from yt._maintenance.backports import zip if sys.version_info >= (3, 11): diff --git a/yt/visualization/tests/test_offaxisprojection.py b/yt/visualization/tests/test_offaxisprojection.py index d317e31d8b..cea6c11fdc 100644 --- a/yt/visualization/tests/test_offaxisprojection.py +++ b/yt/visualization/tests/test_offaxisprojection.py @@ -1,6 +1,7 @@ import itertools as it import os import shutil +import sys import tempfile import unittest @@ -17,6 +18,9 @@ from yt.visualization.image_writer import write_projection from yt.visualization.volume_rendering.api import off_axis_projection +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + # TODO: replace this with pytest.mark.parametrize def expand_keywords(keywords, full=False): @@ -85,7 +89,7 @@ def expand_keywords(keywords, full=False): keys = sorted(keywords) list_of_kwarg_dicts = np.array( [ - dict(zip(keys, prod)) + dict(zip(keys, prod, strict=True)) for prod in it.product(*(keywords[key] for key in keys)) ] ) diff --git a/yt/visualization/tests/test_particle_plot.py b/yt/visualization/tests/test_particle_plot.py index 409902a921..78ecca9003 100644 --- a/yt/visualization/tests/test_particle_plot.py +++ b/yt/visualization/tests/test_particle_plot.py @@ -1,5 +1,6 @@ import os import shutil +import sys import tempfile import unittest from unittest import mock @@ -21,6 +22,9 @@ from yt.visualization.api import ParticlePhasePlot, ParticlePlot, ParticleProjectionPlot from yt.visualization.tests.test_plotwindow import ATTR_ARGS, WIDTH_SPECS +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + def setup_module(): """Test specific setup.""" @@ -431,7 +435,7 @@ def test_particle_plot_offaxis(self): test_ds = fake_particle_ds() Ls = [[1, 1, 1], [0, 1, -0.5]] Ns = [None, [1, 1, 1]] - for L, N in zip(Ls, Ns): + for L, N in zip(Ls, Ns, strict=True): for weight_field in WEIGHT_FIELDS: pplot_off = ParticleProjectionPlot( test_ds, @@ -456,9 +460,18 @@ def test_creation_with_width(self): ylim = [plot.ds.quan(el[0], el[1]) for el in ylim] pwidth = [plot.ds.quan(el[0], el[1]) for el in pwidth] - [assert_array_almost_equal(px, x, 14) for px, x in zip(plot.xlim, xlim)] - [assert_array_almost_equal(py, y, 14) for py, y in zip(plot.ylim, ylim)] - [assert_array_almost_equal(pw, w, 14) for pw, w in zip(plot.width, pwidth)] + [ + assert_array_almost_equal(px, x, 14) + for px, x in zip(plot.xlim, xlim, strict=True) + ] + [ + assert_array_almost_equal(py, y, 14) + for py, y in zip(plot.ylim, ylim, strict=True) + ] + [ + assert_array_almost_equal(pw, w, 14) + for pw, w in zip(plot.width, pwidth, strict=True) + ] def test_particle_plot_instance(): diff --git a/yt/visualization/tests/test_plotwindow.py b/yt/visualization/tests/test_plotwindow.py index a9830e1d8c..b40e81cd78 100644 --- a/yt/visualization/tests/test_plotwindow.py +++ b/yt/visualization/tests/test_plotwindow.py @@ -1,5 +1,6 @@ import os import shutil +import sys import tempfile import unittest from collections import OrderedDict @@ -43,6 +44,9 @@ plot_2d, ) +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + def setup_module(): """Test specific setup.""" @@ -386,9 +390,18 @@ def test_creation_with_width(self): ylim = [plot.ds.quan(el[0], el[1]) for el in ylim] pwidth = [plot.ds.quan(el[0], el[1]) for el in pwidth] - [assert_array_almost_equal(px, x, 14) for px, x in zip(plot.xlim, xlim)] - [assert_array_almost_equal(py, y, 14) for py, y in zip(plot.ylim, ylim)] - [assert_array_almost_equal(pw, w, 14) for pw, w in zip(plot.width, pwidth)] + [ + assert_array_almost_equal(px, x, 14) + for px, x in zip(plot.xlim, xlim, strict=True) + ] + [ + assert_array_almost_equal(py, y, 14) + for py, y in zip(plot.ylim, ylim, strict=True) + ] + [ + assert_array_almost_equal(pw, w, 14) + for pw, w in zip(plot.width, pwidth, strict=True) + ] assert aun == plot._axes_unit_names diff --git a/yt/visualization/volume_rendering/old_camera.py b/yt/visualization/volume_rendering/old_camera.py index cca5ff178a..6a7913e370 100644 --- a/yt/visualization/volume_rendering/old_camera.py +++ b/yt/visualization/volume_rendering/old_camera.py @@ -1,4 +1,5 @@ import builtins +import sys from copy import deepcopy import numpy as np @@ -34,6 +35,9 @@ from .transfer_functions import ProjectionTransferFunction +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + def get_corners(le, re): return np.array( @@ -410,7 +414,7 @@ def draw_coordinate_vectors(self, im, length=0.05, thickness=1): # we flipped it in snapshot to get the orientation correct, so # flip the lines - for vec, color in zip(coord_vectors, colors): + for vec, color in zip(coord_vectors, colors, strict=True): dx = int(np.dot(vec, self.orienter.unit_vectors[0])) dy = int(np.dot(vec, self.orienter.unit_vectors[1])) px = np.array([px0, px0 + dx], dtype="int64") @@ -1921,7 +1925,7 @@ def _setup_box_properties(self, width, center, unit_vectors): def snapshot(self, fn=None, clip_ratio=None, double_check=False, num_threads=0): my_storage = {} offx, offy = np.meshgrid(range(self.nimx), range(self.nimy)) - offxy = zip(offx.ravel(), offy.ravel()) + offxy = zip(offx.ravel(), offy.ravel(), strict=True) for sto, xy in parallel_objects( offxy, self.procs_per_wg, storage=my_storage, dynamic=True @@ -1957,7 +1961,7 @@ def reduce_images(self, im_dict): final_image = 0 if self.comm.rank == 0: offx, offy = np.meshgrid(range(self.nimx), range(self.nimy)) - offxy = zip(offx.ravel(), offy.ravel()) + offxy = zip(offx.ravel(), offy.ravel(), strict=True) nx, ny = self.resolution final_image = np.empty( (nx * self.nimx, ny * self.nimy, 4), dtype="float64", order="C" diff --git a/yt/visualization/volume_rendering/transfer_functions.py b/yt/visualization/volume_rendering/transfer_functions.py index aec7682157..5c0a4e58e8 100644 --- a/yt/visualization/volume_rendering/transfer_functions.py +++ b/yt/visualization/volume_rendering/transfer_functions.py @@ -1,9 +1,14 @@ +import sys + import numpy as np from more_itertools import always_iterable from yt.funcs import mylog from yt.utilities.physical_constants import clight, hcgs, kboltz +if sys.version_info < (3, 10): + from yt._maintenance.backports import zip + class TransferFunction: r"""A transfer function governs the transmission of emission and @@ -433,7 +438,7 @@ def add_gaussian(self, location, width, height): >>> tf = ColorTransferFunction((-10.0, -5.0)) >>> tf.add_gaussian(-9.0, 0.01, [1.0, 0.0, 0.0, 1.0]) """ - for tf, v in zip(self.funcs, height): + for tf, v in zip(self.funcs, height, strict=True): tf.add_gaussian(location, width, v) self.features.append( ( @@ -474,7 +479,7 @@ def add_step(self, start, stop, value): >>> tf = ColorTransferFunction((-10.0, -5.0)) >>> tf.add_step(-6.0, -5.0, [1.0, 1.0, 1.0, 1.0]) """ - for tf, v in zip(self.funcs, value): + for tf, v in zip(self.funcs, value, strict=True): tf.add_step(start, stop, v) self.features.append( ( @@ -890,7 +895,7 @@ def add_layers( alpha = np.ones(N, dtype="float64") elif alpha is None and not self.grey_opacity: alpha = np.logspace(-3, 0, N) - for v, a in zip(np.mgrid[mi : ma : N * 1j], alpha): + for v, a in zip(np.mgrid[mi : ma : N * 1j], alpha, strict=True): self.sample_colormap(v, w, a, colormap=colormap, col_bounds=col_bounds) def get_colormap_image(self, height, width): From 874cb3f8f0112aee1da7c7138ba18c365b710a40 Mon Sep 17 00:00:00 2001 From: Matthew Abruzzo Date: Tue, 16 Jul 2024 10:55:50 -0400 Subject: [PATCH 065/186] Add support for Enzo-E simulations with 0 ghost zones (#4932) --- yt/frontends/enzo_e/data_structures.py | 6 +++++- yt/frontends/enzo_e/io.py | 15 ++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/yt/frontends/enzo_e/data_structures.py b/yt/frontends/enzo_e/data_structures.py index a77c245d0e..16892f9c39 100644 --- a/yt/frontends/enzo_e/data_structures.py +++ b/yt/frontends/enzo_e/data_structures.py @@ -408,7 +408,11 @@ def _parse_parameter_file(self): self.parameters["current_cycle"] = ablock.attrs["cycle"][0] gsi = ablock.attrs["enzo_GridStartIndex"] gei = ablock.attrs["enzo_GridEndIndex"] - self.ghost_zones = gsi[0] + assert len(gsi) == len(gei) == 3 # sanity check + # Enzo-E technically allows each axis to have different ghost zone + # depths (this feature is not really used in practice) + self.ghost_zones = gsi + assert (self.ghost_zones[self.dimensionality :] == 0).all() # sanity check self.root_block_dimensions = root_blocks self.active_grid_dimensions = gei - gsi + 1 self.grid_dimensions = ablock.attrs["enzo_GridDimension"] diff --git a/yt/frontends/enzo_e/io.py b/yt/frontends/enzo_e/io.py index c2a8291b2b..1caf700b63 100644 --- a/yt/frontends/enzo_e/io.py +++ b/yt/frontends/enzo_e/io.py @@ -14,8 +14,17 @@ class EnzoEIOHandler(BaseIOHandler): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._base = self.ds.dimensionality * ( - slice(self.ds.ghost_zones, -self.ds.ghost_zones), + + # precompute the indexing specifying each field's active zone + # -> this assumes that each field in Enzo-E shares the same number of + # ghost-zones. Technically, Enzo-E allows each field to have a + # different number of ghost zones (but this feature isn't currently + # used & it currently doesn't record this information) + # -> our usage of a negative stop value ensures compatability with + # both cell-centered and face-centered fields + self._activezone_idx = tuple( + slice(num_zones, -num_zones) if num_zones > 0 else slice(None) + for num_zones in self.ds.ghost_zones[: self.ds.dimensionality] ) # Determine if particle masses are actually masses or densities. @@ -186,7 +195,7 @@ def _read_obj_field(self, obj, field, fid_data): dg.read(h5py.h5s.ALL, h5py.h5s.ALL, rdata) if close: fid.close() - data = rdata[self._base].T + data = rdata[self._activezone_idx].T if self.ds.dimensionality < 3: nshape = data.shape + (1,) * (3 - self.ds.dimensionality) data = np.reshape(data, nshape) From b93701eb558357c09b128ac61429ae304e67dc22 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Thu, 18 Jul 2024 15:48:12 -0500 Subject: [PATCH 066/186] Doesn't use in if block and early return which causes memory leakage. --- yt/utilities/lib/quad_tree.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt/utilities/lib/quad_tree.pyx b/yt/utilities/lib/quad_tree.pyx index c08f4e9ca9..36d4f236b3 100644 --- a/yt/utilities/lib/quad_tree.pyx +++ b/yt/utilities/lib/quad_tree.pyx @@ -413,8 +413,6 @@ cdef class QuadTree: np.float64_t wtoadd, np.int64_t level): cdef int i, j, n - cdef np.float64_t *vorig - vorig = malloc(sizeof(np.float64_t) * self.nvals) if node.children[0][0] == NULL: if self.merged == -1: for i in range(self.nvals): @@ -432,6 +430,8 @@ cdef class QuadTree: iy[curpos] = node.pos[1] return 1 cdef np.int64_t added = 0 + cdef np.float64_t *vorig + vorig = malloc(sizeof(np.float64_t) * self.nvals) if self.merged == 1: for i in range(self.nvals): vorig[i] = vtoadd[i] From 86cdfd816dfb6c27eceb19d2a06a6f9fd4b17171 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sat, 20 Jul 2024 15:30:49 +0200 Subject: [PATCH 067/186] BLD: switch to stable setuptools build backend --- pyproject.toml | 2 +- setup.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 50d2bd55f5..9e2dc73637 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ requires = [ "numpy>=2.0.0", "ewah-bool-utils>=1.2.0", ] -build-backend = "setuptools.build_meta:__legacy__" +build-backend = "setuptools.build_meta" [project] name = "yt" diff --git a/setup.py b/setup.py index 56a26ef1f0..f77776e337 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,16 @@ import glob import os +import sys from collections import defaultdict from distutils.ccompiler import get_default_compiler from importlib import resources as importlib_resources from setuptools import Distribution, setup +# ensure enclosing directory is in PYTHON_PATH to allow importing from setupext.py +if (script_dir := os.path.dirname(__file__)) not in sys.path: + sys.path.insert(0, script_dir) + from setupext import ( NUMPY_MACROS, check_CPP14_flags, From ec416ff0440ca34c9f5fd38295fe8c8770c6e438 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Fri, 5 Jul 2024 11:31:45 +0200 Subject: [PATCH 068/186] MNT: move cibuildwheel configuration to pyproject.toml --- .github/workflows/wheels.yaml | 10 ---------- pyproject.toml | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index 25d6700171..7bc049aaeb 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -35,16 +35,6 @@ jobs: uses: pypa/cibuildwheel@v2.19.1 with: output-dir: dist - env: - CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-*" - CIBW_SKIP: "*-musllinux_*" # numpy doesn't have wheels for musllinux so we can't build some quickly and without bloating - CIBW_ARCHS_LINUX: "x86_64" - CIBW_ARCHS_MACOS: auto - MACOSX_DEPLOYMENT_TARGET: "10.9" # as of CIBW 2.9, this is the default value, pin it so it can't be bumped silently - CIBW_ARCHS_WINDOWS: auto64 - CIBW_BUILD_VERBOSITY: 1 - CIBW_TEST_EXTRAS: test - CIBW_TEST_COMMAND: pytest -c {project}/pyproject.toml --rootdir . --color=yes --pyargs yt - uses: actions/upload-artifact@v4 with: diff --git a/pyproject.toml b/pyproject.toml index 50d2bd55f5..32bb80d623 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -492,3 +492,21 @@ warn_unused_ignores = true warn_unreachable = true show_error_context = true exclude = "(test_*|lodgeit)" + +[tool.cibuildwheel] +build = "cp39-* cp310-* cp311-* cp312-*" +build-verbosity = 1 +test-skip = "*-musllinux*" +test-extras = "test" +test-command = [ + "pytest -c {project}/pyproject.toml --rootdir . --color=yes --pyargs yt -ra", +] + +[tool.cibuildwheel.linux] +archs = "x86_64" + +[tool.cibuildwheel.macos] +archs = "auto" + +[tool.cibuildwheel.windows] +archs = "auto64" From 7244bffccf5df3a244833608c06acdbca929526f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sun, 21 Jul 2024 12:14:22 +0200 Subject: [PATCH 069/186] TST: skip a flaky test on Windows --- yt/tests/test_load_sample.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/yt/tests/test_load_sample.py b/yt/tests/test_load_sample.py index c5b5189bc4..f76ba4d7f8 100644 --- a/yt/tests/test_load_sample.py +++ b/yt/tests/test_load_sample.py @@ -109,6 +109,12 @@ def test_load_sample_small_dataset( ) +@pytest.mark.skipif( + sys.platform.startswith("win"), + # flakyness is probably due to Windows' infamous lack of time resolution + # overall, this test doesn't seem worth it. + reason="This test is flaky on Windows", +) @requires_module_pytest("pandas", "pooch") @pytest.mark.usefixtures("capturable_logger") def test_load_sample_timeout(tmp_data_dir, caplog): From f30fec5036a8c027693f728bf00622671c8773ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sun, 21 Jul 2024 13:06:03 +0200 Subject: [PATCH 070/186] CLN: cleanup unused internal conf parameter within_pytest --- conftest.py | 1 - yt/config.py | 1 - 2 files changed, 2 deletions(-) diff --git a/conftest.py b/conftest.py index f0786b909d..e845e4e016 100644 --- a/conftest.py +++ b/conftest.py @@ -85,7 +85,6 @@ def pytest_configure(config): Reads in the tests/tests.yaml file. This file contains a list of each answer test's answer file (including the changeset number). """ - ytcfg["yt", "internals", "within_pytest"] = True # Register custom marks for answer tests and big data config.addinivalue_line("markers", "answer_test: Run the answer tests.") config.addinivalue_line( diff --git a/yt/config.py b/yt/config.py index 758454a4f9..8ae7364145 100644 --- a/yt/config.py +++ b/yt/config.py @@ -45,7 +45,6 @@ "ray_tracing_engine": "yt", "internals": { "within_testing": False, - "within_pytest": False, "parallel": False, "strict_requires": False, "global_parallel_rank": 0, From 50faf56c86bffff61aabef4e2ac510d7eb1c3515 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sun, 7 Jul 2024 10:21:30 +0200 Subject: [PATCH 071/186] RFC: simplifications and cleanup --- yt/data_objects/tests/test_derived_quantities.py | 8 +++++--- yt/data_objects/tests/test_sph_data_objects.py | 9 +-------- yt/fields/particle_fields.py | 12 ++---------- yt/frontends/amrex/data_structures.py | 8 +------- yt/frontends/chombo/io.py | 2 +- yt/frontends/enzo_e/data_structures.py | 8 +------- yt/frontends/fits/fields.py | 16 ++++------------ yt/utilities/fortran_utils.py | 13 ++----------- yt/visualization/tests/test_particle_plot.py | 12 +++--------- yt/visualization/tests/test_plotwindow.py | 13 ++++--------- 10 files changed, 24 insertions(+), 77 deletions(-) diff --git a/yt/data_objects/tests/test_derived_quantities.py b/yt/data_objects/tests/test_derived_quantities.py index 60bc9223e4..93e343282b 100644 --- a/yt/data_objects/tests/test_derived_quantities.py +++ b/yt/data_objects/tests/test_derived_quantities.py @@ -191,9 +191,11 @@ def test_in_memory_sph_derived_quantities(): for fex, ans in zip(ex, [[0, 1], [0, 2], [0, 3]], strict=True): assert_equal(fex, ans) - for d, v, l in zip( - "xyz", [1, 2, 3], [[1, 0, 0], [0, 2, 0], [0, 0, 3]], strict=False - ): + for d, v, l in [ + ("x", 1, [1, 0, 0]), + ("y", 2, [0, 2, 0]), + ("z", 3, [0, 0, 3]), + ]: max_d, x, y, z = ad.quantities.max_location(("io", d)) assert_equal(max_d, v) assert_equal([x, y, z], l) diff --git a/yt/data_objects/tests/test_sph_data_objects.py b/yt/data_objects/tests/test_sph_data_objects.py index ebe350cc24..5a1acd4fbc 100644 --- a/yt/data_objects/tests/test_sph_data_objects.py +++ b/yt/data_objects/tests/test_sph_data_objects.py @@ -1,5 +1,3 @@ -import sys - import numpy as np from numpy.testing import assert_allclose, assert_almost_equal, assert_equal @@ -7,11 +5,6 @@ from yt.loaders import load from yt.testing import fake_sph_grid_ds, fake_sph_orientation_ds, requires_file -if sys.version_info >= (3, 10): - pass -else: - from yt._maintenance.backports import zip - def test_point(): ds = fake_sph_orientation_ds() @@ -99,7 +92,7 @@ def test_periodic_region(): for y in coords: for z in coords: center = np.array([x, y, z]) - for n, w in zip((8, 27), (1.0, 2.0), strict=True): + for n, w in [(8, 1.0), (27, 2.0)]: le = center - 0.5 * w re = center + 0.5 * w box = ds.box(le, re) diff --git a/yt/fields/particle_fields.py b/yt/fields/particle_fields.py index c38c2fdf75..e1ab8db2e8 100644 --- a/yt/fields/particle_fields.py +++ b/yt/fields/particle_fields.py @@ -1,5 +1,3 @@ -import sys - import numpy as np from yt.fields.derived_field import ValidateParameter, ValidateSpatial @@ -26,12 +24,6 @@ from .field_functions import get_radius from .vector_operations import create_magnitude_field -if sys.version_info >= (3, 10): - pass -else: - from yt._maintenance.backports import zip - - sph_whitelist_fields = ( "density", "temperature", @@ -192,7 +184,7 @@ def _deposit_field(field, data): return _deposit_field for ax in "xyz": - for method, name in zip(("cic", "sum"), ("cic", "nn"), strict=True): + for method, name in [("cic", "cic"), ("sum", "nn")]: function = _get_density_weighted_deposit_field( f"particle_velocity_{ax}", "code_velocity", method ) @@ -205,7 +197,7 @@ def _deposit_field(field, data): validators=[ValidateSpatial(0)], ) - for method, name in zip(("cic", "sum"), ("cic", "nn"), strict=True): + for method, name in [("cic", "cic"), ("sum", "nn")]: function = _get_density_weighted_deposit_field("age", "code_time", method) registry.add_field( ("deposit", ("%s_" + name + "_age") % (ptype)), diff --git a/yt/frontends/amrex/data_structures.py b/yt/frontends/amrex/data_structures.py index e2a201d5ec..ddf0052e81 100644 --- a/yt/frontends/amrex/data_structures.py +++ b/yt/frontends/amrex/data_structures.py @@ -1,7 +1,6 @@ import glob import os import re -import sys from collections import namedtuple from functools import cached_property from stat import ST_CTIME @@ -26,11 +25,6 @@ WarpXFieldInfo, ) -if sys.version_info >= (3, 10): - pass -else: - from yt._maintenance.backports import zip - # This is what we use to find scientific notation that might include d's # instead of e's. _scinot_finder = re.compile(r"[-+]?[0-9]*\.?[0-9]+([eEdD][-+]?[0-9]+)?") @@ -839,7 +833,7 @@ def _parse_header_file(self): # in a slightly hidden variable. self._max_level = int(header_file.readline()) - for side, init in zip(["left", "right"], [np.zeros, np.ones], strict=True): + for side, init in [("left", np.zeros), ("right", np.ones)]: domain_edge = init(3, dtype="float64") domain_edge[: self.dimensionality] = header_file.readline().split() setattr(self, f"domain_{side}_edge", domain_edge) diff --git a/yt/frontends/chombo/io.py b/yt/frontends/chombo/io.py index e6a95e4dae..288761df70 100644 --- a/yt/frontends/chombo/io.py +++ b/yt/frontends/chombo/io.py @@ -106,7 +106,7 @@ def _read_data(self, grid, field): data = lev[self._data_string][start:stop] data_no_ghost = data.reshape(shape, order="F") ghost_slice = tuple( - slice(g, d + g, None) for g, d in zip(self.ghost, dims, strict=True) + slice(g, g + d) for g, d in zip(self.ghost, dims, strict=True) ) ghost_slice = ghost_slice[0 : self.dim] return data_no_ghost[ghost_slice] diff --git a/yt/frontends/enzo_e/data_structures.py b/yt/frontends/enzo_e/data_structures.py index 5832e09e62..1e451f8690 100644 --- a/yt/frontends/enzo_e/data_structures.py +++ b/yt/frontends/enzo_e/data_structures.py @@ -1,5 +1,4 @@ import os -import sys from functools import cached_property import numpy as np @@ -24,11 +23,6 @@ from yt.utilities.logger import ytLogger as mylog from yt.utilities.on_demand_imports import _h5py as h5py, _libconf as libconf -if sys.version_info >= (3, 10): - pass -else: - from yt._maintenance.backports import zip - class EnzoEGrid(AMRGridPatch): """ @@ -481,7 +475,7 @@ def _set_code_unit_attributes(self): setdefaultattr(self, "velocity_unit", self.quan(k["uvel"], "cm/s")) else: p = self.parameters - for d, u in zip(("length", "time"), ("cm", "s"), strict=True): + for d, u in [("length", "cm"), ("time", "s")]: val = nested_dict_get(p, ("Units", d), default=1) setdefaultattr(self, f"{d}_unit", self.quan(val, u)) mass = nested_dict_get(p, ("Units", "mass")) diff --git a/yt/frontends/fits/fields.py b/yt/frontends/fits/fields.py index 171961845b..2b6260b731 100644 --- a/yt/frontends/fits/fields.py +++ b/yt/frontends/fits/fields.py @@ -1,13 +1,6 @@ -import sys - from yt._typing import KnownFieldsT from yt.fields.field_info_container import FieldInfoContainer -if sys.version_info >= (3, 10): - pass -else: - from yt._maintenance.backports import zip - class FITSFieldInfo(FieldInfoContainer): known_other_fields = () @@ -77,11 +70,10 @@ def _world_f(field, data): return _world_f - for (i, axis), name in zip( - enumerate([self.ds.lon_axis, self.ds.lat_axis]), - [self.ds.lon_name, self.ds.lat_name], - strict=True, - ): + for i, axis, name in [ + (0, self.ds.lon_axis, self.ds.lon_name), + (1, self.ds.lat_axis, self.ds.lat_name), + ]: unit = str(wcs_2d.wcs.cunit[i]) if unit.lower() == "deg": unit = "degree" diff --git a/yt/utilities/fortran_utils.py b/yt/utilities/fortran_utils.py index b26092598f..10ff51ea29 100644 --- a/yt/utilities/fortran_utils.py +++ b/yt/utilities/fortran_utils.py @@ -1,13 +1,9 @@ import io import os import struct -import sys import numpy as np -if sys.version_info < (3, 10): - from yt._maintenance.backports import zip - def read_attrs(f, attrs, endian="="): r"""This function accepts a file pointer and reads from that file pointer @@ -77,9 +73,7 @@ def read_attrs(f, attrs, endian="="): raise OSError( "An error occurred while reading a Fortran " "record. Record length is not equal to expected " - "length: %s %s", - len(a), - len(v), + f"length: {len(a)} {len(v)}" ) for k, val in zip(a, v, strict=True): vv[k] = val @@ -144,11 +138,8 @@ def read_cattrs(f, attrs, endian="="): raise OSError( "An error occurred while reading a Fortran " "record. Record length is not equal to expected " - "length: %s %s", - len(a), - len(v), + f"length: {len(a)} {len(v)}" ) - for k, val in zip(a, v, strict=True): vv[k] = val else: diff --git a/yt/visualization/tests/test_particle_plot.py b/yt/visualization/tests/test_particle_plot.py index 78ecca9003..3243dd20e8 100644 --- a/yt/visualization/tests/test_particle_plot.py +++ b/yt/visualization/tests/test_particle_plot.py @@ -460,18 +460,12 @@ def test_creation_with_width(self): ylim = [plot.ds.quan(el[0], el[1]) for el in ylim] pwidth = [plot.ds.quan(el[0], el[1]) for el in pwidth] - [ + for px, x in zip(plot.xlim, xlim, strict=True): assert_array_almost_equal(px, x, 14) - for px, x in zip(plot.xlim, xlim, strict=True) - ] - [ + for py, y in zip(plot.ylim, ylim, strict=True): assert_array_almost_equal(py, y, 14) - for py, y in zip(plot.ylim, ylim, strict=True) - ] - [ + for pw, w in zip(plot.width, pwidth, strict=True): assert_array_almost_equal(pw, w, 14) - for pw, w in zip(plot.width, pwidth, strict=True) - ] def test_particle_plot_instance(): diff --git a/yt/visualization/tests/test_plotwindow.py b/yt/visualization/tests/test_plotwindow.py index b40e81cd78..b229dae8e4 100644 --- a/yt/visualization/tests/test_plotwindow.py +++ b/yt/visualization/tests/test_plotwindow.py @@ -390,18 +390,13 @@ def test_creation_with_width(self): ylim = [plot.ds.quan(el[0], el[1]) for el in ylim] pwidth = [plot.ds.quan(el[0], el[1]) for el in pwidth] - [ + for px, x in zip(plot.xlim, xlim, strict=True): assert_array_almost_equal(px, x, 14) - for px, x in zip(plot.xlim, xlim, strict=True) - ] - [ + for py, y in zip(plot.ylim, ylim, strict=True): assert_array_almost_equal(py, y, 14) - for py, y in zip(plot.ylim, ylim, strict=True) - ] - [ + for pw, w in zip(plot.width, pwidth, strict=True): assert_array_almost_equal(pw, w, 14) - for pw, w in zip(plot.width, pwidth, strict=True) - ] + assert aun == plot._axes_unit_names From 64c2359908f9a68fd89788514dbba86c7e9e25c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sun, 7 Jul 2024 11:59:27 +0200 Subject: [PATCH 072/186] BUG: fix a bug weigth field was not properly validated in create_profile --- yt/data_objects/profiles.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yt/data_objects/profiles.py b/yt/data_objects/profiles.py index c5fc6ef6a5..5aac6b0b6d 100644 --- a/yt/data_objects/profiles.py +++ b/yt/data_objects/profiles.py @@ -1318,6 +1318,8 @@ def create_profile( data_source.ds.field_info[f].sampling_type == "local" for f in bin_fields + fields ] + if wf is not None: + is_local.append(wf.sampling_type == "local") is_local_or_pfield = [ pf or lf for (pf, lf) in zip(is_pfield, is_local, strict=True) ] From 8c489b4ad6bcfb993f472ed26ee9dc629999e6eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sun, 7 Jul 2024 12:15:39 +0200 Subject: [PATCH 073/186] BUG: fix probable bug in athena IOHandler --- yt/frontends/athena/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/frontends/athena/io.py b/yt/frontends/athena/io.py index d4e2de7ddd..f2893b7dde 100644 --- a/yt/frontends/athena/io.py +++ b/yt/frontends/athena/io.py @@ -24,7 +24,7 @@ class IOHandlerAthena(BaseIOHandler): def _field_dict(self, fhandle): keys = fhandle["field_types"].keys() - val = fhandle["field_types"].keys() + val = fhandle["field_types"].values() return dict(zip(keys, val, strict=True)) def _read_field_names(self, grid): From dd0dde450538d9cf1841dece3d40206f199828e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sun, 7 Jul 2024 10:03:34 +0200 Subject: [PATCH 074/186] STY: ignore rule UP038 --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 50d2bd55f5..d0ace3e0d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -302,6 +302,7 @@ ignore = [ "E501", # line too long "E741", # Do not use variables named 'I', 'O', or 'l' "B018", # Found useless expression. # disabled because ds.index is idiomatic + "UP038", # non-pep604-isinstance ] [tool.ruff.lint.per-file-ignores] From 7b61ef690d2a444ac7636d679cb14f57f525e29e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Tue, 16 Jul 2024 20:29:02 +0200 Subject: [PATCH 075/186] CLN: cleanup unused private method --- yt/frontends/athena/io.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/yt/frontends/athena/io.py b/yt/frontends/athena/io.py index f2893b7dde..61a65f0514 100644 --- a/yt/frontends/athena/io.py +++ b/yt/frontends/athena/io.py @@ -8,7 +8,7 @@ from .data_structures import chk23 if sys.version_info < (3, 10): - from yt._maintenance.backports import zip + pass float_size = {"float": np.dtype(">f4").itemsize, "double": np.dtype(">f8").itemsize} @@ -22,11 +22,6 @@ class IOHandlerAthena(BaseIOHandler): _data_string = "data:datatype=0" _read_table_offset = None - def _field_dict(self, fhandle): - keys = fhandle["field_types"].keys() - val = fhandle["field_types"].values() - return dict(zip(keys, val, strict=True)) - def _read_field_names(self, grid): pass From 91de9ba14ff17904a7386d9d5278060db9ff84a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Mon, 22 Jul 2024 21:50:43 +0200 Subject: [PATCH 076/186] CLN: cleanup following review --- yt/data_objects/region_expression.py | 6 +----- yt/frontends/athena/io.py | 6 ------ 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/yt/data_objects/region_expression.py b/yt/data_objects/region_expression.py index c6082ff93e..7187c5606e 100644 --- a/yt/data_objects/region_expression.py +++ b/yt/data_objects/region_expression.py @@ -1,4 +1,3 @@ -import sys import weakref from functools import cached_property @@ -13,9 +12,6 @@ from .data_containers import _get_ipython_key_completion -if sys.version_info < (3, 10): - from yt._maintenance.backports import zip - class RegionExpression: def __init__(self, ds): @@ -170,7 +166,7 @@ def _create_region(self, bounds_tuple): if d is not None: d = int(d) dims[ax] = d - center = [(cl + cr) / 2.0 for cl, cr in zip(left_edge, right_edge, strict=True)] + center = (left_edge + right_edge) / 2.0 if None not in dims: return self.ds.arbitrary_grid(left_edge, right_edge, dims) return self.ds.region(center, left_edge, right_edge) diff --git a/yt/frontends/athena/io.py b/yt/frontends/athena/io.py index 61a65f0514..b0045675ad 100644 --- a/yt/frontends/athena/io.py +++ b/yt/frontends/athena/io.py @@ -1,5 +1,3 @@ -import sys - import numpy as np from yt.funcs import mylog @@ -7,10 +5,6 @@ from .data_structures import chk23 -if sys.version_info < (3, 10): - pass - - float_size = {"float": np.dtype(">f4").itemsize, "double": np.dtype(">f8").itemsize} axis_list = ["_x", "_y", "_z"] From fc63e27438e6bda1bfc249cdb3be1e546cb6e20f Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Tue, 23 Jul 2024 15:43:25 -0500 Subject: [PATCH 077/186] added note to dev docs that anwer tests are currently run with nose, not pytest --- doc/source/developing/testing.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/source/developing/testing.rst b/doc/source/developing/testing.rst index c1e1ac1d6e..6c2424e0ad 100644 --- a/doc/source/developing/testing.rst +++ b/doc/source/developing/testing.rst @@ -152,6 +152,10 @@ More pytest options can be found by using the ``--help`` flag Answer Testing -------------- +.. note:: + This section documents answer tests run with ``pytest``. The plan is to + switch to using ``pytest`` for answer tests at some point in the future, + but currently (July 2024), answer tests are actually run using ``nose``. What Do Answer Tests Do ^^^^^^^^^^^^^^^^^^^^^^^ From e78993387c98c9c5c015db134ca68783265cd473 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Tue, 23 Jul 2024 15:56:47 -0500 Subject: [PATCH 078/186] removed trailing whitespace --- doc/source/developing/testing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/developing/testing.rst b/doc/source/developing/testing.rst index 6c2424e0ad..1eba0676c9 100644 --- a/doc/source/developing/testing.rst +++ b/doc/source/developing/testing.rst @@ -155,7 +155,7 @@ Answer Testing .. note:: This section documents answer tests run with ``pytest``. The plan is to switch to using ``pytest`` for answer tests at some point in the future, - but currently (July 2024), answer tests are actually run using ``nose``. + but currently (July 2024), answer tests are actually run using ``nose``. What Do Answer Tests Do ^^^^^^^^^^^^^^^^^^^^^^^ From 948eb27adcd192710f8a87299adbca8cf2344b83 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 24 Jul 2024 09:06:42 -0500 Subject: [PATCH 079/186] Update doc/source/developing/testing.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Robert --- doc/source/developing/testing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/developing/testing.rst b/doc/source/developing/testing.rst index 1eba0676c9..ea571a66d6 100644 --- a/doc/source/developing/testing.rst +++ b/doc/source/developing/testing.rst @@ -155,7 +155,7 @@ Answer Testing .. note:: This section documents answer tests run with ``pytest``. The plan is to switch to using ``pytest`` for answer tests at some point in the future, - but currently (July 2024), answer tests are actually run using ``nose``. + but currently (July 2024), answer tests are still implemented and run with ``nose``. What Do Answer Tests Do ^^^^^^^^^^^^^^^^^^^^^^^ From 0263eb28079a06ee6358b42217515581b2af6089 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 24 Jul 2024 12:00:42 -0500 Subject: [PATCH 080/186] add prefixes marking old and new test images --- yt/utilities/answer_testing/framework.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt/utilities/answer_testing/framework.py b/yt/utilities/answer_testing/framework.py index 37358573dd..ad2db01c4f 100644 --- a/yt/utilities/answer_testing/framework.py +++ b/yt/utilities/answer_testing/framework.py @@ -776,9 +776,9 @@ def compare(self, new_result, old_result): def dump_images(new_result, old_result, decimals=10): - tmpfd, old_image = tempfile.mkstemp(suffix=".png") + tmpfd, old_image = tempfile.mkstemp(prefix="baseline_", suffix=".png") os.close(tmpfd) - tmpfd, new_image = tempfile.mkstemp(suffix=".png") + tmpfd, new_image = tempfile.mkstemp(prefix="thisPR_", suffix=".png") os.close(tmpfd) image_writer.write_projection(new_result, new_image) image_writer.write_projection(old_result, old_image) From eec9a29b6f9ece4dca188bee37cb3dbe0c5a2153 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Tue, 13 Feb 2024 19:06:40 -0600 Subject: [PATCH 081/186] draft 1 off-axis projection and small-particle proj. fix (off-axis based heavily on PR 4712) --- yt/utilities/lib/pixelization_routines.pyx | 183 ++++++++++++------ yt/visualization/fixed_resolution.py | 2 + yt/visualization/plot_window.py | 2 + .../volume_rendering/off_axis_projection.py | 25 ++- 4 files changed, 151 insertions(+), 61 deletions(-) diff --git a/yt/utilities/lib/pixelization_routines.pyx b/yt/utilities/lib/pixelization_routines.pyx index 8cc37ab1d1..711a9fcb3c 100644 --- a/yt/utilities/lib/pixelization_routines.pyx +++ b/yt/utilities/lib/pixelization_routines.pyx @@ -1123,6 +1123,7 @@ def pixelize_sph_kernel_projection( np.uint8_t[:, :] mask, any_float[:] posx, any_float[:] posy, + any_float[:] posz, any_float[:] hsml, any_float[:] pmass, any_float[:] pdens, @@ -1134,17 +1135,19 @@ def pixelize_sph_kernel_projection( period=None): cdef np.intp_t xsize, ysize - cdef np.float64_t x_min, x_max, y_min, y_max, prefactor_j + cdef np.float64_t x_min, x_max, y_min, y_max, z_min, z_max, prefactor_j cdef np.int64_t xi, yi, x0, x1, y0, y1, xxi, yyi cdef np.float64_t q_ij2, posx_diff, posy_diff, ih_j2 - cdef np.float64_t x, y, dx, dy, idx, idy, h_j2, px, py - cdef np.float64_t period_x = 0, period_y = 0 - cdef int i, j, ii, jj + cdef np.float64_t x, y, dx, dy, idx, idy, h_j2, px, py, pz + cdef np.float64_t period_x = 0, period_y = 0, period_z = 0 + cdef int i, j, ii, jj, kk cdef np.float64_t[:] _weight_field cdef int * xiter cdef int * yiter + cdef int * ziter cdef np.float64_t * xiterv cdef np.float64_t * yiterv + cdef np.float64_t * ziterv if weight_field is not None: _weight_field = weight_field @@ -1152,6 +1155,7 @@ def pixelize_sph_kernel_projection( if period is not None: period_x = period[0] period_y = period[1] + period_z = period[2] # we find the x and y range over which we have pixels and we find how many # pixels we have in each dimension @@ -1160,6 +1164,8 @@ def pixelize_sph_kernel_projection( x_max = bounds[1] y_min = bounds[2] y_max = bounds[3] + z_min = bounds[4] + z_max = bounds[5] dx = (x_max - x_min) / xsize dy = (y_max - y_min) / ysize @@ -1190,10 +1196,12 @@ def pixelize_sph_kernel_projection( local_buff = malloc(sizeof(np.float64_t) * xsize * ysize) xiterv = malloc(sizeof(np.float64_t) * 2) yiterv = malloc(sizeof(np.float64_t) * 2) + ziterv = malloc(sizeof(np.float64_t) * 2) xiter = malloc(sizeof(int) * 2) yiter = malloc(sizeof(int) * 2) - xiter[0] = yiter[0] = 0 - xiterv[0] = yiterv[0] = 0.0 + ziter = malloc(sizeof(int) * 2) + xiter[0] = yiter[0] = ziter[0] = 0 + xiterv[0] = yiterv[0] = ziterv[0] = 0.0 for i in range(xsize * ysize): local_buff[i] = 0.0 @@ -1202,7 +1210,7 @@ def pixelize_sph_kernel_projection( with gil: PyErr_CheckSignals() - xiter[1] = yiter[1] = 999 + xiter[1] = yiter[1] = ziter[1] = 999 if check_period == 1: if posx[j] - hsml[j] < x_min: @@ -1217,60 +1225,77 @@ def pixelize_sph_kernel_projection( elif posy[j] + hsml[j] > y_max: yiter[1] = -1 yiterv[1] = -period_y + if posz[j] - hsml[j] < z_min: + ziter[1] = +1 + ziterv[1] = period_z + elif posz[j] + hsml[j] > z_max: + ziter[1] = -1 + ziterv[1] = -period_z # we set the smoothing length squared with lower limit of the pixel - h_j2 = fmax(hsml[j]*hsml[j], dx*dy) + # Nope! that causes weird grid resolution dependences and increases + # total values when resolution elements have hsml < grid spacing + h_j2 = hsml[j]*hsml[j] ih_j2 = 1.0/h_j2 prefactor_j = pmass[j] / pdens[j] / hsml[j]**2 * quantity_to_smooth[j] if weight_field is not None: prefactor_j *= _weight_field[j] + + # Discussion point: do we want the hsml margin on the z direction? + # it's consistent with Ray and Region selections, I think, + # but does tend to 'tack on' stuff compared to the nominal depth + for kk in range(2): + # discard if z is outside bounds + if ziter[kk] == 999: continue + pz = posz[j] + ziterv[kk] + if (pz + hsml[j] < z_min) or (pz - hsml[j] > z_max): continue + + for ii in range(2): + if xiter[ii] == 999: continue + px = posx[j] + xiterv[ii] + if (px + hsml[j] < x_min) or (px - hsml[j] > x_max): continue + for jj in range(2): + if yiter[jj] == 999: continue + py = posy[j] + yiterv[jj] + if (py + hsml[j] < y_min) or (py - hsml[j] > y_max): continue + + # here we find the pixels which this particle contributes to + x0 = ((px - hsml[j] - x_min)*idx) + x1 = ((px + hsml[j] - x_min)*idx) + x0 = iclip(x0-1, 0, xsize) + x1 = iclip(x1+1, 0, xsize) - for ii in range(2): - if xiter[ii] == 999: continue - px = posx[j] + xiterv[ii] - if (px + hsml[j] < x_min) or (px - hsml[j] > x_max): continue - for jj in range(2): - if yiter[jj] == 999: continue - py = posy[j] + yiterv[jj] - if (py + hsml[j] < y_min) or (py - hsml[j] > y_max): continue - - # here we find the pixels which this particle contributes to - x0 = ((px - hsml[j] - x_min)*idx) - x1 = ((px + hsml[j] - x_min)*idx) - x0 = iclip(x0-1, 0, xsize) - x1 = iclip(x1+1, 0, xsize) - - y0 = ((py - hsml[j] - y_min)*idy) - y1 = ((py + hsml[j] - y_min)*idy) - y0 = iclip(y0-1, 0, ysize) - y1 = iclip(y1+1, 0, ysize) + y0 = ((py - hsml[j] - y_min)*idy) + y1 = ((py + hsml[j] - y_min)*idy) + y0 = iclip(y0-1, 0, ysize) + y1 = iclip(y1+1, 0, ysize) - # found pixels we deposit on, loop through those pixels - for xi in range(x0, x1): - # we use the centre of the pixel to calculate contribution - x = (xi + 0.5) * dx + x_min + # found pixels we deposit on, loop through those pixels + for xi in range(x0, x1): + # we use the centre of the pixel to calculate contribution + x = (xi + 0.5) * dx + x_min - posx_diff = px - x - posx_diff = posx_diff * posx_diff + posx_diff = px - x + posx_diff = posx_diff * posx_diff - if posx_diff > h_j2: continue + if posx_diff > h_j2: continue - for yi in range(y0, y1): - y = (yi + 0.5) * dy + y_min + for yi in range(y0, y1): + y = (yi + 0.5) * dy + y_min - posy_diff = py - y - posy_diff = posy_diff * posy_diff - if posy_diff > h_j2: continue + posy_diff = py - y + posy_diff = posy_diff * posy_diff + if posy_diff > h_j2: continue - q_ij2 = (posx_diff + posy_diff) * ih_j2 - if q_ij2 >= 1: - continue + q_ij2 = (posx_diff + posy_diff) * ih_j2 + if q_ij2 >= 1: + continue - # see equation 32 of the SPLASH paper - # now we just use the kernel projection - local_buff[xi + yi*xsize] += prefactor_j * itab.interpolate(q_ij2) - mask[xi, yi] = 1 + # see equation 32 of the SPLASH paper + # now we just use the kernel projection + local_buff[xi + yi*xsize] += prefactor_j * itab.interpolate(q_ij2) + mask[xi, yi] = 1 with gil: for xxi in range(xsize): @@ -1864,7 +1889,10 @@ def rotate_particle_coord(np.float64_t[:] px, np.float64_t[:] py, np.float64_t[:] pz, center, + bounds, + periodic, width, + depth, normal_vector, north_vector): # We want to do two rotations, one to first rotate our coordinates to have @@ -1886,28 +1914,50 @@ def rotate_particle_coord(np.float64_t[:] px, cdef np.float64_t[:] px_rotated = np.empty(num_particles, dtype="float64") cdef np.float64_t[:] py_rotated = np.empty(num_particles, dtype="float64") + cdef np.float64_t[:] pz_rotated = np.empty(num_particles, dtype="float64") cdef np.float64_t[:] coordinate_matrix = np.empty(3, dtype="float64") cdef np.float64_t[:] rotated_coordinates cdef np.float64_t[:] rotated_center - rotated_center = rotation_matmul( - rotation_matrix, np.array([center[0], center[1], center[2]])) - + #rotated_center = rotation_matmul( + # rotation_matrix, np.array([center[0], center[1], center[2]])) + rotated_center = np.zeros(3, dtype=center.dtype) # set up the rotated bounds - cdef np.float64_t rot_bounds_x0 = rotated_center[0] - width[0] / 2 - cdef np.float64_t rot_bounds_x1 = rotated_center[0] + width[0] / 2 - cdef np.float64_t rot_bounds_y0 = rotated_center[1] - width[1] / 2 - cdef np.float64_t rot_bounds_y1 = rotated_center[1] + width[1] / 2 + cdef np.float64_t rot_bounds_x0 = rotated_center[0] - 0.5 * width[0] + cdef np.float64_t rot_bounds_x1 = rotated_center[0] + 0.5 * width[0] + cdef np.float64_t rot_bounds_y0 = rotated_center[1] - 0.5 * width[1] + cdef np.float64_t rot_bounds_y1 = rotated_center[1] + 0.5 * width[1] + cdef np.float64_t rot_bounds_z0 = rotated_center[2] - 0.5 * depth[2] + cdef np.float64_t rot_bounds_z1 = rotated_center[2] + 0.5 * depth[2] for i in range(num_particles): coordinate_matrix[0] = px[i] coordinate_matrix[1] = py[i] coordinate_matrix[2] = pz[i] + + # centering: + # make sure this also works for centers close to periodic edges + # added consequence: the center is placed at the origin + # (might as well keep it there in these temporary coordinates) + for ax in range(3): + if not periodic[ax]: continue + period = bounds[2 * ax + 1] - bounds[2 * ax] + coordinate_matrix[ax] -= center[ax] + # abs. difference between points in the volume is <= period + if coordinate_matrix[ax] < -0.5 * period: + coordinate_matrix[ax] += period + if coordinate_matrix[ax] > 0.5 * period: + coordinate_matrix[ax] -= period + rotated_coordinates = rotation_matmul( rotation_matrix, coordinate_matrix) px_rotated[i] = rotated_coordinates[0] py_rotated[i] = rotated_coordinates[1] + pz_rotated[i] = rotated_coordinates[2] - return px_rotated, py_rotated, rot_bounds_x0, rot_bounds_x1, rot_bounds_y0, rot_bounds_y1 + return (px_rotated, py_rotated, pz_rotated, + rot_bounds_x0, rot_bounds_x1, + rot_bounds_y0, rot_bounds_y1, + rot_bounds_z0, rot_bounds_z1) @cython.boundscheck(False) @@ -1921,31 +1971,46 @@ def off_axis_projection_SPH(np.float64_t[:] px, bounds, center, width, + periodic, np.float64_t[:] quantity_to_smooth, np.float64_t[:, :] projection_array, np.uint8_t[:, :] mask, normal_vector, north_vector, - weight_field=None): + weight_field=None, + depth=None): + # periodic: periodicity of the data set: # Do nothing in event of a 0 normal vector if np.allclose(normal_vector, 0.): return + if depth is None: + # set to volume diagonal -> won't exclude anything + depth = np.sqrt((bounds[1] - bounds[0])**2 + + (bounds[3] - bounds[2])**2, + + (bounds[5] - bounds[4])**2) - px_rotated, py_rotated, \ + px_rotated, py_rotated, pz_rotated, \ rot_bounds_x0, rot_bounds_x1, \ - rot_bounds_y0, rot_bounds_y1 = rotate_particle_coord(px, py, pz, - center, width, normal_vector, north_vector) + rot_bounds_y0, rot_bounds_y1, \ + rot_bounds_z0, rot_bounds_z1 = rotate_particle_coord(px, py, pz, + center, bounds, + periodic, + width, depth, + normal_vector, + north_vector) pixelize_sph_kernel_projection(projection_array, mask, px_rotated, py_rotated, + pz_rotated, smoothing_lengths, particle_masses, particle_densities, quantity_to_smooth, [rot_bounds_x0, rot_bounds_x1, - rot_bounds_y0, rot_bounds_y1], + rot_bounds_y0, rot_bounds_y1, + rot_bounds_z0, rot_bounds_z1], weight_field=weight_field, check_period=0) diff --git a/yt/visualization/fixed_resolution.py b/yt/visualization/fixed_resolution.py index d2da6a29e1..464e1bf22a 100644 --- a/yt/visualization/fixed_resolution.py +++ b/yt/visualization/fixed_resolution.py @@ -649,6 +649,7 @@ def _generate_image_and_mask(self, item) -> None: no_ghost=dd.no_ghost, interpolated=dd.interpolated, north_vector=dd.north_vector, + depth=dd.depth, method=dd.method, ) if self.data_source.moment == 2: @@ -679,6 +680,7 @@ def _sq_field(field, data, item: FieldKey): no_ghost=dd.no_ghost, interpolated=dd.interpolated, north_vector=dd.north_vector, + depth=dd.depth, method=dd.method, ) buff = compute_stddev_image(buff2, buff) diff --git a/yt/visualization/plot_window.py b/yt/visualization/plot_window.py index 4534d7325c..8927119170 100644 --- a/yt/visualization/plot_window.py +++ b/yt/visualization/plot_window.py @@ -2273,6 +2273,7 @@ def __init__( le=None, re=None, north_vector=None, + depth=None, method="integrate", data_source=None, *, @@ -2284,6 +2285,7 @@ def __init__( self.axis = None # always true for oblique data objects self.normal_vector = normal_vector self.width = width + self.depth = depth if data_source is None: self.dd = ds.all_data() else: diff --git a/yt/visualization/volume_rendering/off_axis_projection.py b/yt/visualization/volume_rendering/off_axis_projection.py index 392749633e..217518003c 100644 --- a/yt/visualization/volume_rendering/off_axis_projection.py +++ b/yt/visualization/volume_rendering/off_axis_projection.py @@ -30,6 +30,7 @@ def off_axis_projection( no_ghost=False, interpolated=False, north_vector=None, + depth=None, num_threads=1, method="integrate", ): @@ -80,6 +81,7 @@ def off_axis_projection( north_vector : optional, array_like, default None A vector that, if specified, restricts the orientation such that the north vector dotted into the image plane points "up". Useful for rotations + depth: float or num_threads: integer, optional, default 1 Use this many OpenMP threads during projection. method : string @@ -145,6 +147,11 @@ def off_axis_projection( center = data_source.ds.arr(center, "code_length") if not hasattr(width, "units"): width = data_source.ds.arr(width, "code_length") + if depth is not None: + if hasattr(depth, "units"): + depth = depth.to("code_length").d + #depth = data_source.ds.arr(depth, "code_length") + if hasattr(data_source.ds, "_sph_ptypes"): if method != "integrate": @@ -202,16 +209,23 @@ def off_axis_projection( # if weight is None: buf = np.zeros((resolution[0], resolution[1]), dtype="float64") mask = np.ones_like(buf, dtype="uint8") - + + # width from fixed_resolution.py is just the size of the domain x_min = center[0] - width[0] / 2 x_max = center[0] + width[0] / 2 y_min = center[1] - width[1] / 2 y_max = center[1] + width[1] / 2 z_min = center[2] - width[2] / 2 z_max = center[2] + width[2] / 2 + bounds = [x_min, x_max, y_min, y_max, z_min, z_max] + periodic = data_source.ds.periodicity + #le = data_source.ds.domain_left_edge.to("code_length").d + #re = data_source.ds.domain_right_edge.to("code_length").d + #x_min, y_min, z_min = le + #x_max, y_max, z_max = re + finfo = data_source.ds.field_info[item] ounits = finfo.output_units - bounds = [x_min, x_max, y_min, y_max, z_min, z_max] if weight is None: for chunk in data_source.chunks([], "io"): @@ -225,11 +239,13 @@ def off_axis_projection( bounds, center.to("code_length").d, width.to("code_length").d, + periodic, chunk[item].in_units(ounits), buf, mask, normal_vector, north, + depth=depth ) # Assure that the path length unit is in the default length units @@ -263,12 +279,14 @@ def off_axis_projection( bounds, center.to("code_length").d, width.to("code_length").d, + periodic, chunk[item].in_units(ounits), buf, mask, normal_vector, north, weight_field=chunk[weight].in_units(wounits), + depth=depth, ) for chunk in data_source.chunks([], "io"): @@ -282,11 +300,13 @@ def off_axis_projection( bounds, center.to("code_length").d, width.to("code_length").d, + periodic, chunk[weight].to(wounits), weight_buff, mask, normal_vector, north, + depth=depth, ) normalization_2d_utility(buf, weight_buff) @@ -300,6 +320,7 @@ def off_axis_projection( "north_vector": north_vector, "normal_vector": normal_vector, "width": width, + "depth": depth, "units": funits, "type": "SPH smoothed projection", } From 56c93d39e45110bb0d11e0566783e8cc2755badb Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Fri, 16 Feb 2024 21:16:37 -0600 Subject: [PATCH 082/186] debugging: axis-aligned SPH projection works again, off-axis runs, but fails output value tests --- .../coordinates/cartesian_coordinates.py | 58 +++++++++++++------ yt/utilities/lib/pixelization_routines.pyx | 11 ++-- yt/visualization/fixed_resolution.py | 1 - yt/visualization/plot_window.py | 11 ++-- .../volume_rendering/off_axis_projection.py | 38 +++++++----- 5 files changed, 76 insertions(+), 43 deletions(-) diff --git a/yt/geometry/coordinates/cartesian_coordinates.py b/yt/geometry/coordinates/cartesian_coordinates.py index 775de8c6a5..7e1fbb11cb 100644 --- a/yt/geometry/coordinates/cartesian_coordinates.py +++ b/yt/geometry/coordinates/cartesian_coordinates.py @@ -323,11 +323,20 @@ def _ortho_pixelize( # We should be using fcoords field = data_source._determine_fields(field)[0] finfo = data_source.ds.field_info[field] - period = self.period[:2].copy() # dummy here - period[0] = self.period[self.x_axis[dim]] - period[1] = self.period[self.y_axis[dim]] - if hasattr(period, "in_units"): - period = period.in_units("code_length").d + # some coordinate handlers use only projection-plane periods, + # others need all box periods. + period2 = self.period[:2].copy() # dummy here + period2[0] = self.period[self.x_axis[dim]] + period2[1] = self.period[self.y_axis[dim]] + period3 = self.period[:].copy() # dummy here + period3[0] = self.period[self.x_axis[dim]] + period3[1] = self.period[self.y_axis[dim]] + zax = list({0, 1, 2} - {self.x_axis[dim], self.y_axis[dim]}) + period3[2] = self.period[zax] + if hasattr(period2, "in_units"): + period2 = period2.in_units("code_length").d + if hasattr(period3, "in_units"): + period3 = period3.in_units("code_length").d buff = np.full((size[1], size[0]), np.nan, dtype="float64") particle_datasets = (ParticleDataset, StreamParticlesDataset) @@ -349,7 +358,7 @@ def _ortho_pixelize( coord, bounds, int(antialias), - period, + period2, int(periodic), return_mask=True, ) @@ -359,8 +368,13 @@ def _ortho_pixelize( ptype = data_source.ds._sph_ptypes[0] px_name = self.axis_name[self.x_axis[dim]] py_name = self.axis_name[self.y_axis[dim]] + # need z coordinates for depth, + # but name isn't saved in the handler -> use the 'other one' + pz_name = list((set(self.axis_order) - {px_name, py_name}))[0] + ounits = data_source.ds.field_info[field].output_units bnds = data_source.ds.arr(bounds, "code_length").tolist() + if isinstance(data_source, YTParticleProj): weight = data_source.weight_field moment = data_source.moment @@ -369,6 +383,7 @@ def _ortho_pixelize( ya = self.y_axis[dim] # If we're not periodic, we need to clip to the boundary edges # or we get errors about extending off the edge of the region. + # (depth/z range is handled by region setting) if not self.ds.periodicity[xa]: le[xa] = max(bounds[0], self.ds.domain_left_edge[xa]) re[xa] = min(bounds[1], self.ds.domain_right_edge[xa]) @@ -389,6 +404,11 @@ def _ortho_pixelize( data_source=data_source.data_source, ) proj_reg.set_field_parameter("axis", data_source.axis) + # need some z bounds for SPH projection + # -> use source bounds + zax = list({0, 1, 2} - set([xa, ya]))[0] + bnds3 = bnds + [le[zax], re[zax]] + buff = np.zeros(size, dtype="float64") mask_uint8 = np.zeros_like(buff, dtype="uint8") if weight is None: @@ -399,13 +419,14 @@ def _ortho_pixelize( mask_uint8, chunk[ptype, px_name].to("code_length"), chunk[ptype, py_name].to("code_length"), + chunk[ptype, pz_name].to("code_length"), chunk[ptype, "smoothing_length"].to("code_length"), chunk[ptype, "mass"].to("code_mass"), chunk[ptype, "density"].to("code_density"), chunk[field].in_units(ounits), - bnds, + bnds3, check_period=int(periodic), - period=period, + period=period3, ) # We use code length here, but to get the path length right # we need to multiply by the conversion factor between @@ -430,13 +451,14 @@ def _ortho_pixelize( mask_uint8, chunk[ptype, px_name].to("code_length"), chunk[ptype, py_name].to("code_length"), + chunk[ptype, pz_name].to("code_length"), chunk[ptype, "smoothing_length"].to("code_length"), chunk[ptype, "mass"].to("code_mass"), chunk[ptype, "density"].to("code_density"), chunk[field].in_units(ounits), - bnds, + bnds3, check_period=int(periodic), - period=period, + period=period3, weight_field=chunk[weight].in_units(wounits), ) mylog.info( @@ -452,13 +474,14 @@ def _ortho_pixelize( mask_uint8, chunk[ptype, px_name].to("code_length"), chunk[ptype, py_name].to("code_length"), + chunk[ptype, pz_name].to("code_length"), chunk[ptype, "smoothing_length"].to("code_length"), chunk[ptype, "mass"].to("code_mass"), chunk[ptype, "density"].to("code_density"), chunk[weight].in_units(wounits), - bnds, + bnds3, check_period=int(periodic), - period=period, + period=period3, ) normalization_2d_utility(buff, weight_buff) if moment == 2: @@ -471,13 +494,14 @@ def _ortho_pixelize( mask_uint8, chunk[ptype, px_name].to("code_length"), chunk[ptype, py_name].to("code_length"), + chunk[ptype, pz_name].to("code_length"), chunk[ptype, "smoothing_length"].to("code_length"), chunk[ptype, "mass"].to("code_mass"), chunk[ptype, "density"].to("code_density"), chunk[field].in_units(ounits) ** 2, - bnds, + bnds3, check_period=int(periodic), - period=period, + period=period3, weight_field=chunk[weight].in_units(wounits), ) normalization_2d_utility(buff2, weight_buff) @@ -505,7 +529,7 @@ def _ortho_pixelize( chunk[field].in_units(ounits), bnds, check_period=int(periodic), - period=period, + period=period2, ) if normalize: pixelize_sph_kernel_slice( @@ -519,7 +543,7 @@ def _ortho_pixelize( np.ones(chunk[ptype, "density"].shape[0]), bnds, check_period=int(periodic), - period=period, + period=period2, ) if normalize: @@ -608,7 +632,7 @@ def _ortho_pixelize( data_source[field], bounds, int(antialias), - period, + period2, int(periodic), return_mask=True, ) diff --git a/yt/utilities/lib/pixelization_routines.pyx b/yt/utilities/lib/pixelization_routines.pyx index 711a9fcb3c..52d964d9e2 100644 --- a/yt/utilities/lib/pixelization_routines.pyx +++ b/yt/utilities/lib/pixelization_routines.pyx @@ -1176,7 +1176,6 @@ def pixelize_sph_kernel_projection( if kernel_name not in kernel_tables: kernel_tables[kernel_name] = SPHKernelInterpolationTable(kernel_name) cdef SPHKernelInterpolationTable itab = kernel_tables[kernel_name] - with nogil, parallel(): # loop through every particle # NOTE: this loop can be quite time consuming. However it is easily @@ -1204,7 +1203,7 @@ def pixelize_sph_kernel_projection( xiterv[0] = yiterv[0] = ziterv[0] = 0.0 for i in range(xsize * ysize): local_buff[i] = 0.0 - + for j in prange(0, posx.shape[0], schedule="dynamic"): if j % 100000 == 0: with gil: @@ -1920,14 +1919,14 @@ def rotate_particle_coord(np.float64_t[:] px, cdef np.float64_t[:] rotated_center #rotated_center = rotation_matmul( # rotation_matrix, np.array([center[0], center[1], center[2]])) - rotated_center = np.zeros(3, dtype=center.dtype) + rotated_center = np.zeros((3,), dtype=center.dtype) # set up the rotated bounds cdef np.float64_t rot_bounds_x0 = rotated_center[0] - 0.5 * width[0] cdef np.float64_t rot_bounds_x1 = rotated_center[0] + 0.5 * width[0] cdef np.float64_t rot_bounds_y0 = rotated_center[1] - 0.5 * width[1] cdef np.float64_t rot_bounds_y1 = rotated_center[1] + 0.5 * width[1] - cdef np.float64_t rot_bounds_z0 = rotated_center[2] - 0.5 * depth[2] - cdef np.float64_t rot_bounds_z1 = rotated_center[2] + 0.5 * depth[2] + cdef np.float64_t rot_bounds_z0 = rotated_center[2] - 0.5 * depth + cdef np.float64_t rot_bounds_z1 = rotated_center[2] + 0.5 * depth for i in range(num_particles): coordinate_matrix[0] = px[i] @@ -1986,7 +1985,7 @@ def off_axis_projection_SPH(np.float64_t[:] px, if depth is None: # set to volume diagonal -> won't exclude anything depth = np.sqrt((bounds[1] - bounds[0])**2 - + (bounds[3] - bounds[2])**2, + + (bounds[3] - bounds[2])**2 + (bounds[5] - bounds[4])**2) px_rotated, py_rotated, pz_rotated, \ diff --git a/yt/visualization/fixed_resolution.py b/yt/visualization/fixed_resolution.py index 464e1bf22a..782e734ad3 100644 --- a/yt/visualization/fixed_resolution.py +++ b/yt/visualization/fixed_resolution.py @@ -634,7 +634,6 @@ def _generate_image_and_mask(self, item) -> None: ( self.bounds[1] - self.bounds[0], self.bounds[3] - self.bounds[2], - self.bounds[5] - self.bounds[4], ) ) buff = off_axis_projection( diff --git a/yt/visualization/plot_window.py b/yt/visualization/plot_window.py index 8927119170..b05a195fd9 100644 --- a/yt/visualization/plot_window.py +++ b/yt/visualization/plot_window.py @@ -86,14 +86,16 @@ def get_oblique_window_parameters(normal, center, width, ds, depth=None): if len(width) == 2: # Transforming to the cutting plane coordinate system - center = (center - ds.domain_left_edge) / ds.domain_width - 0.5 + # the original dimensionless center messes up off-axis + # SPH projections though + center = ((center - ds.domain_left_edge) / ds.domain_width - 0.5)\ + * ds.domain_width (normal, perp1, perp2) = ortho_find(normal) mat = np.transpose(np.column_stack((perp1, perp2, normal))) center = np.dot(mat, center) w = tuple(el.in_units("code_length") for el in width) bounds = tuple(((2 * (i % 2)) - 1) * w[i // 2] / 2 for i in range(len(w) * 2)) - return (bounds, center) @@ -2423,7 +2425,7 @@ def __init__( fields, center="center", width=None, - depth=(1, "1"), + depth=None, axes_unit=None, weight_field=None, max_level=None, @@ -2450,7 +2452,7 @@ def __init__( ) fields = list(iter_fields(fields))[:] oap_width = ds.arr( - (bounds[1] - bounds[0], bounds[3] - bounds[2], bounds[5] - bounds[4]) + (bounds[1] - bounds[0], bounds[3] - bounds[2]) ) OffAxisProj = OffAxisProjectionDummyDataSource( center_rot, @@ -2465,6 +2467,7 @@ def __init__( le=le, re=re, north_vector=north_vector, + depth=depth, method=method, data_source=data_source, moment=moment, diff --git a/yt/visualization/volume_rendering/off_axis_projection.py b/yt/visualization/volume_rendering/off_axis_projection.py index 217518003c..ac9ba69962 100644 --- a/yt/visualization/volume_rendering/off_axis_projection.py +++ b/yt/visualization/volume_rendering/off_axis_projection.py @@ -81,7 +81,10 @@ def off_axis_projection( north_vector : optional, array_like, default None A vector that, if specified, restricts the orientation such that the north vector dotted into the image plane points "up". Useful for rotations - depth: float or + depth: float, tuple[float, str], or unyt_array or size 1. + specify the depth of the projection region (size along the + line of sight). If no units are given (unyt_array or second + tuple element), code units are assumed. num_threads: integer, optional, default 1 Use this many OpenMP threads during projection. method : string @@ -148,8 +151,14 @@ def off_axis_projection( if not hasattr(width, "units"): width = data_source.ds.arr(width, "code_length") if depth is not None: + # handle units (intrinsic or as a tuple), + # then convert to code length + # float -> assumed to be in code units + if isinstance(depth, tuple): + depth = data_source.ds.arr(np.array([depth[0]]), depth[1]) if hasattr(depth, "units"): depth = depth.to("code_length").d + #depth = data_source.ds.arr(depth, "code_length") @@ -210,23 +219,22 @@ def off_axis_projection( buf = np.zeros((resolution[0], resolution[1]), dtype="float64") mask = np.ones_like(buf, dtype="uint8") - # width from fixed_resolution.py is just the size of the domain - x_min = center[0] - width[0] / 2 - x_max = center[0] + width[0] / 2 - y_min = center[1] - width[1] / 2 - y_max = center[1] + width[1] / 2 - z_min = center[2] - width[2] / 2 - z_max = center[2] + width[2] / 2 - bounds = [x_min, x_max, y_min, y_max, z_min, z_max] - periodic = data_source.ds.periodicity - #le = data_source.ds.domain_left_edge.to("code_length").d - #re = data_source.ds.domain_right_edge.to("code_length").d - #x_min, y_min, z_min = le - #x_max, y_max, z_max = re + ## width from fixed_resolution.py is just the size of the domain + #x_min = center[0] - width[0] / 2 + #x_max = center[0] + width[0] / 2 + #y_min = center[1] - width[1] / 2 + #y_max = center[1] + width[1] / 2 + #z_min = center[2] - width[2] / 2 + #z_max = center[2] + width[2] / 2 + periodic = data_source.ds.periodicity + le = data_source.ds.domain_left_edge.to("code_length").d + re = data_source.ds.domain_right_edge.to("code_length").d + x_min, y_min, z_min = le + x_max, y_max, z_max = re + bounds = [x_min, x_max, y_min, y_max, z_min, z_max] finfo = data_source.ds.field_info[item] ounits = finfo.output_units - if weight is None: for chunk in data_source.chunks([], "io"): off_axis_projection_SPH( From 44d6d4615706b7c857715dedb5c6fbbe1c883746 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Mon, 1 Jul 2024 17:55:50 -0500 Subject: [PATCH 083/186] partway debugging off-axis SPH-proj. --- yt/utilities/lib/pixelization_routines.pyx | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/yt/utilities/lib/pixelization_routines.pyx b/yt/utilities/lib/pixelization_routines.pyx index 52d964d9e2..f8d829e799 100644 --- a/yt/utilities/lib/pixelization_routines.pyx +++ b/yt/utilities/lib/pixelization_routines.pyx @@ -1927,7 +1927,6 @@ def rotate_particle_coord(np.float64_t[:] px, cdef np.float64_t rot_bounds_y1 = rotated_center[1] + 0.5 * width[1] cdef np.float64_t rot_bounds_z0 = rotated_center[2] - 0.5 * depth cdef np.float64_t rot_bounds_z1 = rotated_center[2] + 0.5 * depth - for i in range(num_particles): coordinate_matrix[0] = px[i] coordinate_matrix[1] = py[i] @@ -1938,9 +1937,10 @@ def rotate_particle_coord(np.float64_t[:] px, # added consequence: the center is placed at the origin # (might as well keep it there in these temporary coordinates) for ax in range(3): + # assumed center is zero even if non-periodic + coordinate_matrix[ax] -= center[ax] if not periodic[ax]: continue period = bounds[2 * ax + 1] - bounds[2 * ax] - coordinate_matrix[ax] -= center[ax] # abs. difference between points in the volume is <= period if coordinate_matrix[ax] < -0.5 * period: coordinate_matrix[ax] += period @@ -1961,6 +1961,7 @@ def rotate_particle_coord(np.float64_t[:] px, @cython.boundscheck(False) @cython.wraparound(False) + def off_axis_projection_SPH(np.float64_t[:] px, np.float64_t[:] py, np.float64_t[:] pz, @@ -1997,7 +1998,9 @@ def off_axis_projection_SPH(np.float64_t[:] px, width, depth, normal_vector, north_vector) - + # check_period=0: assumed to be a small region compared to the box + # size. The rotation already ensures that a center close to a + # periodic edge works out fine. pixelize_sph_kernel_projection(projection_array, mask, px_rotated, From 30d60fac633b32193276bcf881504a999a2321f3 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 3 Jul 2024 12:55:41 -0500 Subject: [PATCH 084/186] off-axis projections seem to work --- yt/utilities/lib/pixelization_routines.pyx | 13 ++++++++----- yt/visualization/plot_window.py | 22 ++++++++++++++++++++-- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/yt/utilities/lib/pixelization_routines.pyx b/yt/utilities/lib/pixelization_routines.pyx index f8d829e799..a0fce64c04 100644 --- a/yt/utilities/lib/pixelization_routines.pyx +++ b/yt/utilities/lib/pixelization_routines.pyx @@ -1984,11 +1984,10 @@ def off_axis_projection_SPH(np.float64_t[:] px, if np.allclose(normal_vector, 0.): return if depth is None: - # set to volume diagonal -> won't exclude anything - depth = np.sqrt((bounds[1] - bounds[0])**2 - + (bounds[3] - bounds[2])**2 - + (bounds[5] - bounds[4])**2) - + # set to volume diagonal + margin -> won't exclude anything + depth = 2. * np.sqrt((bounds[1] - bounds[0])**2 + + (bounds[3] - bounds[2])**2 + + (bounds[5] - bounds[4])**2) px_rotated, py_rotated, pz_rotated, \ rot_bounds_x0, rot_bounds_x1, \ rot_bounds_y0, rot_bounds_y1, \ @@ -2001,6 +2000,10 @@ def off_axis_projection_SPH(np.float64_t[:] px, # check_period=0: assumed to be a small region compared to the box # size. The rotation already ensures that a center close to a # periodic edge works out fine. + # since the simple single-coordinate modulo math periodicity + # does not apply to the *rotated* coordinates, the periodicity + # approach implemented for this along-axis projection method + # would fail here pixelize_sph_kernel_projection(projection_array, mask, px_rotated, diff --git a/yt/visualization/plot_window.py b/yt/visualization/plot_window.py index b05a195fd9..7068474f43 100644 --- a/yt/visualization/plot_window.py +++ b/yt/visualization/plot_window.py @@ -13,6 +13,8 @@ from yt._typing import AlphaT from yt.data_objects.image_array import ImageArray from yt.frontends.ytdata.data_structures import YTSpatialPlotDataset +from yt.frontends.sph.data_structures import ParticleDataset +from yt.frontends.stream.data_structures import StreamParticlesDataset from yt.funcs import ( fix_axis, fix_unitary, @@ -2446,16 +2448,32 @@ def __init__( f"off-axis slices are not supported for {ds.geometry!r} geometry\n" f"currently supported geometries: {self._supported_geometries!r}" ) - + # center_rot normalizes the center to (0,0), + # units match bounds + # for SPH data, we want to input the original center + # the cython backend handles centering to this point and + # rotation (bounds, center_rot) = get_oblique_window_parameters( normal, center, width, ds, depth=depth ) + # will probably fail if you try to project an SPH and non-SPH + # field in a single call + # checks for SPH fields copied from the + # _ortho_pixelize method in cartesian_coordinates.py + field = data_source._determine_fields(fields)[0] + finfo = data_source.ds.field_info[field] + particle_datasets = (ParticleDataset, StreamParticlesDataset) + is_sph_field = finfo.is_sph_field + if isinstance(data_source.ds, particle_datasets) and is_sph_field: + center_use = center + else: + center_use = center_rot fields = list(iter_fields(fields))[:] oap_width = ds.arr( (bounds[1] - bounds[0], bounds[3] - bounds[2]) ) OffAxisProj = OffAxisProjectionDummyDataSource( - center_rot, + center_use, ds, normal, oap_width, From 2ff58cd9e3e7de23d11cd85ead4f3da95b946256 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 3 Jul 2024 15:59:50 -0500 Subject: [PATCH 085/186] first attempt to fix pixelize_sph_kernel_slice --- yt/utilities/lib/pixelization_routines.pyx | 36 ++++++++++++++++------ 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/yt/utilities/lib/pixelization_routines.pyx b/yt/utilities/lib/pixelization_routines.pyx index a0fce64c04..2293dad03d 100644 --- a/yt/utilities/lib/pixelization_routines.pyx +++ b/yt/utilities/lib/pixelization_routines.pyx @@ -1489,11 +1489,13 @@ def interpolate_sph_grid_gather(np.float64_t[:, :, :] buff, def pixelize_sph_kernel_slice( np.float64_t[:, :] buff, np.uint8_t[:, :] mask, - np.float64_t[:] posx, np.float64_t[:] posy, + np.float64_t[:] posx, np.float64_t[:] posy, + np.float64_t[:] posz, np.float64_t[:] hsml, np.float64_t[:] pmass, np.float64_t[:] pdens, np.float64_t[:] quantity_to_smooth, - bounds, kernel_name="cubic", + bounds, np.float64_t slicez, + kernel_name="cubic", int check_period=1, period=None): @@ -1501,10 +1503,10 @@ def pixelize_sph_kernel_slice( cdef np.intp_t xsize, ysize cdef np.float64_t x_min, x_max, y_min, y_max, prefactor_j cdef np.int64_t xi, yi, x0, x1, y0, y1, xxi, yyi - cdef np.float64_t q_ij, posx_diff, posy_diff, ih_j - cdef np.float64_t x, y, dx, dy, idx, idy, h_j2, h_j, px, py + cdef np.float64_t q_ij, posx_diff, posy_diff, posz_diff, ih_j + cdef np.float64_t x, y, dx, dy, idx, idy, h_j2, h_j, px, py, pz cdef int i, j, ii, jj - cdef np.float64_t period_x = 0, period_y = 0 + cdef np.float64_t period_x = 0, period_y = 0, period_z = 0 cdef int * xiter cdef int * yiter cdef np.float64_t * xiterv @@ -1513,6 +1515,7 @@ def pixelize_sph_kernel_slice( if period is not None: period_x = period[0] period_y = period[1] + period_z = period[2] xsize, ysize = buff.shape[0], buff.shape[1] @@ -1560,11 +1563,24 @@ def pixelize_sph_kernel_slice( elif posy[j] + hsml[j] > y_max: yiter[1] = -1 yiterv[1] = -period_y - - h_j2 = fmax(hsml[j]*hsml[j], dx*dy) - h_j = math.sqrt(h_j2) + # z of particle might be < hsml from the slice plane + # but across a periodic boundary + if posz[j] - hsml[j] > slicez: + pz = posz[j] - period_z + elif posz[j] + hsml[j] < slicez: + pz = posz[j] + period_z + else: + pz = posz[j] + + h_j2 = hsml[j] * hsml[j] #fmax(hsml[j]*hsml[j], dx*dy) + h_j = hsml[j] #math.sqrt(h_j2) ih_j = 1.0/h_j + posz_diff = pz - slicez + posz_diff = posz_diff * posz_diff + if posz_diff > h_j2: + continue + prefactor_j = pmass[j] / pdens[j] / hsml[j]**3 prefactor_j *= quantity_to_smooth[j] @@ -1606,7 +1622,9 @@ def pixelize_sph_kernel_slice( continue # see equation 4 of the SPLASH paper - q_ij = math.sqrt(posx_diff + posy_diff) * ih_j + q_ij = math.sqrt(posx_diff + + posy_diff + + posz_diff) * ih_j if q_ij >= 1: continue From 13cd93ad953224bd10219dbf742f205d1472e2de Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 3 Jul 2024 16:46:48 -0500 Subject: [PATCH 086/186] attempt 1 to fix axis-aligned SPH slices --- yt/geometry/coordinates/cartesian_coordinates.py | 12 +++++++----- yt/utilities/lib/pixelization_routines.pyx | 3 ++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/yt/geometry/coordinates/cartesian_coordinates.py b/yt/geometry/coordinates/cartesian_coordinates.py index 7e1fbb11cb..ff9e8b3652 100644 --- a/yt/geometry/coordinates/cartesian_coordinates.py +++ b/yt/geometry/coordinates/cartesian_coordinates.py @@ -516,20 +516,21 @@ def _ortho_pixelize( mask_uint8 = np.zeros_like(buff, dtype="uint8") if normalize: buff_den = np.zeros(size, dtype="float64") - + for chunk in data_source.chunks([], "io"): pixelize_sph_kernel_slice( buff, mask_uint8, chunk[ptype, px_name].to("code_length"), chunk[ptype, py_name].to("code_length"), + chunk[ptype, pz_name].to("code_length"), chunk[ptype, "smoothing_length"].to("code_length"), chunk[ptype, "mass"].to("code_mass"), chunk[ptype, "density"].to("code_density"), chunk[field].in_units(ounits), - bnds, + bnds, data_source.coord, check_period=int(periodic), - period=period2, + period=period3, ) if normalize: pixelize_sph_kernel_slice( @@ -537,13 +538,14 @@ def _ortho_pixelize( mask_uint8, chunk[ptype, px_name].to("code_length"), chunk[ptype, py_name].to("code_length"), + chunk[ptype, pz_name].to("code_length"), chunk[ptype, "smoothing_length"].to("code_length"), chunk[ptype, "mass"].to("code_mass"), chunk[ptype, "density"].to("code_density"), np.ones(chunk[ptype, "density"].shape[0]), - bnds, + bnds, data_source.coord, check_period=int(periodic), - period=period2, + period=period3, ) if normalize: diff --git a/yt/utilities/lib/pixelization_routines.pyx b/yt/utilities/lib/pixelization_routines.pyx index 2293dad03d..3821c92f53 100644 --- a/yt/utilities/lib/pixelization_routines.pyx +++ b/yt/utilities/lib/pixelization_routines.pyx @@ -1498,7 +1498,8 @@ def pixelize_sph_kernel_slice( kernel_name="cubic", int check_period=1, period=None): - + # bounds are [x0, x1, y0, y1], slicez is the single coordinate + # of the slice along the normal direction. # similar method to pixelize_sph_kernel_projection cdef np.intp_t xsize, ysize cdef np.float64_t x_min, x_max, y_min, y_max, prefactor_j From 9bca940b848e20772d3e910615eea55e344296e4 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 3 Jul 2024 17:37:39 -0500 Subject: [PATCH 087/186] draft 1 backend SPH cutting plane --- yt/utilities/lib/pixelization_routines.pyx | 39 ++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/yt/utilities/lib/pixelization_routines.pyx b/yt/utilities/lib/pixelization_routines.pyx index 3821c92f53..94994ef870 100644 --- a/yt/utilities/lib/pixelization_routines.pyx +++ b/yt/utilities/lib/pixelization_routines.pyx @@ -2038,6 +2038,45 @@ def off_axis_projection_SPH(np.float64_t[:] px, weight_field=weight_field, check_period=0) +# like slice pixelization, but for off-axis planes +def pixelize_sph_kernel_cutting( + np.float64_t[:, :] buff, + np.uint8_t[:, :] mask, + np.float64_t[:] posx, np.float64_t[:] posy, + np.float64_t[:] posz, + np.float64_t[:] hsml, np.float64_t[:] pmass, + np.float64_t[:] pdens, + np.float64_t[:] quantity_to_smooth, + np.float64_t[3] center, np.float64_t[2] widthxy, + np.float64_t[3] normal_vector, np.float64_t[3] north_vector, + boxbounds, periodic, + kernel_name="cubic", + int check_period=1): + + if check_period == 0: + periodic = np.zeros(3, dtype=bool) + + posx_rot, posy_rot, posz_rot, \ + rot_bounds_x0, rot_bounds_x1, \ + rot_bounds_y0, rot_bounds_y1, \ + rot_bounds_z0, rot_bounds_z1 = rotate_particle_coord(posx, posy, posz, + center, boxbounds, + periodic, + widthxy, 0., + normal_vector, + north_vector) + bounds_rot = np.array([rot_bounds_x0, rot_bounds_x1, + rot_bounds_y0, rot_bounds_y1]) + slicez_rot = rot_bounds_z0 + pixelize_sph_kernel_slice(buff, mask, + posx_rot, posy_rot, posz_rot, + hsml, pmass, pdens, quantity_to_smooth, + bounds_rot, slicez_rot, + kernel_name=kernel_name, + check_period=0, + period=None) + + @cython.boundscheck(False) @cython.wraparound(False) From 22985f74ea786efb883c0b16fb5f7d0118cad2b3 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Thu, 4 Jul 2024 14:55:38 -0500 Subject: [PATCH 088/186] passing dataset kernel names to SPH slice/projection backend; work on slice backend call resolve merge conflict with before/after file marking branch: preserve sph_proj_backend changes in cartesian_coordinates.py --- .../coordinates/cartesian_coordinates.py | 125 +++++++++++++++--- yt/utilities/lib/pixelization_routines.pyx | 6 +- yt/visualization/plot_window.py | 4 +- .../volume_rendering/off_axis_projection.py | 11 +- 4 files changed, 121 insertions(+), 25 deletions(-) diff --git a/yt/geometry/coordinates/cartesian_coordinates.py b/yt/geometry/coordinates/cartesian_coordinates.py index ff9e8b3652..b52b6e36ec 100644 --- a/yt/geometry/coordinates/cartesian_coordinates.py +++ b/yt/geometry/coordinates/cartesian_coordinates.py @@ -12,6 +12,7 @@ pixelize_element_mesh, pixelize_element_mesh_line, pixelize_off_axis_cartesian, + pixelize_sph_kernel_cutting, pixelize_sph_kernel_projection, pixelize_sph_kernel_slice, ) @@ -374,6 +375,11 @@ def _ortho_pixelize( ounits = data_source.ds.field_info[field].output_units bnds = data_source.ds.arr(bounds, "code_length").tolist() + kernel_name = None + if hasattr(data_source.ds, "kernel_name"): + kernel_name = data_source.ds.kernel_name + if kernel_name is None: + kernel_name = "cubic" if isinstance(data_source, YTParticleProj): weight = data_source.weight_field @@ -427,6 +433,7 @@ def _ortho_pixelize( bnds3, check_period=int(periodic), period=period3, + kernel_name=kernel_name ) # We use code length here, but to get the path length right # we need to multiply by the conversion factor between @@ -460,6 +467,7 @@ def _ortho_pixelize( check_period=int(periodic), period=period3, weight_field=chunk[weight].in_units(wounits), + kernel_name=kernel_name ) mylog.info( "Making a fixed resolution buffer of (%s) %d by %d", @@ -482,6 +490,7 @@ def _ortho_pixelize( bnds3, check_period=int(periodic), period=period3, + kernel_name=kernel_name ) normalization_2d_utility(buff, weight_buff) if moment == 2: @@ -503,6 +512,7 @@ def _ortho_pixelize( check_period=int(periodic), period=period3, weight_field=chunk[weight].in_units(wounits), + kernel_name=kernel_name ) normalization_2d_utility(buff2, weight_buff) buff = compute_stddev_image(buff2, buff) @@ -531,6 +541,7 @@ def _ortho_pixelize( bnds, data_source.coord, check_period=int(periodic), period=period3, + kernel_name=kernel_name ) if normalize: pixelize_sph_kernel_slice( @@ -546,6 +557,7 @@ def _ortho_pixelize( bnds, data_source.coord, check_period=int(periodic), period=period3, + kernel_name=kernel_name ) if normalize: @@ -643,28 +655,99 @@ def _ortho_pixelize( def _oblique_pixelize(self, data_source, field, bounds, size, antialias): from yt.frontends.ytdata.data_structures import YTSpatialPlotDataset + from yt.data_objects.selection_objects.slices import YTSlice + from yt.frontends.sph.data_structures import ParticleDataset + from yt.frontends.stream.data_structures import StreamParticlesDataset - indices = np.argsort(data_source["pdx"])[::-1].astype("int64", copy=False) - buff = np.full((size[1], size[0]), np.nan, dtype="float64") - ftype = "index" - if isinstance(data_source.ds, YTSpatialPlotDataset): - ftype = "gas" - mask = pixelize_off_axis_cartesian( - buff, - data_source[ftype, "x"], - data_source[ftype, "y"], - data_source[ftype, "z"], - data_source["px"], - data_source["py"], - data_source["pdx"], - data_source["pdy"], - data_source["pdz"], - data_source.center, - data_source._inv_mat, - indices, - data_source[field], - bounds, - ) + # Determine what sort of data we're dealing with + # -> what backend to use + # copied from the _ortho_pixelize method + field = data_source._determine_fields(field)[0] + _finfo = data_source.ds.field_info[field] + is_sph_field = _finfo.is_sph_field + particle_datasets = (ParticleDataset, StreamParticlesDataset) + #finfo = self.ds._get_field_info(field) + + # SPH data + # only for slices: a function in off_axis_projection.py + # handles projections + if isinstance(data_source.ds, particle_datasets) and is_sph_field \ + and isinstance(data_source, YTSlice): + normalize = getattr(self.ds, "use_sph_normalization", True) + le, re = data_source.data_source.get_bbox() + boxbounds = np.array([le[0], re[0], le[1], re[1], le[2], re[2]]) + periodic = data_source.ds.coordinates.period.astype(bool).v + ptype = field[0] + if ptype == "gas": + ptype = data_source.ds._sph_ptypes[0] + axorder = data_source.ds.coordinates.axis_order + ounits = data_source.ds.field_info[field].output_units + + buff = np.zeros(size, dtype="float64") + mask_uint8 = np.zeros_like(buff, dtype="uint8") + if normalize: + buff_den = np.zeros(size, dtype="float64") + + for chunk in data_source.chunks([], "io"): + pixelize_sph_kernel_cutting( + buff, mask_uint8, + chunk[ptype, axorder[0]].to("code_length"), + chunk[ptype, axorder[1]].to("code_length"), + chunk[ptype, axorder[2]].to("code_length"), + chunk[ptype, "smoothing_length"].to("code_length"), + chunk[ptype, "mass"].to("code_mass"), + chunk[ptype, "density"].to("code_density"), + chunk[field].in_units(ounits), + center, widthxy, + normal_vector, north_vector, + boxbounds, periodic, + kernel_name="cubic", + check_period=1) + if normalize: + pixelize_sph_kernel_slice( + buff_den, + mask_uint8, + chunk[ptype, px_name].to("code_length"), + chunk[ptype, py_name].to("code_length"), + chunk[ptype, pz_name].to("code_length"), + chunk[ptype, "smoothing_length"].to("code_length"), + chunk[ptype, "mass"].to("code_mass"), + chunk[ptype, "density"].to("code_density"), + np.ones(chunk[ptype, "density"].shape[0]), + bnds, data_source.coord, + check_period=int(periodic), + period=period3, + ) + + if normalize: + normalization_2d_utility(buff, buff_den) + + mask = mask_uint8.astype("bool", copy=False) + + # whatever other data this code could handle before the + # SPH option was added + else: + indices = np.argsort(data_source["pdx"])[::-1].astype(np.int_) + buff = np.full((size[1], size[0]), np.nan, dtype="float64") + ftype = "index" + if isinstance(data_source.ds, YTSpatialPlotDataset): + ftype = "gas" + mask = pixelize_off_axis_cartesian( + buff, + data_source[ftype, "x"], + data_source[ftype, "y"], + data_source[ftype, "z"], + data_source["px"], + data_source["py"], + data_source["pdx"], + data_source["pdy"], + data_source["pdz"], + data_source.center, + data_source._inv_mat, + indices, + data_source[field], + bounds, + ) return buff, mask def convert_from_cartesian(self, coord): diff --git a/yt/utilities/lib/pixelization_routines.pyx b/yt/utilities/lib/pixelization_routines.pyx index 94994ef870..3ca1952720 100644 --- a/yt/utilities/lib/pixelization_routines.pyx +++ b/yt/utilities/lib/pixelization_routines.pyx @@ -1997,7 +1997,8 @@ def off_axis_projection_SPH(np.float64_t[:] px, normal_vector, north_vector, weight_field=None, - depth=None): + depth=None, + kernel_name="cubic"): # periodic: periodicity of the data set: # Do nothing in event of a 0 normal vector if np.allclose(normal_vector, 0.): @@ -2036,7 +2037,8 @@ def off_axis_projection_SPH(np.float64_t[:] px, rot_bounds_y0, rot_bounds_y1, rot_bounds_z0, rot_bounds_z1], weight_field=weight_field, - check_period=0) + check_period=0, + kernel_name=kernel_name) # like slice pixelization, but for off-axis planes def pixelize_sph_kernel_cutting( diff --git a/yt/visualization/plot_window.py b/yt/visualization/plot_window.py index 7068474f43..9c6143cf65 100644 --- a/yt/visualization/plot_window.py +++ b/yt/visualization/plot_window.py @@ -2222,7 +2222,9 @@ def __init__( f"off-axis slices are not supported for {ds.geometry!r} geometry\n" f"currently supported geometries: {self._supported_geometries!r}" ) - + # bounds are in cutting plane coordinates, centered on 0: + # [xmin, xmax, ymin, ymax]. Can derive width/height back + # from these. (bounds, center_rot) = get_oblique_window_parameters(normal, center, width, ds) if field_parameters is None: field_parameters = {} diff --git a/yt/visualization/volume_rendering/off_axis_projection.py b/yt/visualization/volume_rendering/off_axis_projection.py index ac9ba69962..6f741a1004 100644 --- a/yt/visualization/volume_rendering/off_axis_projection.py +++ b/yt/visualization/volume_rendering/off_axis_projection.py @@ -235,6 +235,12 @@ def off_axis_projection( bounds = [x_min, x_max, y_min, y_max, z_min, z_max] finfo = data_source.ds.field_info[item] ounits = finfo.output_units + kernel_name = None + if hasattr(data_source.ds, "kernel_name"): + kernel_name = data_source.ds.kernel_name + if kernel_name is None: + kernel_name = "cubic" + if weight is None: for chunk in data_source.chunks([], "io"): off_axis_projection_SPH( @@ -253,7 +259,8 @@ def off_axis_projection( mask, normal_vector, north, - depth=depth + depth=depth, + kernel_name=kernel_name, ) # Assure that the path length unit is in the default length units @@ -295,6 +302,7 @@ def off_axis_projection( north, weight_field=chunk[weight].in_units(wounits), depth=depth, + kernel_name=kernel_name, ) for chunk in data_source.chunks([], "io"): @@ -315,6 +323,7 @@ def off_axis_projection( normal_vector, north, depth=depth, + kernel_name=kernel_name, ) normalization_2d_utility(buf, weight_buff) From 6a874fe7f21290d828c85ec841772f90d284963d Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Thu, 4 Jul 2024 15:30:30 -0500 Subject: [PATCH 089/186] draft 1 fixed slices, new off-axis slices SPH --- .../coordinates/cartesian_coordinates.py | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/yt/geometry/coordinates/cartesian_coordinates.py b/yt/geometry/coordinates/cartesian_coordinates.py index b52b6e36ec..e9d2199ee7 100644 --- a/yt/geometry/coordinates/cartesian_coordinates.py +++ b/yt/geometry/coordinates/cartesian_coordinates.py @@ -682,6 +682,17 @@ def _oblique_pixelize(self, data_source, field, bounds, size, antialias): ptype = data_source.ds._sph_ptypes[0] axorder = data_source.ds.coordinates.axis_order ounits = data_source.ds.field_info[field].output_units + widthxy = np.array(((bounds[1] - bounds[0]).to("code_length"), + (bounds[3] - bounds[2]).to("code_length"))) + kernel_name = None + if hasattr(data_source.ds, "kernel_name"): + kernel_name = data_source.ds.kernel_name + if kernel_name is None: + kernel_name = "cubic" + # data_source should be a YTCuttingPlane object + normal_vector = data_source.normal + north_vector = data_source._y_vec + center = data_source.center.to("code_length") buff = np.zeros(size, dtype="float64") mask_uint8 = np.zeros_like(buff, dtype="uint8") @@ -690,7 +701,8 @@ def _oblique_pixelize(self, data_source, field, bounds, size, antialias): for chunk in data_source.chunks([], "io"): pixelize_sph_kernel_cutting( - buff, mask_uint8, + buff, + mask_uint8, chunk[ptype, axorder[0]].to("code_length"), chunk[ptype, axorder[1]].to("code_length"), chunk[ptype, axorder[2]].to("code_length"), @@ -701,22 +713,25 @@ def _oblique_pixelize(self, data_source, field, bounds, size, antialias): center, widthxy, normal_vector, north_vector, boxbounds, periodic, - kernel_name="cubic", - check_period=1) + kernel_name=kernel_name, + check_period=1, + ) if normalize: - pixelize_sph_kernel_slice( + pixelize_sph_kernel_cutting( buff_den, mask_uint8, - chunk[ptype, px_name].to("code_length"), - chunk[ptype, py_name].to("code_length"), - chunk[ptype, pz_name].to("code_length"), + chunk[ptype, axorder[0]].to("code_length"), + chunk[ptype, axorder[1]].to("code_length"), + chunk[ptype, axorder[2]].to("code_length"), chunk[ptype, "smoothing_length"].to("code_length"), chunk[ptype, "mass"].to("code_mass"), chunk[ptype, "density"].to("code_density"), np.ones(chunk[ptype, "density"].shape[0]), - bnds, data_source.coord, - check_period=int(periodic), - period=period3, + center, widthxy, + normal_vector, north_vector, + boxbounds, periodic, + kernel_name=kernel_name, + check_period=1 ) if normalize: From 8b3ca657d8326e4dc4576f6bb068f7a42f29fc3e Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Fri, 5 Jul 2024 16:41:11 -0500 Subject: [PATCH 090/186] further bugfix attempts --- yt/utilities/lib/pixelization_routines.pyx | 4 ++-- yt/visualization/plot_window.py | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/yt/utilities/lib/pixelization_routines.pyx b/yt/utilities/lib/pixelization_routines.pyx index 3ca1952720..7afc6fc560 100644 --- a/yt/utilities/lib/pixelization_routines.pyx +++ b/yt/utilities/lib/pixelization_routines.pyx @@ -2049,8 +2049,8 @@ def pixelize_sph_kernel_cutting( np.float64_t[:] hsml, np.float64_t[:] pmass, np.float64_t[:] pdens, np.float64_t[:] quantity_to_smooth, - np.float64_t[3] center, np.float64_t[2] widthxy, - np.float64_t[3] normal_vector, np.float64_t[3] north_vector, + center, widthxy, + normal_vector, north_vector, boxbounds, periodic, kernel_name="cubic", int check_period=1): diff --git a/yt/visualization/plot_window.py b/yt/visualization/plot_window.py index 9c6143cf65..4798154581 100644 --- a/yt/visualization/plot_window.py +++ b/yt/visualization/plot_window.py @@ -1595,6 +1595,7 @@ class SlicePlot(NormalPlot): def __new__( # type: ignore cls, ds, normal, fields, *args, **kwargs ) -> Union["AxisAlignedSlicePlot", "OffAxisSlicePlot"]: + print('SlicePlot call cls: ', cls) if cls is SlicePlot: normal = cls.sanitize_normal_vector(ds, normal) if isinstance(normal, str): @@ -1602,6 +1603,8 @@ def __new__( # type: ignore else: cls = OffAxisSlicePlot self = object.__new__(cls) + print('SlicePlot result cls: ', cls) + print('Sliceplot return self: ', self) return self # type: ignore [return-value] @@ -1822,9 +1825,12 @@ def __init__( normal = self.sanitize_normal_vector(ds, normal) # this will handle time series data and controllers axis = fix_axis(normal, ds) + #print('center at SlicePlot init: ', center) + #print('current domain left edge: ', ds.domain_left_edge) (bounds, center, display_center) = get_window_parameters( axis, center, width, ds ) + # print('center after get_window_parameters: ', center) if field_parameters is None: field_parameters = {} From 31aae231cf13374db018892f01dc9297ae1f4f79 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Fri, 5 Jul 2024 18:26:01 -0500 Subject: [PATCH 091/186] debugged SPH slices to run; outcome clearly wrong off-axis --- .../coordinates/cartesian_coordinates.py | 26 +++++++++++-------- yt/visualization/plot_window.py | 2 +- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/yt/geometry/coordinates/cartesian_coordinates.py b/yt/geometry/coordinates/cartesian_coordinates.py index e9d2199ee7..e709cfd55b 100644 --- a/yt/geometry/coordinates/cartesian_coordinates.py +++ b/yt/geometry/coordinates/cartesian_coordinates.py @@ -655,7 +655,7 @@ def _ortho_pixelize( def _oblique_pixelize(self, data_source, field, bounds, size, antialias): from yt.frontends.ytdata.data_structures import YTSpatialPlotDataset - from yt.data_objects.selection_objects.slices import YTSlice + from yt.data_objects.selection_objects.slices import YTCuttingPlane from yt.frontends.sph.data_structures import ParticleDataset from yt.frontends.stream.data_structures import StreamParticlesDataset @@ -672,9 +672,10 @@ def _oblique_pixelize(self, data_source, field, bounds, size, antialias): # only for slices: a function in off_axis_projection.py # handles projections if isinstance(data_source.ds, particle_datasets) and is_sph_field \ - and isinstance(data_source, YTSlice): + and isinstance(data_source, YTCuttingPlane): normalize = getattr(self.ds, "use_sph_normalization", True) - le, re = data_source.data_source.get_bbox() + le = data_source.ds.domain_left_edge.to("code_length") + re = data_source.ds.domain_right_edge.to("code_length") boxbounds = np.array([le[0], re[0], le[1], re[1], le[2], re[2]]) periodic = data_source.ds.coordinates.period.astype(bool).v ptype = field[0] @@ -682,16 +683,19 @@ def _oblique_pixelize(self, data_source, field, bounds, size, antialias): ptype = data_source.ds._sph_ptypes[0] axorder = data_source.ds.coordinates.axis_order ounits = data_source.ds.field_info[field].output_units - widthxy = np.array(((bounds[1] - bounds[0]).to("code_length"), - (bounds[3] - bounds[2]).to("code_length"))) + # input bounds are in code length units already + widthxy = np.array((bounds[1] - bounds[0], + bounds[3] - bounds[2])) kernel_name = None if hasattr(data_source.ds, "kernel_name"): kernel_name = data_source.ds.kernel_name if kernel_name is None: kernel_name = "cubic" # data_source should be a YTCuttingPlane object - normal_vector = data_source.normal - north_vector = data_source._y_vec + # dimensionless unyt normal/north + # -> numpy array cython can deal with + normal_vector = data_source.normal.v + north_vector = data_source._y_vec.v center = data_source.center.to("code_length") buff = np.zeros(size, dtype="float64") @@ -703,10 +707,10 @@ def _oblique_pixelize(self, data_source, field, bounds, size, antialias): pixelize_sph_kernel_cutting( buff, mask_uint8, - chunk[ptype, axorder[0]].to("code_length"), - chunk[ptype, axorder[1]].to("code_length"), - chunk[ptype, axorder[2]].to("code_length"), - chunk[ptype, "smoothing_length"].to("code_length"), + chunk[ptype, axorder[0]].to("code_length").v, + chunk[ptype, axorder[1]].to("code_length").v, + chunk[ptype, axorder[2]].to("code_length").v, + chunk[ptype, "smoothing_length"].to("code_length").v, chunk[ptype, "mass"].to("code_mass"), chunk[ptype, "density"].to("code_density"), chunk[field].in_units(ounits), diff --git a/yt/visualization/plot_window.py b/yt/visualization/plot_window.py index 4798154581..b2a49538cf 100644 --- a/yt/visualization/plot_window.py +++ b/yt/visualization/plot_window.py @@ -2230,7 +2230,7 @@ def __init__( ) # bounds are in cutting plane coordinates, centered on 0: # [xmin, xmax, ymin, ymax]. Can derive width/height back - # from these. + # from these. unit is code_length (bounds, center_rot) = get_oblique_window_parameters(normal, center, width, ds) if field_parameters is None: field_parameters = {} From 690d64f847308bea0c50d14afd73db00e14e37ca Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Fri, 5 Jul 2024 19:45:03 -0500 Subject: [PATCH 092/186] partial debug along-axis SPH slices --- yt/geometry/coordinates/cartesian_coordinates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/geometry/coordinates/cartesian_coordinates.py b/yt/geometry/coordinates/cartesian_coordinates.py index e709cfd55b..8c060e7f42 100644 --- a/yt/geometry/coordinates/cartesian_coordinates.py +++ b/yt/geometry/coordinates/cartesian_coordinates.py @@ -412,7 +412,7 @@ def _ortho_pixelize( proj_reg.set_field_parameter("axis", data_source.axis) # need some z bounds for SPH projection # -> use source bounds - zax = list({0, 1, 2} - set([xa, ya]))[0] + zax = list({0, 1, 2} - {xa, ya})[0] bnds3 = bnds + [le[zax], re[zax]] buff = np.zeros(size, dtype="float64") From 695f1969a70d6baaa90ad944ff63789e19dd1c4e Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Sat, 6 Jul 2024 08:59:26 -0500 Subject: [PATCH 093/186] fix complaint from ruff --- yt/utilities/lib/pixelization_routines.pyx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/yt/utilities/lib/pixelization_routines.pyx b/yt/utilities/lib/pixelization_routines.pyx index 7afc6fc560..62c8c3c960 100644 --- a/yt/utilities/lib/pixelization_routines.pyx +++ b/yt/utilities/lib/pixelization_routines.pyx @@ -2061,12 +2061,12 @@ def pixelize_sph_kernel_cutting( posx_rot, posy_rot, posz_rot, \ rot_bounds_x0, rot_bounds_x1, \ rot_bounds_y0, rot_bounds_y1, \ - rot_bounds_z0, rot_bounds_z1 = rotate_particle_coord(posx, posy, posz, - center, boxbounds, - periodic, - widthxy, 0., - normal_vector, - north_vector) + rot_bounds_z0, _ = rotate_particle_coord(posx, posy, posz, + center, boxbounds, + periodic, + widthxy, 0., + normal_vector, + north_vector) bounds_rot = np.array([rot_bounds_x0, rot_bounds_x1, rot_bounds_y0, rot_bounds_y1]) slicez_rot = rot_bounds_z0 From 80cb90da112f625684ca4bf2baf11f83a90f64a2 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 10 Jul 2024 18:48:58 -0500 Subject: [PATCH 094/186] debugging: axis-aligned slices pass tests --- .../coordinates/cartesian_coordinates.py | 47 ++++++++++++------- yt/utilities/lib/pixelization_routines.pyx | 30 ++++++++---- yt/visualization/plot_window.py | 3 -- 3 files changed, 51 insertions(+), 29 deletions(-) diff --git a/yt/geometry/coordinates/cartesian_coordinates.py b/yt/geometry/coordinates/cartesian_coordinates.py index 8c060e7f42..61835bfe20 100644 --- a/yt/geometry/coordinates/cartesian_coordinates.py +++ b/yt/geometry/coordinates/cartesian_coordinates.py @@ -169,7 +169,7 @@ def pixelize( bounds, size, antialias=True, - periodic=True, + periodic=None, *, return_mask=False, ): @@ -178,6 +178,18 @@ def pixelize( two-dimensional image plots. Relies on several sampling routines written in cython """ + if periodic is None: + if np.all(data_source.ds.periodicity): + periodic = True + elif not np.any(data_source.ds.periodicity): + periodic = False + else: + msg = ('Pixelization routines in CartesianCoordinate' + 'Handler can currently only deal with datasets ' + 'that are periodic in all or none of their ' + f'dimensions. This dataset {data_source.ds}' + f'had periodicity {data_source.ds.periodicity}') + raise NotImplementedError(msg) index = data_source.ds.index if hasattr(index, "meshes") and not isinstance( index.meshes[0], SemiStructuredMesh @@ -528,17 +540,19 @@ def _ortho_pixelize( buff_den = np.zeros(size, dtype="float64") for chunk in data_source.chunks([], "io"): + hsmlname = "smoothing_length" pixelize_sph_kernel_slice( buff, mask_uint8, - chunk[ptype, px_name].to("code_length"), - chunk[ptype, py_name].to("code_length"), - chunk[ptype, pz_name].to("code_length"), - chunk[ptype, "smoothing_length"].to("code_length"), - chunk[ptype, "mass"].to("code_mass"), - chunk[ptype, "density"].to("code_density"), - chunk[field].in_units(ounits), - bnds, data_source.coord, + chunk[ptype, px_name].to("code_length").v, + chunk[ptype, py_name].to("code_length").v, + chunk[ptype, pz_name].to("code_length").v, + chunk[ptype, hsmlname].to("code_length").v, + chunk[ptype, "mass"].to("code_mass").v, + chunk[ptype, "density"].to("code_density").v, + chunk[field].in_units(ounits).v, + bnds, + data_source.coord.to("code_length").v, check_period=int(periodic), period=period3, kernel_name=kernel_name @@ -547,14 +561,15 @@ def _ortho_pixelize( pixelize_sph_kernel_slice( buff_den, mask_uint8, - chunk[ptype, px_name].to("code_length"), - chunk[ptype, py_name].to("code_length"), - chunk[ptype, pz_name].to("code_length"), - chunk[ptype, "smoothing_length"].to("code_length"), - chunk[ptype, "mass"].to("code_mass"), - chunk[ptype, "density"].to("code_density"), + chunk[ptype, px_name].to("code_length").v, + chunk[ptype, py_name].to("code_length").v, + chunk[ptype, pz_name].to("code_length").v, + chunk[ptype, hsmlname].to("code_length").v, + chunk[ptype, "mass"].to("code_mass").v, + chunk[ptype, "density"].to("code_density").v, np.ones(chunk[ptype, "density"].shape[0]), - bnds, data_source.coord, + bnds, + data_source.coord.to("code_length").v, check_period=int(periodic), period=period3, kernel_name=kernel_name diff --git a/yt/utilities/lib/pixelization_routines.pyx b/yt/utilities/lib/pixelization_routines.pyx index 62c8c3c960..2de77bf6ee 100644 --- a/yt/utilities/lib/pixelization_routines.pyx +++ b/yt/utilities/lib/pixelization_routines.pyx @@ -1494,10 +1494,18 @@ def pixelize_sph_kernel_slice( np.float64_t[:] hsml, np.float64_t[:] pmass, np.float64_t[:] pdens, np.float64_t[:] quantity_to_smooth, - bounds, np.float64_t slicez, + bounds, + np.float64_t slicez, kernel_name="cubic", int check_period=1, period=None): + #print("bounds, slicez, kernel_name, check_period, period") + #print(bounds) + #print(slicez) + #print(kernel_name) + #print(check_period) + #print(period) + #print() # bounds are [x0, x1, y0, y1], slicez is the single coordinate # of the slice along the normal direction. # similar method to pixelize_sph_kernel_projection @@ -1531,7 +1539,7 @@ def pixelize_sph_kernel_slice( idy = 1.0/dy kernel = get_kernel_func(kernel_name) - + #print('particle index, ii, jj, px, py, pz') with nogil, parallel(): # NOTE see note in pixelize_sph_kernel_projection local_buff = malloc(sizeof(np.float64_t) * xsize * ysize) @@ -1548,18 +1556,19 @@ def pixelize_sph_kernel_slice( if j % 100000 == 0: with gil: PyErr_CheckSignals() - + #with gil: + # print(j) xiter[1] = yiter[1] = 999 - + pz = posz[j] if check_period == 1: if posx[j] - hsml[j] < x_min: - xiter[1] = +1 + xiter[1] = 1 xiterv[1] = period_x elif posx[j] + hsml[j] > x_max: xiter[1] = -1 xiterv[1] = -period_x if posy[j] - hsml[j] < y_min: - yiter[1] = +1 + yiter[1] = 1 yiterv[1] = period_y elif posy[j] + hsml[j] > y_max: yiter[1] = -1 @@ -1570,8 +1579,6 @@ def pixelize_sph_kernel_slice( pz = posz[j] - period_z elif posz[j] + hsml[j] < slicez: pz = posz[j] + period_z - else: - pz = posz[j] h_j2 = hsml[j] * hsml[j] #fmax(hsml[j]*hsml[j], dx*dy) h_j = hsml[j] #math.sqrt(h_j2) @@ -1603,7 +1610,8 @@ def pixelize_sph_kernel_slice( y1 = ( (py + hsml[j] - y_min) * idy) y0 = iclip(y0-1, 0, ysize) y1 = iclip(y1+1, 0, ysize) - + #with gil: + # print(ii, jj, px, py, pz) # Now we know which pixels to deposit onto for this particle, # so loop over them and add this particle's contribution for xi in range(x0, x1): @@ -1917,7 +1925,7 @@ def rotate_particle_coord(np.float64_t[:] px, # the normal vector be the z-axis (i.e., the viewer's perspective), and then # another rotation to make the north-vector be the y-axis (i.e., north). # Fortunately, total_rotation_matrix = rotation_matrix_1 x rotation_matrix_2 - cdef int num_particles = np.size(px) + cdef np.int64_t num_particles = np.size(px) cdef np.float64_t[:] z_axis = np.array([0., 0., 1.], dtype="float64") cdef np.float64_t[:] y_axis = np.array([0., 1., 0.], dtype="float64") cdef np.float64_t[:, :] normal_rotation_matrix @@ -1936,6 +1944,8 @@ def rotate_particle_coord(np.float64_t[:] px, cdef np.float64_t[:] coordinate_matrix = np.empty(3, dtype="float64") cdef np.float64_t[:] rotated_coordinates cdef np.float64_t[:] rotated_center + cdef np.int64_t i + cdef int ax #rotated_center = rotation_matmul( # rotation_matrix, np.array([center[0], center[1], center[2]])) rotated_center = np.zeros((3,), dtype=center.dtype) diff --git a/yt/visualization/plot_window.py b/yt/visualization/plot_window.py index b2a49538cf..af603e0d20 100644 --- a/yt/visualization/plot_window.py +++ b/yt/visualization/plot_window.py @@ -1595,7 +1595,6 @@ class SlicePlot(NormalPlot): def __new__( # type: ignore cls, ds, normal, fields, *args, **kwargs ) -> Union["AxisAlignedSlicePlot", "OffAxisSlicePlot"]: - print('SlicePlot call cls: ', cls) if cls is SlicePlot: normal = cls.sanitize_normal_vector(ds, normal) if isinstance(normal, str): @@ -1603,8 +1602,6 @@ def __new__( # type: ignore else: cls = OffAxisSlicePlot self = object.__new__(cls) - print('SlicePlot result cls: ', cls) - print('Sliceplot return self: ', self) return self # type: ignore [return-value] From 1907b2fc2b28e6c58168e7b0f1f297353462caf7 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 10 Jul 2024 19:16:13 -0500 Subject: [PATCH 095/186] fix a ruff complaint --- yt/visualization/plot_window.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/visualization/plot_window.py b/yt/visualization/plot_window.py index af603e0d20..6edec738e6 100644 --- a/yt/visualization/plot_window.py +++ b/yt/visualization/plot_window.py @@ -12,9 +12,9 @@ from yt._maintenance.deprecation import issue_deprecation_warning from yt._typing import AlphaT from yt.data_objects.image_array import ImageArray -from yt.frontends.ytdata.data_structures import YTSpatialPlotDataset from yt.frontends.sph.data_structures import ParticleDataset from yt.frontends.stream.data_structures import StreamParticlesDataset +from yt.frontends.ytdata.data_structures import YTSpatialPlotDataset from yt.funcs import ( fix_axis, fix_unitary, From a80228aab074da91f268f30c99fbfac1e10f388e Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Thu, 11 Jul 2024 18:52:46 -0500 Subject: [PATCH 096/186] on/off-axis SPH slices actually work now --- yt/geometry/coordinates/cartesian_coordinates.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/yt/geometry/coordinates/cartesian_coordinates.py b/yt/geometry/coordinates/cartesian_coordinates.py index 61835bfe20..3df343fe5a 100644 --- a/yt/geometry/coordinates/cartesian_coordinates.py +++ b/yt/geometry/coordinates/cartesian_coordinates.py @@ -577,7 +577,7 @@ def _ortho_pixelize( if normalize: normalization_2d_utility(buff, buff_den) - + mask = mask_uint8.astype("bool", copy=False) if smoothing_style == "gather": @@ -692,7 +692,7 @@ def _oblique_pixelize(self, data_source, field, bounds, size, antialias): le = data_source.ds.domain_left_edge.to("code_length") re = data_source.ds.domain_right_edge.to("code_length") boxbounds = np.array([le[0], re[0], le[1], re[1], le[2], re[2]]) - periodic = data_source.ds.coordinates.period.astype(bool).v + periodic = data_source.ds.periodicity ptype = field[0] if ptype == "gas": ptype = data_source.ds._sph_ptypes[0] @@ -757,6 +757,9 @@ def _oblique_pixelize(self, data_source, field, bounds, size, antialias): normalization_2d_utility(buff, buff_den) mask = mask_uint8.astype("bool", copy=False) + # swap axes for image plotting + mask = mask.swapaxes(0, 1) + buff = buff.swapaxes(0, 1) # whatever other data this code could handle before the # SPH option was added From 8e5aca2e97b969cefc2e8abd4067b10d46999f67 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Thu, 11 Jul 2024 22:50:36 -0500 Subject: [PATCH 097/186] hopefully fixed induced grid projection bug; SPH projections ignore periodic arg in pixelize, use dataset periodicity instead --- .../coordinates/cartesian_coordinates.py | 49 +++++++++---------- yt/utilities/lib/pixelization_routines.pyx | 28 +++++++---- 2 files changed, 43 insertions(+), 34 deletions(-) diff --git a/yt/geometry/coordinates/cartesian_coordinates.py b/yt/geometry/coordinates/cartesian_coordinates.py index 3df343fe5a..038550adda 100644 --- a/yt/geometry/coordinates/cartesian_coordinates.py +++ b/yt/geometry/coordinates/cartesian_coordinates.py @@ -169,7 +169,7 @@ def pixelize( bounds, size, antialias=True, - periodic=None, + periodic=True, *, return_mask=False, ): @@ -178,18 +178,6 @@ def pixelize( two-dimensional image plots. Relies on several sampling routines written in cython """ - if periodic is None: - if np.all(data_source.ds.periodicity): - periodic = True - elif not np.any(data_source.ds.periodicity): - periodic = False - else: - msg = ('Pixelization routines in CartesianCoordinate' - 'Handler can currently only deal with datasets ' - 'that are periodic in all or none of their ' - f'dimensions. This dataset {data_source.ds}' - f'had periodicity {data_source.ds.periodicity}') - raise NotImplementedError(msg) index = data_source.ds.index if hasattr(index, "meshes") and not isinstance( index.meshes[0], SemiStructuredMesh @@ -376,6 +364,7 @@ def _ortho_pixelize( return_mask=True, ) elif isinstance(data_source.ds, particle_datasets) and is_sph_field: + # SPH handling ptype = field[0] if ptype == "gas": ptype = data_source.ds._sph_ptypes[0] @@ -384,7 +373,20 @@ def _ortho_pixelize( # need z coordinates for depth, # but name isn't saved in the handler -> use the 'other one' pz_name = list((set(self.axis_order) - {px_name, py_name}))[0] - + + # ignore default True periodic argument + # (not actually supplied by a call from + # FixedResolutionBuffer), and use the dataset periodicity + # instead + xa = self.x_axis[dim] + ya = self.y_axis[dim] + #axorder = data_source.ds.coordinates.axis_order + za = list({0, 1, 2} - {xa, ya})[0] + ds_periodic = data_source.ds.periodicity + _periodic = np.array(ds_periodic) + _periodic[0] = ds_periodic[xa] + _periodic[1] = ds_periodic[ya] + _periodic[2] = ds_periodic[za] ounits = data_source.ds.field_info[field].output_units bnds = data_source.ds.arr(bounds, "code_length").tolist() kernel_name = None @@ -393,12 +395,10 @@ def _ortho_pixelize( if kernel_name is None: kernel_name = "cubic" - if isinstance(data_source, YTParticleProj): + if isinstance(data_source, YTParticleProj): # projection weight = data_source.weight_field moment = data_source.moment le, re = data_source.data_source.get_bbox() - xa = self.x_axis[dim] - ya = self.y_axis[dim] # If we're not periodic, we need to clip to the boundary edges # or we get errors about extending off the edge of the region. # (depth/z range is handled by region setting) @@ -424,8 +424,7 @@ def _ortho_pixelize( proj_reg.set_field_parameter("axis", data_source.axis) # need some z bounds for SPH projection # -> use source bounds - zax = list({0, 1, 2} - {xa, ya})[0] - bnds3 = bnds + [le[zax], re[zax]] + bnds3 = bnds + [le[za], re[za]] buff = np.zeros(size, dtype="float64") mask_uint8 = np.zeros_like(buff, dtype="uint8") @@ -443,7 +442,7 @@ def _ortho_pixelize( chunk[ptype, "density"].to("code_density"), chunk[field].in_units(ounits), bnds3, - check_period=int(periodic), + _check_period=_periodic.astype("int"), period=period3, kernel_name=kernel_name ) @@ -476,7 +475,7 @@ def _ortho_pixelize( chunk[ptype, "density"].to("code_density"), chunk[field].in_units(ounits), bnds3, - check_period=int(periodic), + _check_period=_periodic.astype("int"), period=period3, weight_field=chunk[weight].in_units(wounits), kernel_name=kernel_name @@ -500,7 +499,7 @@ def _ortho_pixelize( chunk[ptype, "density"].to("code_density"), chunk[weight].in_units(wounits), bnds3, - check_period=int(periodic), + _check_period=_periodic.astype("int"), period=period3, kernel_name=kernel_name ) @@ -521,7 +520,7 @@ def _ortho_pixelize( chunk[ptype, "density"].to("code_density"), chunk[field].in_units(ounits) ** 2, bnds3, - check_period=int(periodic), + _check_period=_periodic.astype("int"), period=period3, weight_field=chunk[weight].in_units(wounits), kernel_name=kernel_name @@ -553,7 +552,7 @@ def _ortho_pixelize( chunk[field].in_units(ounits).v, bnds, data_source.coord.to("code_length").v, - check_period=int(periodic), + _check_period=_periodic.astype("int"), period=period3, kernel_name=kernel_name ) @@ -570,7 +569,7 @@ def _ortho_pixelize( np.ones(chunk[ptype, "density"].shape[0]), bnds, data_source.coord.to("code_length").v, - check_period=int(periodic), + _check_period=_periodic.astype("int"), period=period3, kernel_name=kernel_name ) diff --git a/yt/utilities/lib/pixelization_routines.pyx b/yt/utilities/lib/pixelization_routines.pyx index 2de77bf6ee..3a27345004 100644 --- a/yt/utilities/lib/pixelization_routines.pyx +++ b/yt/utilities/lib/pixelization_routines.pyx @@ -1131,7 +1131,7 @@ def pixelize_sph_kernel_projection( bounds, kernel_name="cubic", weight_field=None, - int check_period=1, + _check_period = (1, 1, 1), period=None): cdef np.intp_t xsize, ysize @@ -1148,6 +1148,7 @@ def pixelize_sph_kernel_projection( cdef np.float64_t * xiterv cdef np.float64_t * yiterv cdef np.float64_t * ziterv + cdef np.int8_t[3] check_period if weight_field is not None: _weight_field = weight_field @@ -1156,7 +1157,8 @@ def pixelize_sph_kernel_projection( period_x = period[0] period_y = period[1] period_z = period[2] - + for i in range(3): + check_period[i] = np.int8(_check_period[i]) # we find the x and y range over which we have pixels and we find how many # pixels we have in each dimension xsize, ysize = buff.shape[0], buff.shape[1] @@ -1211,19 +1213,21 @@ def pixelize_sph_kernel_projection( xiter[1] = yiter[1] = ziter[1] = 999 - if check_period == 1: + if check_period[0] == 1: if posx[j] - hsml[j] < x_min: xiter[1] = +1 xiterv[1] = period_x elif posx[j] + hsml[j] > x_max: xiter[1] = -1 xiterv[1] = -period_x + if check_period[1] == 1: if posy[j] - hsml[j] < y_min: yiter[1] = +1 yiterv[1] = period_y elif posy[j] + hsml[j] > y_max: yiter[1] = -1 yiterv[1] = -period_y + if check_period[2] == 1: if posz[j] - hsml[j] < z_min: ziter[1] = +1 ziterv[1] = period_z @@ -1497,7 +1501,7 @@ def pixelize_sph_kernel_slice( bounds, np.float64_t slicez, kernel_name="cubic", - int check_period=1, + _check_period = (1, 1, 1), period=None): #print("bounds, slicez, kernel_name, check_period, period") #print(bounds) @@ -1520,14 +1524,16 @@ def pixelize_sph_kernel_slice( cdef int * yiter cdef np.float64_t * xiterv cdef np.float64_t * yiterv + cdef np.int8_t[3] check_period if period is not None: period_x = period[0] period_y = period[1] period_z = period[2] - + for i in range(3): + check_period[i] = np.int8(_check_period[i]) + xsize, ysize = buff.shape[0], buff.shape[1] - x_min = bounds[0] x_max = bounds[1] y_min = bounds[2] @@ -1560,19 +1566,21 @@ def pixelize_sph_kernel_slice( # print(j) xiter[1] = yiter[1] = 999 pz = posz[j] - if check_period == 1: + if check_period[0] == 1: if posx[j] - hsml[j] < x_min: xiter[1] = 1 xiterv[1] = period_x elif posx[j] + hsml[j] > x_max: xiter[1] = -1 xiterv[1] = -period_x + if check_period[1] == 1: if posy[j] - hsml[j] < y_min: yiter[1] = 1 yiterv[1] = period_y elif posy[j] + hsml[j] > y_max: yiter[1] = -1 yiterv[1] = -period_y + if check_period[2] == 1: # z of particle might be < hsml from the slice plane # but across a periodic boundary if posz[j] - hsml[j] > slicez: @@ -1933,6 +1941,7 @@ def rotate_particle_coord(np.float64_t[:] px, cdef np.float64_t[:, :] north_rotation_matrix cdef np.float64_t[:, :] rotation_matrix + normal_rotation_matrix = get_rotation_matrix(normal_vector, z_axis) transformed_north_vector = np.matmul(normal_rotation_matrix, north_vector) north_rotation_matrix = get_rotation_matrix(transformed_north_vector, y_axis) @@ -2034,6 +2043,7 @@ def off_axis_projection_SPH(np.float64_t[:] px, # does not apply to the *rotated* coordinates, the periodicity # approach implemented for this along-axis projection method # would fail here + check_period = np.array([0, 0, 0], dtype="int") pixelize_sph_kernel_projection(projection_array, mask, px_rotated, @@ -2047,7 +2057,7 @@ def off_axis_projection_SPH(np.float64_t[:] px, rot_bounds_y0, rot_bounds_y1, rot_bounds_z0, rot_bounds_z1], weight_field=weight_field, - check_period=0, + _check_period=check_period, kernel_name=kernel_name) # like slice pixelization, but for off-axis planes @@ -2085,7 +2095,7 @@ def pixelize_sph_kernel_cutting( hsml, pmass, pdens, quantity_to_smooth, bounds_rot, slicez_rot, kernel_name=kernel_name, - check_period=0, + _check_period=np.zeros(3, dtype="int"), period=None) From 9cc98ba378a2b0251cd79b9e9633af10c5e310f3 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Thu, 11 Jul 2024 23:45:40 -0500 Subject: [PATCH 098/186] partial debug issues from pull request tests --- yt/visualization/plot_window.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/yt/visualization/plot_window.py b/yt/visualization/plot_window.py index 6edec738e6..636674aba2 100644 --- a/yt/visualization/plot_window.py +++ b/yt/visualization/plot_window.py @@ -2465,10 +2465,16 @@ def __init__( # field in a single call # checks for SPH fields copied from the # _ortho_pixelize method in cartesian_coordinates.py + + ## data_source might be None here + ## (OffAxisProjectionDummyDataSource gets used later) + if data_source is None: + data_source = ds.all_data() field = data_source._determine_fields(fields)[0] finfo = data_source.ds.field_info[field] - particle_datasets = (ParticleDataset, StreamParticlesDataset) is_sph_field = finfo.is_sph_field + particle_datasets = (ParticleDataset, StreamParticlesDataset) + if isinstance(data_source.ds, particle_datasets) and is_sph_field: center_use = center else: From 429536dd8f30a4e3befa8f28f4fed6d07cfbb35f Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Fri, 12 Jul 2024 17:30:35 -0500 Subject: [PATCH 099/186] attempted fix bounds issues in off_axis_projection calls (non-SPH ds) --- yt/visualization/fixed_resolution.py | 3 ++ yt/visualization/plot_window.py | 34 +++++++++++++------ .../volume_rendering/off_axis_projection.py | 8 +++-- .../volume_rendering/old_camera.py | 3 +- 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/yt/visualization/fixed_resolution.py b/yt/visualization/fixed_resolution.py index 782e734ad3..e19c9c58f0 100644 --- a/yt/visualization/fixed_resolution.py +++ b/yt/visualization/fixed_resolution.py @@ -630,10 +630,13 @@ def _generate_image_and_mask(self, item) -> None: self.buff_size[1], ) dd = self.data_source + # only need the first two for SPH, + # but need the third one for other data formats. width = self.ds.arr( ( self.bounds[1] - self.bounds[0], self.bounds[3] - self.bounds[2], + self.bounds[5] - self.bounds[4], ) ) buff = off_axis_projection( diff --git a/yt/visualization/plot_window.py b/yt/visualization/plot_window.py index 636674aba2..2503d3b5c1 100644 --- a/yt/visualization/plot_window.py +++ b/yt/visualization/plot_window.py @@ -82,14 +82,15 @@ def get_window_parameters(axis, center, width, ds): return (bounds, center, display_center) -def get_oblique_window_parameters(normal, center, width, ds, depth=None): +def get_oblique_window_parameters(normal, center, width, ds, + depth=None, get3bounds=False): center, display_center = ds.coordinates.sanitize_center(center, axis=None) width = ds.coordinates.sanitize_width(normal, width, depth) if len(width) == 2: # Transforming to the cutting plane coordinate system # the original dimensionless center messes up off-axis - # SPH projections though + # SPH projections though -> don't use this center there center = ((center - ds.domain_left_edge) / ds.domain_width - 0.5)\ * ds.domain_width (normal, perp1, perp2) = ortho_find(normal) @@ -98,6 +99,14 @@ def get_oblique_window_parameters(normal, center, width, ds, depth=None): w = tuple(el.in_units("code_length") for el in width) bounds = tuple(((2 * (i % 2)) - 1) * w[i // 2] / 2 for i in range(len(w) * 2)) + if get3bounds and depth is None: + # off-axis projection, depth not specified + # -> set 'large enough' depth using half the box diagonal + margin + d2 = ds.domain_width[0].in_units("code_length")**2 + d2 += ds.domain_width[1].in_units("code_length")**2 + d2 += ds.domain_width[2].in_units("code_length")**2 + diag = np.sqrt(d2) + bounds = bounds + (-0.51 * diag, 0.51 * diag) return (bounds, center) @@ -2450,16 +2459,20 @@ def __init__( ): if ds.geometry not in self._supported_geometries: raise NotImplementedError( - f"off-axis slices are not supported for {ds.geometry!r} geometry\n" - f"currently supported geometries: {self._supported_geometries!r}" + "off-axis slices are not supported" + f" for {ds.geometry!r} geometry\n" + "currently supported geometries:" + f" {self._supported_geometries!r}" ) # center_rot normalizes the center to (0,0), # units match bounds # for SPH data, we want to input the original center # the cython backend handles centering to this point and - # rotation + # rotation. + # get3bounds gets a depth 0.5 * diagonal + margin in the + # depth=None case. (bounds, center_rot) = get_oblique_window_parameters( - normal, center, width, ds, depth=depth + normal, center, width, ds, depth=depth, get3bounds=True, ) # will probably fail if you try to project an SPH and non-SPH # field in a single call @@ -2480,14 +2493,15 @@ def __init__( else: center_use = center_rot fields = list(iter_fields(fields))[:] - oap_width = ds.arr( - (bounds[1] - bounds[0], bounds[3] - bounds[2]) - ) + #oap_width = ds.arr( + # (bounds[1] - bounds[0], + # bounds[3] - bounds[2]) + #) OffAxisProj = OffAxisProjectionDummyDataSource( center_use, ds, normal, - oap_width, + width, fields, interpolated, weight=weight_field, diff --git a/yt/visualization/volume_rendering/off_axis_projection.py b/yt/visualization/volume_rendering/off_axis_projection.py index 6f741a1004..5ca35b297c 100644 --- a/yt/visualization/volume_rendering/off_axis_projection.py +++ b/yt/visualization/volume_rendering/off_axis_projection.py @@ -233,6 +233,8 @@ def off_axis_projection( x_min, y_min, z_min = le x_max, y_max, z_max = re bounds = [x_min, x_max, y_min, y_max, z_min, z_max] + # only need (rotated) x/y widths + _width = (width.to("code_length").d)[:2] finfo = data_source.ds.field_info[item] ounits = finfo.output_units kernel_name = None @@ -252,7 +254,7 @@ def off_axis_projection( chunk[ptype, "smoothing_length"].to("code_length").d, bounds, center.to("code_length").d, - width.to("code_length").d, + _width, periodic, chunk[item].in_units(ounits), buf, @@ -293,7 +295,7 @@ def off_axis_projection( chunk[ptype, "smoothing_length"].to("code_length").d, bounds, center.to("code_length").d, - width.to("code_length").d, + _width, periodic, chunk[item].in_units(ounits), buf, @@ -315,7 +317,7 @@ def off_axis_projection( chunk[ptype, "smoothing_length"].to("code_length").d, bounds, center.to("code_length").d, - width.to("code_length").d, + _width, periodic, chunk[weight].to(wounits), weight_buff, diff --git a/yt/visualization/volume_rendering/old_camera.py b/yt/visualization/volume_rendering/old_camera.py index cca5ff178a..0ca8489814 100644 --- a/yt/visualization/volume_rendering/old_camera.py +++ b/yt/visualization/volume_rendering/old_camera.py @@ -2438,7 +2438,8 @@ def _render(self, double_check, num_threads, image, sampler, msg): data_object_registry["stereospherical_camera"] = StereoSphericalCamera - +# replaced in volume_rendering API by the function of the same name in +# yt/visualization/volume_rendering/off_axis_projection def off_axis_projection( ds, center, From a088d5fa3be979eba33f44979f139cfda587bf4c Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Fri, 12 Jul 2024 17:45:25 -0500 Subject: [PATCH 100/186] attempted fix: ParticleImageBuffer counts on rotate_particle_coord function, but I changed the function --- yt/utilities/lib/pixelization_routines.pyx | 58 +++++++++++++++++++++- yt/visualization/fixed_resolution.py | 4 +- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/yt/utilities/lib/pixelization_routines.pyx b/yt/utilities/lib/pixelization_routines.pyx index 3a27345004..fab75a19d3 100644 --- a/yt/utilities/lib/pixelization_routines.pyx +++ b/yt/utilities/lib/pixelization_routines.pyx @@ -1916,7 +1916,63 @@ def pixelize_element_mesh_line(np.ndarray[np.float64_t, ndim=2] coords, free(field_vals) return arc_length, plot_values +# intended for use in ParticleImageBuffer +@cython.boundscheck(False) +@cython.wraparound(False) +def rotate_particle_coord_pib(np.float64_t[:] px, + np.float64_t[:] py, + np.float64_t[:] pz, + center, + width, + normal_vector, + north_vector): + # We want to do two rotations, one to first rotate our coordinates to have + # the normal vector be the z-axis (i.e., the viewer's perspective), and then + # another rotation to make the north-vector be the y-axis (i.e., north). + # Fortunately, total_rotation_matrix = rotation_matrix_1 x rotation_matrix_2 + cdef int num_particles = np.size(px) + cdef np.float64_t[:] z_axis = np.array([0., 0., 1.], dtype="float64") + cdef np.float64_t[:] y_axis = np.array([0., 1., 0.], dtype="float64") + cdef np.float64_t[:, :] normal_rotation_matrix + cdef np.float64_t[:] transformed_north_vector + cdef np.float64_t[:, :] north_rotation_matrix + cdef np.float64_t[:, :] rotation_matrix + normal_rotation_matrix = get_rotation_matrix(normal_vector, z_axis) + transformed_north_vector = np.matmul(normal_rotation_matrix, north_vector) + north_rotation_matrix = get_rotation_matrix(transformed_north_vector, y_axis) + rotation_matrix = np.matmul(north_rotation_matrix, normal_rotation_matrix) + + cdef np.float64_t[:] px_rotated = np.empty(num_particles, dtype="float64") + cdef np.float64_t[:] py_rotated = np.empty(num_particles, dtype="float64") + cdef np.float64_t[:] coordinate_matrix = np.empty(3, dtype="float64") + cdef np.float64_t[:] rotated_coordinates + cdef np.float64_t[:] rotated_center + rotated_center = rotation_matmul( + rotation_matrix, np.array([center[0], center[1], center[2]])) + + # set up the rotated bounds + cdef np.float64_t rot_bounds_x0 = rotated_center[0] - width[0] / 2 + cdef np.float64_t rot_bounds_x1 = rotated_center[0] + width[0] / 2 + cdef np.float64_t rot_bounds_y0 = rotated_center[1] - width[1] / 2 + cdef np.float64_t rot_bounds_y1 = rotated_center[1] + width[1] / 2 + + for i in range(num_particles): + coordinate_matrix[0] = px[i] + coordinate_matrix[1] = py[i] + coordinate_matrix[2] = pz[i] + rotated_coordinates = rotation_matmul( + rotation_matrix, coordinate_matrix) + px_rotated[i] = rotated_coordinates[0] + py_rotated[i] = rotated_coordinates[1] + + return px_rotated, py_rotated, rot_bounds_x0, rot_bounds_x1, rot_bounds_y0, rot_bounds_y1 + +# version intended for SPH off-axis slices/projections +# includes dealing with periodic boundaries, but also +# shifts particles so center -> origin. +# therefore, don't want to use this in the ParticleImageBuffer, +# which expects differently centered coordinates. @cython.boundscheck(False) @cython.wraparound(False) def rotate_particle_coord(np.float64_t[:] px, @@ -1941,7 +1997,6 @@ def rotate_particle_coord(np.float64_t[:] px, cdef np.float64_t[:, :] north_rotation_matrix cdef np.float64_t[:, :] rotation_matrix - normal_rotation_matrix = get_rotation_matrix(normal_vector, z_axis) transformed_north_vector = np.matmul(normal_rotation_matrix, north_vector) north_rotation_matrix = get_rotation_matrix(transformed_north_vector, y_axis) @@ -1999,7 +2054,6 @@ def rotate_particle_coord(np.float64_t[:] px, @cython.boundscheck(False) @cython.wraparound(False) - def off_axis_projection_SPH(np.float64_t[:] px, np.float64_t[:] py, np.float64_t[:] pz, diff --git a/yt/visualization/fixed_resolution.py b/yt/visualization/fixed_resolution.py index e19c9c58f0..f597393a5d 100644 --- a/yt/visualization/fixed_resolution.py +++ b/yt/visualization/fixed_resolution.py @@ -17,7 +17,7 @@ ) from yt.utilities.lib.pixelization_routines import ( pixelize_cylinder, - rotate_particle_coord, + rotate_particle_coord_pib, ) from yt.utilities.math_utils import compute_stddev_image from yt.utilities.on_demand_imports import _h5py as h5py @@ -749,7 +749,7 @@ def _generate_image_and_mask(self, item) -> None: if hasattr(w, "to_value"): w = w.to_value("code_length") wd.append(w) - x_data, y_data, *bounds = rotate_particle_coord( + x_data, y_data, *bounds = rotate_particle_coord_pib( dd[ftype, "particle_position_x"].to_value("code_length"), dd[ftype, "particle_position_y"].to_value("code_length"), dd[ftype, "particle_position_z"].to_value("code_length"), From effd35109e62d17e90003f6aa18af74ecf5464fe Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Fri, 12 Jul 2024 17:53:54 -0500 Subject: [PATCH 101/186] add new pz argument to test_off_axis_SPH pixelize_sph_kernel_projection call --- yt/visualization/volume_rendering/tests/test_off_axis_SPH.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yt/visualization/volume_rendering/tests/test_off_axis_SPH.py b/yt/visualization/volume_rendering/tests/test_off_axis_SPH.py index dc5f4ae13a..57f6f85d52 100644 --- a/yt/visualization/volume_rendering/tests/test_off_axis_SPH.py +++ b/yt/visualization/volume_rendering/tests/test_off_axis_SPH.py @@ -26,6 +26,7 @@ def test_no_rotation(): width = right_edge - left_edge px = ad["all", "particle_position_x"] py = ad["all", "particle_position_y"] + pz = ad["all", "particle_position_y"] hsml = ad["all", "smoothing_length"] quantity_to_smooth = ad["gas", "density"] density = ad["io", "density"] @@ -38,7 +39,7 @@ def test_no_rotation(): ds, center, normal_vector, width, resolution, ("gas", "density") ) pixelize_sph_kernel_projection( - buf2, mask, px, py, hsml, mass, density, quantity_to_smooth, bounds + buf2, mask, px, py, pz, hsml, mass, density, quantity_to_smooth, bounds ) assert_almost_equal(buf1.ndarray_view(), buf2) From 33b2c102947d11058965cd7694cd0d461b1300f5 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Fri, 12 Jul 2024 18:27:55 -0500 Subject: [PATCH 102/186] attempted fix issue with non-coordinate center arg off_axis_projection --- yt/visualization/plot_window.py | 5 ++++- yt/visualization/volume_rendering/off_axis_projection.py | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/yt/visualization/plot_window.py b/yt/visualization/plot_window.py index 2503d3b5c1..2fdc46edc7 100644 --- a/yt/visualization/plot_window.py +++ b/yt/visualization/plot_window.py @@ -22,6 +22,7 @@ iter_fields, mylog, obj_length, + parse_center_array, validate_moment, ) from yt.geometry.api import Geometry @@ -2489,7 +2490,9 @@ def __init__( particle_datasets = (ParticleDataset, StreamParticlesDataset) if isinstance(data_source.ds, particle_datasets) and is_sph_field: - center_use = center + center_use = center = parse_center_array(center, + ds=data_source.ds, + axis=None) else: center_use = center_rot fields = list(iter_fields(fields))[:] diff --git a/yt/visualization/volume_rendering/off_axis_projection.py b/yt/visualization/volume_rendering/off_axis_projection.py index 5ca35b297c..44f03ade0c 100644 --- a/yt/visualization/volume_rendering/off_axis_projection.py +++ b/yt/visualization/volume_rendering/off_axis_projection.py @@ -242,7 +242,6 @@ def off_axis_projection( kernel_name = data_source.ds.kernel_name if kernel_name is None: kernel_name = "cubic" - if weight is None: for chunk in data_source.chunks([], "io"): off_axis_projection_SPH( From 0d13bebcc06ced43134c0b8b7094c03734511624 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Fri, 12 Jul 2024 18:46:56 -0500 Subject: [PATCH 103/186] attempted fix ruff complaint import sorting --- yt/geometry/coordinates/cartesian_coordinates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/geometry/coordinates/cartesian_coordinates.py b/yt/geometry/coordinates/cartesian_coordinates.py index 038550adda..603a052088 100644 --- a/yt/geometry/coordinates/cartesian_coordinates.py +++ b/yt/geometry/coordinates/cartesian_coordinates.py @@ -668,10 +668,10 @@ def _ortho_pixelize( return buff, mask def _oblique_pixelize(self, data_source, field, bounds, size, antialias): - from yt.frontends.ytdata.data_structures import YTSpatialPlotDataset from yt.data_objects.selection_objects.slices import YTCuttingPlane from yt.frontends.sph.data_structures import ParticleDataset from yt.frontends.stream.data_structures import StreamParticlesDataset + from yt.frontends.ytdata.data_structures import YTSpatialPlotDataset # Determine what sort of data we're dealing with # -> what backend to use From 15fcea4e062ccb9fe0eaf7d8177f0b7d8e7582eb Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Fri, 12 Jul 2024 18:53:44 -0500 Subject: [PATCH 104/186] removed unneeded parentheses --- yt/geometry/coordinates/cartesian_coordinates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/geometry/coordinates/cartesian_coordinates.py b/yt/geometry/coordinates/cartesian_coordinates.py index 603a052088..96f43e0c4d 100644 --- a/yt/geometry/coordinates/cartesian_coordinates.py +++ b/yt/geometry/coordinates/cartesian_coordinates.py @@ -372,7 +372,7 @@ def _ortho_pixelize( py_name = self.axis_name[self.y_axis[dim]] # need z coordinates for depth, # but name isn't saved in the handler -> use the 'other one' - pz_name = list((set(self.axis_order) - {px_name, py_name}))[0] + pz_name = list(set(self.axis_order) - {px_name, py_name})[0] # ignore default True periodic argument # (not actually supplied by a call from From 5bb4bd7ac2294d2aebb4dd3eebf5c3d71ee69e96 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Tue, 16 Jul 2024 19:35:57 -0500 Subject: [PATCH 105/186] SPH scatter gridding fixes, tests passed --- .../construction_data_containers.py | 6 ++- yt/utilities/lib/pixelization_routines.pyx | 46 +++++++++++++++---- 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/yt/data_objects/construction_data_containers.py b/yt/data_objects/construction_data_containers.py index 03c10f92c1..b6dfd4dd8b 100644 --- a/yt/data_objects/construction_data_containers.py +++ b/yt/data_objects/construction_data_containers.py @@ -649,6 +649,8 @@ class YTCoveringGrid(YTSelectionContainer3D): level : int The resolution level data to which data will be gridded. Level 0 is the root grid dx for that dataset. + (The grid resolution will be simulation size / 2**level along + each grid axis.) left_edge : array_like The left edge of the region to be extracted. Specify units by supplying a YTArray, otherwise code length units are assumed. @@ -1001,8 +1003,8 @@ def _fill_sph_particles(self, fields): period = self.ds.coordinates.period.copy() if hasattr(period, "in_units"): period = period.in_units("code_length").d - # TODO maybe there is a better way of handling this - is_periodic = int(any(self.ds.periodicity)) + # check periodicity per dimension + is_periodic = self.ds.periodicity if smoothing_style == "scatter": for field in fields: diff --git a/yt/utilities/lib/pixelization_routines.pyx b/yt/utilities/lib/pixelization_routines.pyx index fab75a19d3..5752944f81 100644 --- a/yt/utilities/lib/pixelization_routines.pyx +++ b/yt/utilities/lib/pixelization_routines.pyx @@ -1669,7 +1669,7 @@ def pixelize_sph_kernel_arbitrary_grid(np.float64_t[:, :, :] buff, np.float64_t[:] pdens, np.float64_t[:] quantity_to_smooth, bounds, pbar=None, kernel_name="cubic", - int check_period=1, period=None): + check_period=True, period=None): cdef np.intp_t xsize, ysize, zsize cdef np.float64_t x_min, x_max, y_min, y_max, z_min, z_max, prefactor_j @@ -1685,10 +1685,21 @@ def pixelize_sph_kernel_arbitrary_grid(np.float64_t[:, :, :] buff, cdef np.float64_t xiterv[2] cdef np.float64_t yiterv[2] cdef np.float64_t ziterv[2] + cdef int[3] periodic + xiter[0] = yiter[0] = ziter[0] = 0 xiterv[0] = yiterv[0] = ziterv[0] = 0.0 + if hasattr(check_period, "__len__"): + periodic[0] = int(check_period[0]) + periodic[1] = int(check_period[1]) + periodic[2] = int(check_period[2]) + else: + _cp = int(check_period) + periodic[0] = _cp + periodic[1] = _cp + periodic[2] = _cp if period is not None: period_x = period[0] period_y = period[1] @@ -1710,7 +1721,17 @@ def pixelize_sph_kernel_arbitrary_grid(np.float64_t[:, :, :] buff, idz = 1.0/dz kernel = get_kernel_func(kernel_name) - + + # nogil seems dangerous here, but there are no actual parallel + # sections (e.g., prange instead of range) used here. + # However, for future writers: + # !! the final buff array mutation has no protections against + # !! race conditions (e.g., OpenMP's atomic read/write), and + # !! cython doesn't seem to provide such options. + # (other routines in this file use private variable buffer arrays + # and add everything together at the end, but grid arrays can get + # big fast, and having such a large array in each thread could + # cause memory use issues.) with nogil: # TODO make this parallel without using too much memory for j in range(0, posx.shape[0]): @@ -1719,23 +1740,26 @@ def pixelize_sph_kernel_arbitrary_grid(np.float64_t[:, :, :] buff, if(pbar is not None): pbar.update(50000) PyErr_CheckSignals() + # end with gil xiter[1] = yiter[1] = ziter[1] = 999 xiterv[1] = yiterv[1] = ziterv[1] = 0.0 - if check_period == 1: + if periodic[0] == 1: if posx[j] - hsml[j] < x_min: xiter[1] = +1 xiterv[1] = period_x elif posx[j] + hsml[j] > x_max: xiter[1] = -1 xiterv[1] = -period_x + if periodic[1] == 1: if posy[j] - hsml[j] < y_min: yiter[1] = +1 yiterv[1] = period_y elif posy[j] + hsml[j] > y_max: yiter[1] = -1 yiterv[1] = -period_y + if periodic[2] == 1: if posz[j] - hsml[j] < z_min: ziter[1] = +1 ziterv[1] = period_z @@ -1743,8 +1767,8 @@ def pixelize_sph_kernel_arbitrary_grid(np.float64_t[:, :, :] buff, ziter[1] = -1 ziterv[1] = -period_z - h_j3 = fmax(hsml[j]*hsml[j]*hsml[j], dx*dy*dz) - h_j = math.cbrt(h_j3) + #h_j3 = fmax(hsml[j]*hsml[j]*hsml[j], dx*dy*dz) + h_j = hsml[j] #math.cbrt(h_j3) h_j2 = h_j*h_j ih_j = 1/h_j @@ -1805,11 +1829,17 @@ def pixelize_sph_kernel_arbitrary_grid(np.float64_t[:, :, :] buff, continue # see equation 4 of the SPLASH paper - q_ij = math.sqrt(posx_diff + posy_diff + posz_diff) * ih_j + q_ij = math.sqrt(posx_diff + + posy_diff + + posz_diff) * ih_j if q_ij >= 1: continue - - buff[xi, yi, zi] += prefactor_j * kernel(q_ij) + # shared variable buff should not + # be mutatated in a nogil section + # where different threads may change + # the same array element + buff[xi, yi, zi] += prefactor_j \ + * kernel(q_ij) def pixelize_element_mesh_line(np.ndarray[np.float64_t, ndim=2] coords, From 222500ed66c77c972754462324d328c46e264cdc Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Tue, 16 Jul 2024 20:02:46 -0500 Subject: [PATCH 106/186] minor documentation updates --- doc/source/analyzing/generating_processed_data.rst | 12 ++++++++---- doc/source/visualizing/plots.rst | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/doc/source/analyzing/generating_processed_data.rst b/doc/source/analyzing/generating_processed_data.rst index 84abf87eb7..5f7866b4ca 100644 --- a/doc/source/analyzing/generating_processed_data.rst +++ b/doc/source/analyzing/generating_processed_data.rst @@ -54,7 +54,8 @@ the transformation of a variable mesh of points consisting of positions and sizes into a fixed-size array that appears like an image. This process is that of pixelization, which yt handles transparently internally. You can access this functionality by constructing a -:class:`~yt.visualization.fixed_resolution.FixedResolutionBuffer` and supplying +:class:`~yt.visualization.fixed_resolution.FixedResolutionBuffer` +and supplying to it your :class:`~yt.data_objects.data_containers.YTSelectionContainer2D` object, as well as some information about how you want the final image to look. You can specify both the bounds of the image (in the appropriate x-y plane) and @@ -62,8 +63,8 @@ the resolution of the output image. You can then have yt pixelize any field you like. .. note:: In previous versions of yt, there was a special class of - FixedResolutionBuffer for off-axis slices. This is no longer - necessary. + FixedResolutionBuffer for off-axis slices. This is still used + for off-axis SPH data projections: OffAxisFixedResolutionBuffer. To create :class:`~yt.data_objects.data_containers.YTSelectionContainer2D` objects, you can access them as described in :ref:`data-objects`, specifically the section @@ -99,7 +100,10 @@ this, see :ref:`saving-grid-data-containers`. In the FITS case, there is an option for setting the ``units`` of the coordinate system in the file. If you want to overwrite a file with the same name, set ``clobber=True``. -The :class:`~yt.visualization.fixed_resolution.FixedResolutionBuffer` can even be exported +The :class:`~yt.visualization.fixed_resolution.FixedResolutionBuffer` +(and its +:class:`~yt.visualization.fixed_resolution.OffAxisProjectionFixedResolutionBuffer` +subclass) can even be exported as a 2D dataset itself, which may be operated on in the same way as any other dataset in yt: .. code-block:: python diff --git a/doc/source/visualizing/plots.rst b/doc/source/visualizing/plots.rst index ec7b6109a5..1cc9d3572b 100644 --- a/doc/source/visualizing/plots.rst +++ b/doc/source/visualizing/plots.rst @@ -293,8 +293,8 @@ argument. Optionally, a ``north_vector`` can be specified to fix the orientation of the image plane. .. note:: Not every data types have support for off-axis slices yet. - Currently, this operation is supported for grid based data with cartesian geometry. - In some cases (like SPH data) an off-axis projection over a thin region might be used instead. + Currently, this operation is supported for grid based and SPH data with cartesian geometry. + In some cases an off-axis projection over a thin region might be used instead. .. _projection-plots: From f5c476ad55ee1df9a16f6d924f92261e18bea7b0 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 17 Jul 2024 13:44:09 -0500 Subject: [PATCH 107/186] doc updates I forgot to save --- doc/source/quickstart/4)_Data_Objects_and_Time_Series.ipynb | 6 ++++-- doc/source/visualizing/plots.rst | 6 ++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/source/quickstart/4)_Data_Objects_and_Time_Series.ipynb b/doc/source/quickstart/4)_Data_Objects_and_Time_Series.ipynb index 49592f19f1..351719dce7 100644 --- a/doc/source/quickstart/4)_Data_Objects_and_Time_Series.ipynb +++ b/doc/source/quickstart/4)_Data_Objects_and_Time_Series.ipynb @@ -337,6 +337,8 @@ "\n", "There are two different types of covering grids: unsmoothed and smoothed. Smoothed grids will be filled through a cascading interpolation process; they will be filled at level 0, interpolated to level 1, filled at level 1, interpolated to level 2, filled at level 2, etc. This will help to reduce edge effects. Unsmoothed covering grids will not be interpolated, but rather values will be duplicated multiple times.\n", "\n", + "For SPH datasets, the covering grid gives the SPH-interpolated value of a field at each grid cell center. This is done for unsmoothed grids; smoothed grids are not available for SPH data.\n", + "\n", "Here we create an unsmoothed covering grid at level 2, with the left edge at `[0.0, 0.0, 0.0]` and with dimensions equal to those that would cover the entire domain at level 2. We can then ask for the Density field, which will be a 3D array." ] }, @@ -385,13 +387,13 @@ ], "metadata": { "kernelspec": { - "name": "python3", "display_name": "Python 3.9.5 64-bit ('yt-dev': pyenv)", "metadata": { "interpreter": { "hash": "14363bd97bed451d1329fb3e06aa057a9e955a9421c5343dd7530f5497723a41" } - } + }, + "name": "python3" }, "language_info": { "codemirror_mode": { diff --git a/doc/source/visualizing/plots.rst b/doc/source/visualizing/plots.rst index 1cc9d3572b..72e5aac758 100644 --- a/doc/source/visualizing/plots.rst +++ b/doc/source/visualizing/plots.rst @@ -433,6 +433,8 @@ by applying the In this use case, the volume renderer casts a set of plane parallel rays, one for each pixel in the image. The data values along each ray are summed, creating the final image buffer. +For SPH datsets, the coordinates are instead simply rotated before the axis-aligned +projection function is applied. .. _off-axis-projection-function: @@ -652,10 +654,6 @@ simply pass ``all`` as the first argument of the field tuple: Additional Notes for Plotting Particle Data ------------------------------------------- -An important caveat when visualizing particle data is that off-axis slice plotting is -not available for any particle data. However, axis-aligned slice plots (as described in -:ref:`slice-plots`) will work. - Since version 4.2.0, off-axis projections ares supported for non-SPH particle data. Previous to that, this operation was only supported for SPH particles. Two historical workaround methods were available for plotting non-SPH particles with off-axis From 7d8b676ae70a1ff3764d36af9beaa09a2d3d97e3 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 17 Jul 2024 13:44:37 -0500 Subject: [PATCH 108/186] fix ruff comment: not declaring unused h_j3 --- yt/utilities/lib/pixelization_routines.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yt/utilities/lib/pixelization_routines.pyx b/yt/utilities/lib/pixelization_routines.pyx index 5752944f81..7469d492af 100644 --- a/yt/utilities/lib/pixelization_routines.pyx +++ b/yt/utilities/lib/pixelization_routines.pyx @@ -1675,7 +1675,8 @@ def pixelize_sph_kernel_arbitrary_grid(np.float64_t[:, :, :] buff, cdef np.float64_t x_min, x_max, y_min, y_max, z_min, z_max, prefactor_j cdef np.int64_t xi, yi, zi, x0, x1, y0, y1, z0, z1 cdef np.float64_t q_ij, posx_diff, posy_diff, posz_diff, px, py, pz - cdef np.float64_t x, y, z, dx, dy, dz, idx, idy, idz, h_j3, h_j2, h_j, ih_j + cdef np.float64_t x, y, z, dx, dy, dz, idx, idy, idz, h_j2, h_j, ih_j + # cdef np.float64_t h_j3 cdef int j, ii, jj, kk cdef np.float64_t period_x = 0, period_y = 0, period_z = 0 From 170e9af1c5a7269ef125cadf4e67863495313188 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:33:04 -0500 Subject: [PATCH 109/186] fix test_offaxis_moment failure --- yt/visualization/plot_window.py | 5 ++- .../tests/test_offaxisprojection.py | 34 +++++++++++++++---- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/yt/visualization/plot_window.py b/yt/visualization/plot_window.py index 2fdc46edc7..68db155ad1 100644 --- a/yt/visualization/plot_window.py +++ b/yt/visualization/plot_window.py @@ -2490,9 +2490,8 @@ def __init__( particle_datasets = (ParticleDataset, StreamParticlesDataset) if isinstance(data_source.ds, particle_datasets) and is_sph_field: - center_use = center = parse_center_array(center, - ds=data_source.ds, - axis=None) + center_use = parse_center_array(center, ds=data_source.ds, + axis=None) else: center_use = center_rot fields = list(iter_fields(fields))[:] diff --git a/yt/visualization/tests/test_offaxisprojection.py b/yt/visualization/tests/test_offaxisprojection.py index d317e31d8b..e1c0fed5e9 100644 --- a/yt/visualization/tests/test_offaxisprojection.py +++ b/yt/visualization/tests/test_offaxisprojection.py @@ -235,10 +235,30 @@ def _vlos_sq(field, data): moment=2, buff_size=(400, 400), ) - assert_rel_equal( - np.sqrt( - p1.frb["gas", "velocity_los_squared"] - p1.frb["gas", "velocity_los"] ** 2 - ), - p2.frb["gas", "velocity_los"], - 10, - ) + ## this failed because some - **2 values come out + ## marginally < 0, resulting in unmatched NaN values in the + ## first assert_rel_equal argument. The compute_stddev_image + ## function used in OffAxisProjectionPlot checks for and deals + ## with these cases. + #assert_rel_equal( + # np.sqrt( + # p1.frb["gas", "velocity_los_squared"] - p1.frb["gas", "velocity_los"] ** 2 + # ), + # p2.frb["gas", "velocity_los"], + # 10, + #) + p1_expsq = p1.frb["gas", "velocity_los_squared"] + p1_sqexp = p1.frb["gas", "velocity_los"] ** 2 + p1res = np.sqrt(p1_expsq - p1_sqexp) + # set values to zero that have **2 - **2 < 0, but + # the absolute values are much smaller than the smallest + # postive values of **2 and **2 + # (i.e., the difference is pretty much zero) + mindiff = 1e-10 * min(np.min(p1_expsq[p1_expsq > 0]), + np.min(p1_sqexp[p1_sqexp > 0])) + print(mindiff) + setzero = np.logical_and(p1_expsq - p1_sqexp < 0, + p1_expsq - p1_sqexp > -1. * mindiff) + p1res[setzero] = 0. + p2res = p2.frb["gas", "velocity_los"] + assert_rel_equal(p1res, p2res, 10) \ No newline at end of file From 9b91e4fb0ce89f3c81ccccafdc9c149da7aee5d7 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:35:35 -0500 Subject: [PATCH 110/186] fix ruff comment end of file? --- yt/visualization/tests/test_offaxisprojection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yt/visualization/tests/test_offaxisprojection.py b/yt/visualization/tests/test_offaxisprojection.py index e1c0fed5e9..39299703d0 100644 --- a/yt/visualization/tests/test_offaxisprojection.py +++ b/yt/visualization/tests/test_offaxisprojection.py @@ -261,4 +261,5 @@ def _vlos_sq(field, data): p1_expsq - p1_sqexp > -1. * mindiff) p1res[setzero] = 0. p2res = p2.frb["gas", "velocity_los"] - assert_rel_equal(p1res, p2res, 10) \ No newline at end of file + assert_rel_equal(p1res, p2res, 10) + \ No newline at end of file From 71cc131d242a6ac63fc3c17a1a83168849cb0739 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 17 Jul 2024 16:09:53 -0500 Subject: [PATCH 111/186] draft 1 adding SPH pixelization tests (copied from separate file) --- .../tests/test_sph_pixelization.py | 618 +++++++++++++++++- yt/testing.py | 298 ++++++++- .../tests/test_offaxisprojection.py | 146 ++++- 3 files changed, 1058 insertions(+), 4 deletions(-) diff --git a/yt/geometry/coordinates/tests/test_sph_pixelization.py b/yt/geometry/coordinates/tests/test_sph_pixelization.py index 6de8b29f77..d42ca6de8f 100644 --- a/yt/geometry/coordinates/tests/test_sph_pixelization.py +++ b/yt/geometry/coordinates/tests/test_sph_pixelization.py @@ -1,7 +1,24 @@ +import pytest + +import numpy as np +import unyt + import yt -from yt.testing import assert_rel_equal, requires_file +from yt.data_objects.selection_objects.region import YTRegion +from yt.testing import ( + assert_rel_equal, + cubicspline_python, + distancematrix, + integrate_kernel, + fake_random_sph_ds, + fake_sph_flexible_grid_ds, + requires_file) from yt.utilities.math_utils import compute_stddev_image +## off-axis projection tests for SPH data are in +## yt/visualization/tests/test_offaxisprojection.py + + magneticum = "MagneticumCluster/snap_132" mag_kwargs = { @@ -36,3 +53,602 @@ def _vysq(field, data): ) sigy = compute_stddev_image(prj1.frb["gas", "vysq"], prj1.frb["gas", "velocity_y"]) assert_rel_equal(sigy, prj2.frb["gas", "velocity_y"].d, 10) + +def test_sph_projection_basic1(): + ''' + small, uniform grid: expected values for given dl? + pixel centers at 0.5, 1., 1.5, 2., 2.5 + particles at 0.5, 1.5, 2.5 + ''' + bbox = np.array([[0., 3.]] * 3) + ds = fake_sph_flexible_grid_ds(hsml_factor=1.0, nperside=3, + bbox=bbox) + # works, but no depth control (at least without specific filters) + proj = ds.proj(("gas", "density"), 2) + frb = proj.to_frb(width=(2.5, 'cm'), + resolution=(5, 5), + height=(2.5, 'cm'), + center=np.array([1.5, 1.5, 1.5]), + periodic=False) + out = frb.get_image(('gas', 'density')) + + expected_out = np.zeros((5, 5), dtype=np.float64) + dl_1part = integrate_kernel(cubicspline_python, 0., 0.5) + linedens_1part = dl_1part * 1. # unit mass, density + linedens = 3. * linedens_1part + expected_out[::2, ::2] = linedens + + assert_rel_equal(expected_out, out.v, 5) + #return out + +def test_sph_projection_basic2(): + ''' + small, uniform grid: expected values for given dl? + pixel centers at 0.5, 1., 1.5, 2., 2.5 + particles at 0.5, 1.5, 2.5 + but hsml radii are 0.25 -> try non-zero impact parameters, + other pixels are still zero. + ''' + bbox = np.array([[0., 3.]] * 3) + ds = fake_sph_flexible_grid_ds(hsml_factor=0.5, nperside=3, + bbox=bbox) + proj = ds.proj(("gas", "density"), 2) + frb = proj.to_frb(width=(2.5, 'cm'), + resolution=(5, 5), + height=(2.5, 'cm'), + center=np.array([1.375, 1.375, 1.5]), + periodic=False) + out = frb.get_image(('gas', 'density')) + + expected_out = np.zeros((5, 5), dtype=np.float64) + dl_1part = integrate_kernel(cubicspline_python, + np.sqrt(2) * 0.125, + 0.25) + linedens_1part = dl_1part * 1. # unit mass, density + linedens = 3. * linedens_1part + expected_out[::2, ::2] = linedens + + #print(expected_out) + #print(out.v) + assert_rel_equal(expected_out, out.v, 4) + #return out + +def get_dataset_sphrefine(reflevel: int = 1): + ''' + constant density particle grid, + with increasing particle sampling + ''' + lenfact = (1./3.)**(reflevel - 1) + massfact = lenfact**3 + nperside = 3**reflevel + + e1hat = np.array([lenfact, 0, 0]) + e2hat = np.array([0, lenfact, 0]) + e3hat = np.array([0, 0, lenfact]) + hsml_factor = lenfact + bbox = np.array([[0., 3.]] * 3) + offsets = np.ones(3, dtype=np.float64) * 0.5 # in units of ehat + + def refmass(i: int, j: int, k: int) -> float: + return massfact + unitrho = 1. / massfact # want density 1 for decreasing mass + + ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, + nperside=nperside, + periodic=True, + e1hat=e1hat, + e2hat=e2hat, + e3hat=e3hat, + offsets=offsets, + massgenerator=refmass, + unitrho=unitrho, + bbox=bbox, + ) + return ds + +def getdata_test_gridproj2(): + # initial pixel centers at 0.5, 1., 1.5, 2., 2.5 + # particles at 0.5, 1.5, 2.5 + # refine particle grid, check if pixel values remain the + # same in the pixels passing through initial particle centers + outlist = [] + dss = [] + for rl in range(1, 4): + ds = get_dataset_sphrefine(reflevel=rl) + proj = ds.proj(("gas", "density"), 2) + frb = proj.to_frb(width=(2.5, 'cm'), + resolution=(5, 5), + height=(2.5, 'cm'), + center=np.array([1.5, 1.5, 1.5]), + periodic=False) + out = frb.get_image(('gas', 'density')) + outlist.append(out) + dss.append(ds) + return outlist, dss + +def test_sph_gridproj_reseffect1(): + ''' + Comparing same pixel centers with higher particle resolution. + The pixel centers are at x/y coordinates [0.5, 1., 1.5, 2., 2.5] + at the first level, the spacing halves at each level. + Checking the pixels at [0.5, 1.5, 2.5], + which should have the same values at each resolution. + ''' + imgs, _ = getdata_test_gridproj2() + ref = imgs[-1] + for i, img in enumerate(imgs): + assert_rel_equal(img[::img.shape[0] // 2, ::img.shape[1] // 2], + ref[::ref.shape[0] // 2, ::ref.shape[1] // 2], 4) + +def test_sph_gridproj_reseffect2(): + ''' + refine the pixel grid instead of the particle grid + ''' + ds = get_dataset_sphrefine(reflevel=2) + proj = ds.proj(("gas", "density"), 2) + imgs = {} + maxrl = 5 + for rl in range(1, maxrl + 1): + npix = 1 + 2**(rl + 1) + margin = 0.5 - 0.5**(rl + 1) + frb = proj.to_frb(width=(3. - 2. * margin, 'cm'), + resolution=(npix, npix), + height=(3. - 2. * margin, 'cm'), + center=np.array([1.5, 1.5, 1.5]), + periodic=False) + out = frb.get_image(('gas', 'density')) + imgs[rl] = out + ref = imgs[maxrl] + pixspace_ref = 2**(maxrl) + for rl in imgs: + img = imgs[rl] + pixspace = 2**(rl) + #print(f'Grid refinement level {rl}:') + assert_rel_equal(img[::pixspace, ::pixspace], + ref[::pixspace_ref, ::pixspace_ref], 4) + + +@pytest.mark.parametrize("weighted", [True, False]) +@pytest.mark.parametrize("periodic", [True, False]) +@pytest.mark.parametrize("depth", [None, (1., "cm")]) +@pytest.mark.parametrize("shiftcenter", [False, True]) +@pytest.mark.parametrize("axis", [0, 1, 2]) +def test_sph_proj_general_alongaxes(axis: int, + shiftcenter: bool, + depth : float | None, + periodic: bool, + weighted: bool) -> None: + ''' + The previous projection tests were for a specific issue. + Here, we test more functionality of the projections. + We just send lines of sight through pixel centers for convenience. + Particles at [0.5, 1.5, 2.5] (in each coordinate) + smoothing lengths 0.25 + all particles have mass 1., density 1.5, + except the single center particle, with mass 2., density 3. + + Parameters: + ----------- + axis: {0, 1, 2} + projection axis (aligned with sim. axis) + shiftcenter: bool + shift the coordinates to center the projection on. + (The grid is offset to this same center) + depth: float or None + depth of the projection slice + periodic: bool + assume periodic boundary conditions, or not + weighted: bool + make a weighted projection (density-weighted density), or not + + Returns: + -------- + None + ''' + if shiftcenter: + center = unyt.unyt_array(np.array((0.625, 0.625, 0.625)), 'cm') + else: + center = unyt.unyt_array(np.array((1.5, 1.5, 1.5)), 'cm') + bbox = unyt.unyt_array(np.array([[0., 3.], [0., 3.], [0., 3.]]), 'cm') + hsml_factor = 0.5 + unitrho = 1.5 + # test correct centering, particle selection + def makemasses(i, j, k): + if i == j == k == 1: + return 2. + else: + return 1. + # m / rho, factor 1. / hsml**2 is included in the kernel integral + # (density is adjusted, so same for center particle) + prefactor = 1. / unitrho #/ (0.5 * 0.5)**2 + dl_cen = integrate_kernel(cubicspline_python, 0., 0.25) + + # result shouldn't depend explicitly on the center if we re-center + # the data, unless we get cut-offs in the non-periodic case + ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, + periodic=periodic, + offsets=np.full(3, 0.5), + massgenerator=makemasses, + unitrho=unitrho, + bbox=bbox.v, + recenter=center.v) + if depth is None: + source = ds.all_data() + else: + depth = unyt.unyt_quantity(*depth) + le = np.array(ds.domain_left_edge) + re = np.array(ds.domain_right_edge) + le[axis] = center[axis] - 0.5 * depth + re[axis] = center[axis] + 0.5 * depth + cen = 0.5 * (le + re) + reg = YTRegion(center=cen, + left_edge=le, + right_edge=re, + ds=ds) + source = reg + + # we don't actually want a plot, it's just a straightforward, + # common way to get an frb / image array + if weighted: + toweight_field = ("gas", "density") + else: + toweight_field = None + prj = yt.ProjectionPlot(ds, axis, ("gas", "density"), + width=(2.5, "cm"), + weight_field=toweight_field, + buff_size=(5, 5), + center=center, + data_source=source) + img = prj.frb.data[('gas', 'density')] + if weighted: + expected_out = np.zeros((5, 5,), dtype=img.v.dtype) + expected_out[::2, ::2] = unitrho + if depth is None: + ## during shift, particle coords do wrap around edges + #if (not periodic) and shiftcenter: + # # weight 1. for unitrho, 2. for 2. * untrho + # expected_out[2, 2] *= 5. / 3. + #else: + # weight (2 * 1.) for unitrho, (1 * 2.) for 2. * unitrho + expected_out[2, 2] *= 1.5 + else: + # only 2 * unitrho element included + expected_out[2, 2] *= 2. + else: + expected_out = np.zeros((5, 5,), dtype=img.v.dtype) + expected_out[::2, ::2] = dl_cen * prefactor * unitrho + if depth is None: + # 3 particles per l.o.s., including the denser one + expected_out *= 3. + expected_out[2, 2] *= 4. / 3. + else: + # 1 particle per l.o.s., including the denser one + expected_out[2, 2] *= 2. + # grid is shifted to the left -> 'missing' stuff at the left + if (not periodic) and shiftcenter: + expected_out[:1, :] = 0. + expected_out[:, :1] = 0. + #print(axis, shiftcenter, depth, periodic, weighted) + #print(expected_out) + #print(img.v) + assert_rel_equal(expected_out, img.v, 5) + +@pytest.mark.parametrize("periodic", [True, False]) +@pytest.mark.parametrize("shiftcenter", [False, True]) +@pytest.mark.parametrize("zoff", [0., 0.1, 0.5, 1.]) +@pytest.mark.parametrize("axis", [0, 1, 2]) +def test_sph_slice_general_alongaxes(axis: int, + shiftcenter: bool, + periodic: bool, + zoff: float) -> None: + ''' + Particles at [0.5, 1.5, 2.5] (in each coordinate) + smoothing lengths 0.25 + all particles have mass 1., density 1.5, + except the single center particle, with mass 2., density 3. + + Parameters: + ----------- + axis: {0, 1, 2} + projection axis (aligned with sim. axis) + northvector: tuple + y-axis direction in the final plot (direction vector) + shiftcenter: bool + shift the coordinates to center the projection on. + (The grid is offset to this same center) + zoff: float + offset of the slice plane from the SPH particle center plane + periodic: bool + assume periodic boundary conditions, or not + + Returns: + -------- + None + ''' + if shiftcenter: + center = unyt.unyt_array(np.array((0.625, 0.625, 0.625)), 'cm') + else: + center = unyt.unyt_array(np.array((1.5, 1.5, 1.5)), 'cm') + bbox = unyt.unyt_array(np.array([[0., 3.], [0., 3.], [0., 3.]]), 'cm') + hsml_factor = 0.5 + unitrho = 1.5 + # test correct centering, particle selection + def makemasses(i, j, k): + if i == j == k == 1: + return 2. + elif i == j == k == 2: + return 3. + else: + return 1. + + # result shouldn't depend explicitly on the center if we re-center + # the data, unless we get cut-offs in the non-periodic case + ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, + periodic=periodic, + offsets=np.full(3, 0.5), + massgenerator=makemasses, + unitrho=unitrho, + bbox=bbox.v, + recenter=center.v) + ad = ds.all_data() + #print(ad[('gas', 'position')]) + outgridsize = 10 + width = 2.5 + _center = center.to("cm").v.copy() + _center[axis] += zoff + + # we don't actually want a plot, it's just a straightforward, + # common way to get an frb / image array + slc = yt.SlicePlot(ds, axis, ("gas", "density"), + width=(width, "cm"), + buff_size=(outgridsize,) * 2, + center=(_center, "cm")) + img = slc.frb.data[('gas', 'density')] + + # center is same in non-projection coords + if axis == 0: + ci = 1 + else: + ci = 0 + gridcens = _center[ci] - 0.5 * width \ + + 0.5 * width / outgridsize \ + + np.arange(outgridsize) * width / outgridsize + xgrid = np.repeat(gridcens, outgridsize) + ygrid = np.tile(gridcens, outgridsize) + zgrid = np.full(outgridsize**2, _center[axis]) + gridcoords = np.empty((outgridsize**2, 3), dtype=xgrid.dtype) + if axis == 2: + gridcoords[:, 0] = xgrid + gridcoords[:, 1] = ygrid + gridcoords[:, 2] = zgrid + elif axis == 0: + gridcoords[:, 0] = zgrid + gridcoords[:, 1] = xgrid + gridcoords[:, 2] = ygrid + elif axis == 1: + gridcoords[:, 0] = ygrid + gridcoords[:, 1] = zgrid + gridcoords[:, 2] = xgrid + ad = ds.all_data() + sphcoords = np.array([(ad[("gas", "x")]).to("cm"), + (ad[("gas", "y")]).to("cm"), + (ad[("gas", "z")]).to("cm"), + ]).T + print('sphcoords:') + print(sphcoords) + print('gridcoords:') + print(gridcoords) + dists = distancematrix(gridcoords, sphcoords, + periodic=(periodic,)*3, + periods=np.array([3., 3., 3.])) + print('dists <= 1:') + print(dists <= 1) + sml = (ad[("gas", "smoothing_length")]).to("cm") + normkern = cubicspline_python(dists / sml.v[np.newaxis, :]) + sphcontr = normkern / sml[np.newaxis, :]**3 * ad[("gas", "mass")] + contsum = np.sum(sphcontr, axis=1) + sphweights = normkern / sml[np.newaxis, :]**3 \ + * ad[("gas", "mass")] / ad[("gas", "density")] + weights = np.sum(sphweights, axis=1) + expected = contsum / weights + expected = expected.reshape((outgridsize, outgridsize)) + expected[np.isnan(expected)] = 0. # convention in the slices + + print('expected:\n', expected.v) + print('recovered:\n', img.v) + assert_rel_equal(expected.v, img.v, 5) + + + +@pytest.mark.parametrize("periodic", [True, False]) +@pytest.mark.parametrize("shiftcenter", [False, True]) +@pytest.mark.parametrize("northvector", [None, (1.e-4, 1., 0.)]) +@pytest.mark.parametrize("zoff", [0., 0.1, 0.5, 1.]) +def test_sph_slice_general_offaxis( + northvector: tuple[float, float, float] | None, + shiftcenter: bool, + zoff: float, + periodic: bool, + ) -> None: + ''' + Same as the on-axis slices, but we rotate the basis vectors + to test whether roations are handled ok. the rotation is chosen to + be small so that in/exclusion of particles within bboxes, etc. + works out the same way. + Particles at [0.5, 1.5, 2.5] (in each coordinate) + smoothing lengths 0.25 + all particles have mass 1., density 1.5, + except the single center particle, with mass 2., density 3. + + Parameters: + ----------- + northvector: tuple + y-axis direction in the final plot (direction vector) + shiftcenter: bool + shift the coordinates to center the projection on. + (The grid is offset to this same center) + zoff: float + offset of the slice plane from the SPH particle center plane + periodic: bool + assume periodic boundary conditions, or not + + Returns: + -------- + None + ''' + if shiftcenter: + center = np.array((0.625, 0.625, 0.625)) # cm + else: + center = np.array((1.5, 1.5, 1.5)) # cm + bbox = unyt.unyt_array(np.array([[0., 3.], [0., 3.], [0., 3.]]), 'cm') + hsml_factor = 0.5 + unitrho = 1.5 + # test correct centering, particle selection + def makemasses(i, j, k): + if i == j == k == 1: + return 2. + else: + return 1. + # try to make sure dl differences from periodic wrapping are small + epsilon = 1e-4 + projaxis = np.array([epsilon, 0.00, np.sqrt(1. - epsilon**2)]) + e1dir = projaxis / np.sqrt(np.sum(projaxis**2)) + if northvector is None: + e2dir = np.array([0., 1., 0.]) + else: + e2dir = np.asarray(northvector) + e2dir = e2dir - np.sum(e1dir * e2dir) * e2dir # orthonormalize + e2dir /= np.sqrt(np.sum(e2dir**2)) + e3dir = np.cross(e2dir, e1dir) + + outgridsize = 10 + width = 2.5 + _center = center.copy() + _center += zoff * e1dir + + ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, + periodic=periodic, + offsets=np.full(3, 0.5), + massgenerator=makemasses, + unitrho=unitrho, + bbox=bbox.v, + recenter=center, + e1hat=e1dir, e2hat=e2dir, e3hat=e3dir) + + source = ds.all_data() + # couple to dataset -> right unit registry + center = ds.arr(center, 'cm') + print('position:\n', source['gas','position']) + slc = yt.SlicePlot(ds, e1dir, ("gas", "density"), + width=(width, "cm"), + buff_size=(outgridsize,) * 2, + center=(_center, "cm"), + north_vector=e2dir) + img = slc.frb.data[('gas', 'density')] + + # center is same in x/y (e3dir/e2dir) + gridcenx = np.dot(_center, e3dir) - 0.5 * width \ + + 0.5 * width / outgridsize \ + + np.arange(outgridsize) * width / outgridsize + gridceny = np.dot(_center, e2dir) - 0.5 * width \ + + 0.5 * width / outgridsize \ + + np.arange(outgridsize) * width / outgridsize + xgrid = np.repeat(gridcenx, outgridsize) + ygrid = np.tile(gridceny, outgridsize) + zgrid = np.full(outgridsize**2, np.dot(_center, e1dir)) + gridcoords = (xgrid[:, np.newaxis] * e3dir[np.newaxis, :] + + ygrid[:, np.newaxis] * e2dir[np.newaxis, :] + + zgrid[:, np.newaxis] * e1dir[np.newaxis, :]) + print('gridcoords:') + print(gridcoords) + ad = ds.all_data() + sphcoords = np.array([(ad[("gas", "x")]).to("cm"), + (ad[("gas", "y")]).to("cm"), + (ad[("gas", "z")]).to("cm"), + ]).T + dists = distancematrix(gridcoords, sphcoords, + periodic=(periodic,)*3, + periods=np.array([3., 3., 3.])) + sml = (ad[("gas", "smoothing_length")]).to("cm") + normkern = cubicspline_python(dists / sml.v[np.newaxis, :]) + sphcontr = normkern / sml[np.newaxis, :]**3 * ad[("gas", "mass")] + contsum = np.sum(sphcontr, axis=1) + sphweights = normkern / sml[np.newaxis, :]**3 \ + * ad[("gas", "mass")] / ad[("gas", "density")] + weights = np.sum(sphweights, axis=1) + expected = contsum / weights + expected = expected.reshape((outgridsize, outgridsize)) + expected = expected.T # transposed for image plotting + expected[np.isnan(expected)] = 0. # convention in the slices + + #print(axis, shiftcenter, depth, periodic, weighted) + print('expected:\n', expected.v) + print('recovered:\n', img.v) + assert_rel_equal(expected.v, img.v, 4) + +# only axis-aligned; testing YTArbitraryGrid, YTCoveringGrid +@pytest.mark.parametrize("periodic", [True, False, (True, True, False)]) +@pytest.mark.parametrize("wholebox", [True, False]) +def test_sph_grid(periodic: bool | tuple[bool, bool, bool], + wholebox: bool): + bbox = np.array([[-1., 3.], [1., 5.2], [-1., 3.]]) + ds = fake_random_sph_ds(50, bbox, periodic=periodic) + + if not hasattr(periodic, "__len__"): + periodic = (periodic,) * 3 + + if wholebox: + left = bbox[:, 0].copy() + level = 2 + ncells = np.array([2**level] * 3) + print('left: ', left) + print('ncells: ', ncells) + resgrid = ds.covering_grid(level, tuple(left), ncells) + right = bbox[:, 1].copy() + xedges = np.linspace(left[0], right[0], ncells[0] + 1) + yedges = np.linspace(left[1], right[1], ncells[1] + 1) + zedges = np.linspace(left[2], right[2], ncells[2] + 1) + else: + left = np.array([-1., 1.8, -1.]) + right = np.array([2.5, 5.2, 2.5]) + ncells = np.array([3, 4, 4]) + resgrid = ds.arbitrary_grid(left, right, dims=ncells) + xedges = np.linspace(left[0], right[0], ncells[0] + 1) + yedges = np.linspace(left[1], right[1], ncells[1] + 1) + zedges = np.linspace(left[2], right[2], ncells[2] + 1) + res = resgrid["gas", "density"] + xcens = 0.5 * (xedges[:-1] + xedges[1:]) + ycens = 0.5 * (yedges[:-1] + yedges[1:]) + zcens = 0.5 * (zedges[:-1] + zedges[1:]) + + ad = ds.all_data() + sphcoords = np.array([(ad[("gas", "x")]).to("cm"), + (ad[("gas", "y")]).to("cm"), + (ad[("gas", "z")]).to("cm"), + ]).T + gridx, gridy, gridz = np.meshgrid(xcens, ycens, zcens, + indexing='ij') + outshape = gridx.shape + gridx = gridx.flatten() + gridy = gridy.flatten() + gridz = gridz.flatten() + gridcoords = np.array([gridx, gridy, gridz]).T + periods = bbox[:, 1] - bbox[:, 0] + dists = distancematrix(gridcoords, sphcoords, + periodic=periodic, + periods=periods) + sml = (ad[("gas", "smoothing_length")]).to("cm") + normkern = cubicspline_python(dists / sml.v[np.newaxis, :]) + sphcontr = normkern / sml[np.newaxis, :]**3 * ad[("gas", "mass")] + contsum = np.sum(sphcontr, axis=1) + sphweights = normkern / sml[np.newaxis, :]**3 \ + * ad[("gas", "mass")] / ad[("gas", "density")] + weights = np.sum(sphweights, axis=1) + expected = contsum / weights + expected = expected.reshape(outshape) + expected[np.isnan(expected)] = 0. # convention in the slices + + #print(axis, shiftcenter, depth, periodic, weighted) + print('expected:\n', expected.v) + print('recovered:\n', res.v) + assert_rel_equal(expected.v, res.v, 4) diff --git a/yt/testing.py b/yt/testing.py index 2205439ea1..d519f32c79 100644 --- a/yt/testing.py +++ b/yt/testing.py @@ -9,6 +9,7 @@ from functools import wraps from importlib.util import find_spec from shutil import which +from typing import Callable from unittest import SkipTest import matplotlib @@ -19,10 +20,13 @@ from yt._maintenance.deprecation import issue_deprecation_warning from yt.config import ytcfg +from yt.frontends.stream.data_structures import StreamParticlesDataset from yt.funcs import is_sequence -from yt.loaders import load +from yt.loaders import load, load_particles from yt.units.yt_array import YTArray, YTQuantity + + ANSWER_TEST_TAG = "answer_test" @@ -79,6 +83,101 @@ def assert_rel_equal(a1, a2, decimals, err_msg="", verbose=True): np.array(a1) / np.array(a2), 1.0, decimals, err_msg=err_msg, verbose=verbose ) +# tested: volume integral is 1. +def cubicspline_python(x: float) -> float: + ''' + cubic spline SPH kernel function for testing against more + effiecient cython methods + + Parameters + ---------- + x: + impact parameter / smoothing length [dimenionless] + + Returns + ------- + value of the kernel function + ''' + # C is 8/pi + _c = 8. / np.pi + x = np.asarray(x) + kernel = np.zeros(x.shape, dtype=x.dtype) + half1 = np.where(np.logical_and(x >=0., x <= 0.5)) + kernel[half1] = 1. - 6. * x[half1]**2 * (1. - x[half1]) + half2 = np.where(np.logical_and(x > 0.5, x <= 1.0)) + kernel[half2] = 2. * (1. - x[half2])**3 + return kernel * _c + +def integrate_kernel(kernelfunc: Callable[[float], float], + b: float, hsml: float) -> float: + """ + integrates a kernel function over a line passing entirely + through it + + Parameters: + ----------- + kernelfunc: + the kernel function to integrate + b: + impact parameter + hsml: + smoothing length [same units as impact parameter] + + Returns: + -------- + the integral of the SPH kernel function. + units: 1 / units of b and hsml + """ + pre = 1. / hsml**2 + x = b / hsml + xmax = np.sqrt(1. - x**2) + xmin = -1. * xmax + xe = np.linspace(xmin, xmax, 500) # shape: 500, x.shape + xc = 0.5 * (xe[:-1, ...] + xe[1:, ...]) + dx = np.diff(xe, axis=0) + spv = kernelfunc(np.sqrt(xc**2 + x**2)) + integral = np.sum(spv * dx, axis=0) + return pre * integral + +def distancematrix(pos3_i0: np.ndarray[float], + pos3_i1: np.ndarray[float], + periodic: tuple[bool] = (True,) * 3, + periods: np.ndarray = np.array([0., 0., 0.]), + ) -> np.ndarray[float]: + ''' + Calculates the distances between two arrays of points. + + Parameters: + ---------- + pos3_i0: shape (first number of points, 3) + positions of the first set of points. The second index is + for positions along the different cartesian axes + pos3_i1: shape (second number of points, 3) + as pos3_i0, but for the second set of points + periodic: + are the positions along each axis periodic (True) or not + periods: + the periods along each axis. Ignored if positions in a given + direction are not periodic. + + Returns: + -------- + a 2D-array of distances between postions `pos3_i0` (changes along + index 0) and `pos3_i1` (changes along index 1) + + ''' + d2 = np.zeros((len(pos3_i0), len(pos3_i1)), dtype=pos3_i0.dtype) + for ax in range(3): + # 'center on' pos3_i1 + _d = pos3_i0[:, ax, np.newaxis] - pos3_i1[np.newaxis, :, ax] + if periodic[ax]: + _period = periods[ax] + _d += 0.5 * _period # center on half box size + _d %= _period # wrap coordinate to 0 -- boxsize range + _d -= 0.5 * _period # center back to zero + d2 += _d**2 + return np.sqrt(d2) + def amrspace(extent, levels=7, cells=8): """Creates two numpy arrays representing the left and right bounds of @@ -686,6 +785,203 @@ def fake_sph_grid_ds(hsml_factor=1.0): return load_particles(data=data, length_unit=1.0, bbox=bbox) +def constantmass(i: int, j: int, k: int) -> float: + return 1. + +def fake_sph_flexible_grid_ds( + hsml_factor: float = 1.0, + nperside: int = 3, + periodic: bool = True, + e1hat: np.ndarray[float] = np.array([1, 0, 0]), + e2hat: np.ndarray[float] = np.array([0, 1, 0]), + e3hat: np.ndarray[float] = np.array([0, 0, 1]), + offsets: np.ndarray[float] = 0.5 * np.ones((3,), dtype=np.float64), + massgenerator: Callable[[int, int, int], float] = constantmass, + unitrho: float = 1., + bbox: np.ndarray | None = None, + recenter: np.ndarray | None = None, + ) -> StreamParticlesDataset: + """Returns an in-memory SPH dataset useful for testing + + Parameters: + ----------- + hsml_factor: + all particles have smoothing lengths of `hsml_factor` * 0.5 + nperside: + the dataset will have `nperside`**3 particles, arranged + uniformly on a 3D grid + periodic: + are the positions taken to be periodic? (applies to all + coordinate axes) + e1hat: shape (3,) + the first basis vector defining the 3D grid. If the basis + vectors are not normalized to 1 or not orthogonal, the spacing + or overlap between SPH particles will be affected, but this is + allowed. + e2hat: shape (3,) + the second basis vector defining the 3D grid. (See `e1hat`.) + e3hat: shape (3,) + the third basis vector defining the 3D grid. (See `e1hat`.) + offsets: shape (3,) + the the zero point of the 3D grid along each coordinate axis + massgenerator: + a function assigning a mass to each particle, as a function of + the e[1-3]hat indices, in order + unitrho: + defines the density for a particle with mass 1 ('g'), and the + standard (uniform) grid `hsml_factor`. + bbox: if np.ndarray, shape is (2, 3) + the assumed enclosing volume of the particles. Should enclose + all the coordinate values. If not specified, a bbox is defined + which encloses all coordinates values with a margin. If + `periodic`, the size of the `bbox` along each coordinate is + also the period along that axis. + recenter: + if not `None`, after generating the grid, the positions are + periodically shifted to move the old center to this positions. + Useful for testing periodicity handling. + This shift is relative to the halfway positions of the bbox + edges. + + Returns: + -------- + A `StreamParticlesDataset` object with particle positions, masses, + velocities (zero), smoothing lengths, and densities specified. + Values are in cgs units. + """ + + npart = nperside**3 + + pos = np.empty((npart, 3), dtype=np.float64) + mass = np.empty((npart,), dtype=np.float64) + for i in range(0, nperside): + for j in range(0, nperside): + for k in range(0, nperside): + _pos = (offsets[0] + i) * e1hat \ + + (offsets[1] + j) * e2hat \ + + (offsets[2] + k) * e3hat + ind = nperside**2 * i + nperside * j + k + pos[ind, :] = _pos + mass[ind] = massgenerator(i, j, k) + rho = unitrho * mass + + if bbox is None: + eps = 1e-3 + margin = (1. + eps) * hsml_factor + bbox = np.array([[np.min(pos[:, 0]) - margin, + np.max(pos[:, 0]) + margin], + [np.min(pos[:, 1]) - margin, + np.max(pos[:, 1]) + margin], + [np.min(pos[:, 2]) - margin, + np.max(pos[:, 2]) + margin], + ]) + + if recenter is not None: + periods = bbox[:, 1] - bbox[:, 0] + # old center -> new position + pos += -0.5 * periods[np.newaxis, :] + recenter[np.newaxis, :] + # wrap coordinates -> all in [0, boxsize) range + pos %= periods[np.newaxis, :] + # shift back to original bbox range + pos += (bbox[:, 0])[np.newaxis, :] + if not periodic: + # remove points outside bbox to avoid errors: + okinds = np.ones(len(mass), dtype=bool) + for ax in [0, 1, 2]: + okinds &= pos[:, ax] < bbox[ax, 1] + okinds &= pos[:, ax] >= bbox[ax, 0] + npart = sum(okinds) + else: + okinds = slice(None, None, None) + + data = { + "particle_position_x": (np.copy(pos[okinds, 0]), "cm"), + "particle_position_y": (np.copy(pos[okinds, 1]), "cm"), + "particle_position_z": (np.copy(pos[okinds, 2]), "cm"), + "particle_mass": (mass[okinds], "g"), + "particle_velocity_x": (np.zeros(npart), "cm/s"), + "particle_velocity_y": (np.zeros(npart), "cm/s"), + "particle_velocity_z": (np.zeros(npart), "cm/s"), + "smoothing_length": (np.ones(npart) * 0.5 * hsml_factor, "cm"), + "density": (rho[okinds], "g/cm**3"), + } + + ds = load_particles(data=data, + bbox=bbox, periodicity=(periodic,) * 3, + length_unit=1., mass_unit=1., time_unit=1., + velocity_unit=1.) + ds.kernel_name = 'cubic' + return ds + + +def fake_random_sph_ds(npart: int, bbox: np.ndarray, + periodic: bool | tuple[bool, bool, bool] = True, + massrange: tuple[float, float] = (0.5, 2.), + hsmlrange: tuple[float, float] = (0.5, 2.), + unitrho: float = 1., + ) -> StreamParticlesDataset: + """Returns an in-memory SPH dataset useful for testing + + Parameters: + ----------- + npart: + number of particles to generate + bbox: shape: (3, 2), units: "cm" + the assumed enclosing volume of the particles. Particle + positions are drawn uniformly from these ranges. + periodic: + are the positions taken to be periodic? If a single value, + that value is applied to all axes + massrange: + particle masses are drawn uniformly from this range (unit: "g") + hsmlrange: units: "cm" + particle smoothing lengths are drawn uniformly from this range + unitrho: + defines the density for a particle with mass 1 ("g"), and + smoothing length 1 ("cm"). + + Returns: + -------- + A `StreamParticlesDataset` object with particle positions, masses, + velocities (zero), smoothing lengths, and densities specified. + Values are in cgs units. + """ + + if not hasattr(periodic, "__len__"): + periodic = (periodic, ) * 3 + + posx = np.random.uniform(low=bbox[0][0], high=bbox[0][1], + size=npart) + posy = np.random.uniform(low=bbox[1][0], high=bbox[1][1], + size=npart) + posz = np.random.uniform(low=bbox[2][0], high=bbox[2][1], + size=npart) + mass = np.random.uniform(low=massrange[0], high=massrange[1], + size=npart) + hsml = np.random.uniform(low=hsmlrange[0], high=hsmlrange[1], + size=npart) + dens = mass / hsml**3 * unitrho + + data = { + "particle_position_x": (posx, "cm"), + "particle_position_y": (posy, "cm"), + "particle_position_z": (posz, "cm"), + "particle_mass": (mass, "g"), + "particle_velocity_x": (np.zeros(npart), "cm/s"), + "particle_velocity_y": (np.zeros(npart), "cm/s"), + "particle_velocity_z": (np.zeros(npart), "cm/s"), + "smoothing_length": (hsml, "cm"), + "density": (dens, "g/cm**3"), + } + + ds = load_particles(data=data, + bbox=bbox, periodicity=periodic, + length_unit=1., mass_unit=1., time_unit=1., + velocity_unit=1.) + ds.kernel_name = 'cubic' + return ds + + def construct_octree_mask(prng=RandomState(0x1D3D3D3), refined=None): # noqa B008 # Implementation taken from url: # http://docs.hyperion-rt.org/en/stable/advanced/indepth_oct.html diff --git a/yt/visualization/tests/test_offaxisprojection.py b/yt/visualization/tests/test_offaxisprojection.py index 39299703d0..42b9b5faaa 100644 --- a/yt/visualization/tests/test_offaxisprojection.py +++ b/yt/visualization/tests/test_offaxisprojection.py @@ -1,19 +1,27 @@ import itertools as it import os +import pytest import shutil import tempfile import unittest import numpy as np from numpy.testing import assert_equal +import unyt from yt.testing import ( assert_fname, assert_rel_equal, + cubicspline_python, + integrate_kernel, fake_octree_ds, fake_random_ds, + fake_sph_flexible_grid_ds, ) -from yt.visualization.api import OffAxisProjectionPlot, OffAxisSlicePlot +from yt.visualization.api import ( + OffAxisProjectionPlot, + OffAxisSlicePlot, + ProjectionPlot) from yt.visualization.image_writer import write_projection from yt.visualization.volume_rendering.api import off_axis_projection @@ -262,4 +270,138 @@ def _vlos_sq(field, data): p1res[setzero] = 0. p2res = p2.frb["gas", "velocity_los"] assert_rel_equal(p1res, p2res, 10) - \ No newline at end of file + + +@pytest.mark.parametrize("weighted", [True, False]) +@pytest.mark.parametrize("periodic", [True, False]) +@pytest.mark.parametrize("depth", [None, (1., "cm"), (0.5, "cm")]) +@pytest.mark.parametrize("shiftcenter", [False, True]) +@pytest.mark.parametrize("northvector", [None, (1.e-4, 1., 0.)]) +def test_sph_proj_general_offaxis( + northvector: tuple[float, float, float] | None, + shiftcenter: bool, + depth: tuple[float, str] | None, + periodic: bool, + weighted: bool) -> None: + ''' + Same as the on-axis projections, but we rotate the basis vectors + to test whether roations are handled ok. the rotation is chosen to + be small so that in/exclusion of particles within bboxes, etc. + works out the same way. + We just send lines of sight through pixel centers for convenience. + Particles at [0.5, 1.5, 2.5] (in each coordinate) + smoothing lengths 0.25 + all particles have mass 1., density 1.5, + except the single center particle, with mass 2., density 3. + + Parameters: + ----------- + northvector: tuple + y-axis direction in the final plot (direction vector) + shiftcenter: bool + shift the coordinates to center the projection on. + (The grid is offset to this same center) + depth: float or None + depth of the projection slice + periodic: bool + assume periodic boundary conditions, or not + weighted: bool + make a weighted projection (density-weighted density), or not + + Returns: + -------- + None + ''' + if shiftcenter: + center = np.array((0.625, 0.625, 0.625)) # cm + else: + center = np.array((1.5, 1.5, 1.5)) # cm + bbox = unyt.unyt_array(np.array([[0., 3.], [0., 3.], [0., 3.]]), 'cm') + hsml_factor = 0.5 + unitrho = 1.5 + # test correct centering, particle selection + def makemasses(i, j, k): + if i == j == k == 1: + return 2. + else: + return 1. + #dl_shift = _integrate_kernel(_cubicspline_python, + # np.sqrt(2) * 0.125, 0.25) + + # result shouldn't depend explicitly on the center if we re-center + # the data, unless we get cut-offs in the non-periodic case + # *almost* the z-axis + # try to make sure dl differences from periodic wrapping are small + epsilon = 1e-4 + projaxis = np.array([epsilon, 0.00, np.sqrt(1. - epsilon**2)]) + e1dir = projaxis / np.sqrt(np.sum(projaxis**2)) + # TODO: figure out other (default) axes for basis vectors here + if northvector is None: + e2dir = np.array([0., 1., 0.]) + else: + e2dir = np.asarray(northvector) + e2dir = e2dir - np.sum(e1dir * e2dir) * e2dir # orthonormalize + e2dir /= np.sqrt(np.sum(e2dir**2)) + e3dir = np.cross(e1dir, e2dir) + + ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, + periodic=periodic, + offsets=np.full(3, 0.5), + massgenerator=makemasses, + unitrho=unitrho, + bbox=bbox.v, + recenter=center, + e1hat=e1dir, e2hat=e2dir, e3hat=e3dir) + + source = ds.all_data() + # couple to dataset -> right unit registry + center = ds.arr(center, 'cm') + #print('position:\n', source['gas','position']) + + # m / rho, factor 1. / hsml**2 is included in the kernel integral + # (density is adjusted, so same for center particle) + prefactor = 1. / unitrho #/ (0.5 * 0.5)**2 + dl_cen = integrate_kernel(cubicspline_python, 0., 0.25) + + if weighted: + toweight_field = ("gas", "density") + else: + toweight_field = None + # we don't actually want a plot, it's just a straightforward, + # common way to get an frb / image array + prj = ProjectionPlot(ds, projaxis, ("gas", "density"), + width=(2.5, "cm"), + weight_field=toweight_field, + buff_size=(5, 5), + center=center, + data_source=source, + north_vector=northvector, + depth=depth) + img = prj.frb.data[('gas', 'density')] + if weighted: + # periodic shifts will modify the (relative) dl values a bit + expected_out = np.zeros((5, 5,), dtype=img.v.dtype) + expected_out[::2, ::2] = unitrho + if depth is None: + expected_out[2, 2] *= 1.5 + else: + # only 2 * unitrho element included + expected_out[2, 2] *= 2. + else: + expected_out = np.zeros((5, 5,), dtype=img.v.dtype) + expected_out[::2, ::2] = dl_cen * prefactor * unitrho + if depth is None: + # 3 particles per l.o.s., including the denser one + expected_out *= 3. + expected_out[2, 2] *= 4. / 3. + else: + # 1 particle per l.o.s., including the denser one + expected_out[2, 2] *= 2. + # grid is shifted to the left -> 'missing' stuff at the left + if (not periodic) and shiftcenter: + expected_out[:1, :] = 0. + expected_out[:, :1] = 0. + #print(axis, shiftcenter, depth, periodic, weighted) + print('expected:\n', expected_out) + print('recovered:\n', img.v) + assert_rel_equal(expected_out, img.v, 4) From 159167bead3fd6a818cd8ab1999fbffaeebd7588 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 17 Jul 2024 19:10:34 -0500 Subject: [PATCH 112/186] fix ruff issues --- .../tests/test_sph_pixelization.py | 5 ++- yt/testing.py | 33 ++++++++++--------- .../tests/test_offaxisprojection.py | 2 +- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/yt/geometry/coordinates/tests/test_sph_pixelization.py b/yt/geometry/coordinates/tests/test_sph_pixelization.py index d42ca6de8f..5c6bd91677 100644 --- a/yt/geometry/coordinates/tests/test_sph_pixelization.py +++ b/yt/geometry/coordinates/tests/test_sph_pixelization.py @@ -1,6 +1,5 @@ -import pytest - import numpy as np +import pytest import unyt import yt @@ -176,7 +175,7 @@ def test_sph_gridproj_reseffect1(): ''' imgs, _ = getdata_test_gridproj2() ref = imgs[-1] - for i, img in enumerate(imgs): + for img in imgs: assert_rel_equal(img[::img.shape[0] // 2, ::img.shape[1] // 2], ref[::ref.shape[0] // 2, ::ref.shape[1] // 2], 4) diff --git a/yt/testing.py b/yt/testing.py index d519f32c79..14b58ed950 100644 --- a/yt/testing.py +++ b/yt/testing.py @@ -13,8 +13,8 @@ from unittest import SkipTest import matplotlib -import numpy as np from more_itertools import always_iterable +import numpy as np from numpy.random import RandomState from unyt.exceptions import UnitOperationError @@ -139,10 +139,11 @@ def integrate_kernel(kernelfunc: Callable[[float], float], integral = np.sum(spv * dx, axis=0) return pre * integral +_zeroperiods = np.array([0., 0., 0.]) def distancematrix(pos3_i0: np.ndarray[float], pos3_i1: np.ndarray[float], periodic: tuple[bool] = (True,) * 3, - periods: np.ndarray = np.array([0., 0., 0.]), + periods: np.ndarray = _zeroperiods, ) -> np.ndarray[float]: ''' Calculates the distances between two arrays of points. @@ -788,14 +789,18 @@ def fake_sph_grid_ds(hsml_factor=1.0): def constantmass(i: int, j: int, k: int) -> float: return 1. +_xhat = np.array([1, 0, 0]) +_yhat = np.array([0, 1, 0]) +_zhat = np.array([0, 0, 1]) +_floathalves = 0.5 * np.ones((3,), dtype=np.float64) def fake_sph_flexible_grid_ds( hsml_factor: float = 1.0, nperside: int = 3, periodic: bool = True, - e1hat: np.ndarray[float] = np.array([1, 0, 0]), - e2hat: np.ndarray[float] = np.array([0, 1, 0]), - e3hat: np.ndarray[float] = np.array([0, 0, 1]), - offsets: np.ndarray[float] = 0.5 * np.ones((3,), dtype=np.float64), + e1hat: np.ndarray[float] = _xhat, + e2hat: np.ndarray[float] = _yhat, + e3hat: np.ndarray[float] = _zhat, + offsets: np.ndarray[float] = _floathalves, massgenerator: Callable[[int, int, int], float] = constantmass, unitrho: float = 1., bbox: np.ndarray | None = None, @@ -949,17 +954,13 @@ def fake_random_sph_ds(npart: int, bbox: np.ndarray, if not hasattr(periodic, "__len__"): periodic = (periodic, ) * 3 + gen = np.random.default_rng(seed=0) - posx = np.random.uniform(low=bbox[0][0], high=bbox[0][1], - size=npart) - posy = np.random.uniform(low=bbox[1][0], high=bbox[1][1], - size=npart) - posz = np.random.uniform(low=bbox[2][0], high=bbox[2][1], - size=npart) - mass = np.random.uniform(low=massrange[0], high=massrange[1], - size=npart) - hsml = np.random.uniform(low=hsmlrange[0], high=hsmlrange[1], - size=npart) + posx = gen.uniform(low=bbox[0][0], high=bbox[0][1], size=npart) + posy = gen.uniform(low=bbox[1][0], high=bbox[1][1], size=npart) + posz = gen.uniform(low=bbox[2][0], high=bbox[2][1], size=npart) + mass = gen.uniform(low=massrange[0], high=massrange[1], size=npart) + hsml = gen.uniform(low=hsmlrange[0], high=hsmlrange[1], size=npart) dens = mass / hsml**3 * unitrho data = { diff --git a/yt/visualization/tests/test_offaxisprojection.py b/yt/visualization/tests/test_offaxisprojection.py index 42b9b5faaa..02f8eb9380 100644 --- a/yt/visualization/tests/test_offaxisprojection.py +++ b/yt/visualization/tests/test_offaxisprojection.py @@ -1,12 +1,12 @@ import itertools as it import os -import pytest import shutil import tempfile import unittest import numpy as np from numpy.testing import assert_equal +import pytest import unyt from yt.testing import ( From 2f589c95f7373589f6926e8c6a67fc4046d29a3e Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 17 Jul 2024 19:19:02 -0500 Subject: [PATCH 113/186] order imports correctly? --- yt/geometry/coordinates/tests/test_sph_pixelization.py | 2 +- yt/testing.py | 2 +- yt/visualization/tests/test_offaxisprojection.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/yt/geometry/coordinates/tests/test_sph_pixelization.py b/yt/geometry/coordinates/tests/test_sph_pixelization.py index 5c6bd91677..89b2d8003a 100644 --- a/yt/geometry/coordinates/tests/test_sph_pixelization.py +++ b/yt/geometry/coordinates/tests/test_sph_pixelization.py @@ -8,9 +8,9 @@ assert_rel_equal, cubicspline_python, distancematrix, - integrate_kernel, fake_random_sph_ds, fake_sph_flexible_grid_ds, + integrate_kernel, requires_file) from yt.utilities.math_utils import compute_stddev_image diff --git a/yt/testing.py b/yt/testing.py index 14b58ed950..68b6cc4619 100644 --- a/yt/testing.py +++ b/yt/testing.py @@ -13,8 +13,8 @@ from unittest import SkipTest import matplotlib -from more_itertools import always_iterable import numpy as np +from more_itertools import always_iterable from numpy.random import RandomState from unyt.exceptions import UnitOperationError diff --git a/yt/visualization/tests/test_offaxisprojection.py b/yt/visualization/tests/test_offaxisprojection.py index 02f8eb9380..7b7f7e6966 100644 --- a/yt/visualization/tests/test_offaxisprojection.py +++ b/yt/visualization/tests/test_offaxisprojection.py @@ -5,18 +5,18 @@ import unittest import numpy as np -from numpy.testing import assert_equal import pytest import unyt +from numpy.testing import assert_equal from yt.testing import ( assert_fname, assert_rel_equal, cubicspline_python, - integrate_kernel, fake_octree_ds, fake_random_ds, fake_sph_flexible_grid_ds, + integrate_kernel, ) from yt.visualization.api import ( OffAxisProjectionPlot, From aecc0513ded478f0dfc0fc73519e11e74cf35700 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:21:35 -0500 Subject: [PATCH 114/186] separate out pytest-only tests, add to nose ignore lists --- nose_ignores.txt | 2 + tests/tests.yaml | 2 + .../tests/test_sph_pixelization.py | 451 ----------------- .../tests/test_sph_pixelization_pytestonly.py | 459 ++++++++++++++++++ .../tests/test_offaxisprojection.py | 142 +----- .../test_offaxisprojection_pytestonly.py | 143 ++++++ 6 files changed, 607 insertions(+), 592 deletions(-) create mode 100644 yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py create mode 100644 yt/visualization/tests/test_offaxisprojection_pytestonly.py diff --git a/nose_ignores.txt b/nose_ignores.txt index 115b917d59..b5529a6ecc 100644 --- a/nose_ignores.txt +++ b/nose_ignores.txt @@ -45,3 +45,5 @@ --ignore-file=test_set_log_level\.py --ignore-file=test_field_parsing\.py --ignore-file=test_disks\.py +--ignore-file=test_offaxisprojection_pytestonly\.py +--ignore-file=test_sph_pixelization_pytestonly\.py diff --git a/tests/tests.yaml b/tests/tests.yaml index d0f12ad1e5..d7de9b9650 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -221,5 +221,7 @@ other_tests: - "--exclude-test=yt.frontends.gdf.tests.test_outputs.TestGDF" - "--exclude-test=yt.frontends.adaptahop.tests.test_outputs" - "--exclude-test=yt.frontends.stream.tests.test_stream_particles.test_stream_non_cartesian_particles" + - "--ignore-file=test_offaxisprojection_pytestonly\\.py" + - "--ignore-file=test_sph_pixelization_pytestonly\\.py" cookbook: - 'doc/source/cookbook/tests/test_cookbook.py' diff --git a/yt/geometry/coordinates/tests/test_sph_pixelization.py b/yt/geometry/coordinates/tests/test_sph_pixelization.py index 89b2d8003a..da5c9984bf 100644 --- a/yt/geometry/coordinates/tests/test_sph_pixelization.py +++ b/yt/geometry/coordinates/tests/test_sph_pixelization.py @@ -1,14 +1,9 @@ import numpy as np -import pytest -import unyt import yt -from yt.data_objects.selection_objects.region import YTRegion from yt.testing import ( assert_rel_equal, cubicspline_python, - distancematrix, - fake_random_sph_ds, fake_sph_flexible_grid_ds, integrate_kernel, requires_file) @@ -205,449 +200,3 @@ def test_sph_gridproj_reseffect2(): #print(f'Grid refinement level {rl}:') assert_rel_equal(img[::pixspace, ::pixspace], ref[::pixspace_ref, ::pixspace_ref], 4) - - -@pytest.mark.parametrize("weighted", [True, False]) -@pytest.mark.parametrize("periodic", [True, False]) -@pytest.mark.parametrize("depth", [None, (1., "cm")]) -@pytest.mark.parametrize("shiftcenter", [False, True]) -@pytest.mark.parametrize("axis", [0, 1, 2]) -def test_sph_proj_general_alongaxes(axis: int, - shiftcenter: bool, - depth : float | None, - periodic: bool, - weighted: bool) -> None: - ''' - The previous projection tests were for a specific issue. - Here, we test more functionality of the projections. - We just send lines of sight through pixel centers for convenience. - Particles at [0.5, 1.5, 2.5] (in each coordinate) - smoothing lengths 0.25 - all particles have mass 1., density 1.5, - except the single center particle, with mass 2., density 3. - - Parameters: - ----------- - axis: {0, 1, 2} - projection axis (aligned with sim. axis) - shiftcenter: bool - shift the coordinates to center the projection on. - (The grid is offset to this same center) - depth: float or None - depth of the projection slice - periodic: bool - assume periodic boundary conditions, or not - weighted: bool - make a weighted projection (density-weighted density), or not - - Returns: - -------- - None - ''' - if shiftcenter: - center = unyt.unyt_array(np.array((0.625, 0.625, 0.625)), 'cm') - else: - center = unyt.unyt_array(np.array((1.5, 1.5, 1.5)), 'cm') - bbox = unyt.unyt_array(np.array([[0., 3.], [0., 3.], [0., 3.]]), 'cm') - hsml_factor = 0.5 - unitrho = 1.5 - # test correct centering, particle selection - def makemasses(i, j, k): - if i == j == k == 1: - return 2. - else: - return 1. - # m / rho, factor 1. / hsml**2 is included in the kernel integral - # (density is adjusted, so same for center particle) - prefactor = 1. / unitrho #/ (0.5 * 0.5)**2 - dl_cen = integrate_kernel(cubicspline_python, 0., 0.25) - - # result shouldn't depend explicitly on the center if we re-center - # the data, unless we get cut-offs in the non-periodic case - ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, - periodic=periodic, - offsets=np.full(3, 0.5), - massgenerator=makemasses, - unitrho=unitrho, - bbox=bbox.v, - recenter=center.v) - if depth is None: - source = ds.all_data() - else: - depth = unyt.unyt_quantity(*depth) - le = np.array(ds.domain_left_edge) - re = np.array(ds.domain_right_edge) - le[axis] = center[axis] - 0.5 * depth - re[axis] = center[axis] + 0.5 * depth - cen = 0.5 * (le + re) - reg = YTRegion(center=cen, - left_edge=le, - right_edge=re, - ds=ds) - source = reg - - # we don't actually want a plot, it's just a straightforward, - # common way to get an frb / image array - if weighted: - toweight_field = ("gas", "density") - else: - toweight_field = None - prj = yt.ProjectionPlot(ds, axis, ("gas", "density"), - width=(2.5, "cm"), - weight_field=toweight_field, - buff_size=(5, 5), - center=center, - data_source=source) - img = prj.frb.data[('gas', 'density')] - if weighted: - expected_out = np.zeros((5, 5,), dtype=img.v.dtype) - expected_out[::2, ::2] = unitrho - if depth is None: - ## during shift, particle coords do wrap around edges - #if (not periodic) and shiftcenter: - # # weight 1. for unitrho, 2. for 2. * untrho - # expected_out[2, 2] *= 5. / 3. - #else: - # weight (2 * 1.) for unitrho, (1 * 2.) for 2. * unitrho - expected_out[2, 2] *= 1.5 - else: - # only 2 * unitrho element included - expected_out[2, 2] *= 2. - else: - expected_out = np.zeros((5, 5,), dtype=img.v.dtype) - expected_out[::2, ::2] = dl_cen * prefactor * unitrho - if depth is None: - # 3 particles per l.o.s., including the denser one - expected_out *= 3. - expected_out[2, 2] *= 4. / 3. - else: - # 1 particle per l.o.s., including the denser one - expected_out[2, 2] *= 2. - # grid is shifted to the left -> 'missing' stuff at the left - if (not periodic) and shiftcenter: - expected_out[:1, :] = 0. - expected_out[:, :1] = 0. - #print(axis, shiftcenter, depth, periodic, weighted) - #print(expected_out) - #print(img.v) - assert_rel_equal(expected_out, img.v, 5) - -@pytest.mark.parametrize("periodic", [True, False]) -@pytest.mark.parametrize("shiftcenter", [False, True]) -@pytest.mark.parametrize("zoff", [0., 0.1, 0.5, 1.]) -@pytest.mark.parametrize("axis", [0, 1, 2]) -def test_sph_slice_general_alongaxes(axis: int, - shiftcenter: bool, - periodic: bool, - zoff: float) -> None: - ''' - Particles at [0.5, 1.5, 2.5] (in each coordinate) - smoothing lengths 0.25 - all particles have mass 1., density 1.5, - except the single center particle, with mass 2., density 3. - - Parameters: - ----------- - axis: {0, 1, 2} - projection axis (aligned with sim. axis) - northvector: tuple - y-axis direction in the final plot (direction vector) - shiftcenter: bool - shift the coordinates to center the projection on. - (The grid is offset to this same center) - zoff: float - offset of the slice plane from the SPH particle center plane - periodic: bool - assume periodic boundary conditions, or not - - Returns: - -------- - None - ''' - if shiftcenter: - center = unyt.unyt_array(np.array((0.625, 0.625, 0.625)), 'cm') - else: - center = unyt.unyt_array(np.array((1.5, 1.5, 1.5)), 'cm') - bbox = unyt.unyt_array(np.array([[0., 3.], [0., 3.], [0., 3.]]), 'cm') - hsml_factor = 0.5 - unitrho = 1.5 - # test correct centering, particle selection - def makemasses(i, j, k): - if i == j == k == 1: - return 2. - elif i == j == k == 2: - return 3. - else: - return 1. - - # result shouldn't depend explicitly on the center if we re-center - # the data, unless we get cut-offs in the non-periodic case - ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, - periodic=periodic, - offsets=np.full(3, 0.5), - massgenerator=makemasses, - unitrho=unitrho, - bbox=bbox.v, - recenter=center.v) - ad = ds.all_data() - #print(ad[('gas', 'position')]) - outgridsize = 10 - width = 2.5 - _center = center.to("cm").v.copy() - _center[axis] += zoff - - # we don't actually want a plot, it's just a straightforward, - # common way to get an frb / image array - slc = yt.SlicePlot(ds, axis, ("gas", "density"), - width=(width, "cm"), - buff_size=(outgridsize,) * 2, - center=(_center, "cm")) - img = slc.frb.data[('gas', 'density')] - - # center is same in non-projection coords - if axis == 0: - ci = 1 - else: - ci = 0 - gridcens = _center[ci] - 0.5 * width \ - + 0.5 * width / outgridsize \ - + np.arange(outgridsize) * width / outgridsize - xgrid = np.repeat(gridcens, outgridsize) - ygrid = np.tile(gridcens, outgridsize) - zgrid = np.full(outgridsize**2, _center[axis]) - gridcoords = np.empty((outgridsize**2, 3), dtype=xgrid.dtype) - if axis == 2: - gridcoords[:, 0] = xgrid - gridcoords[:, 1] = ygrid - gridcoords[:, 2] = zgrid - elif axis == 0: - gridcoords[:, 0] = zgrid - gridcoords[:, 1] = xgrid - gridcoords[:, 2] = ygrid - elif axis == 1: - gridcoords[:, 0] = ygrid - gridcoords[:, 1] = zgrid - gridcoords[:, 2] = xgrid - ad = ds.all_data() - sphcoords = np.array([(ad[("gas", "x")]).to("cm"), - (ad[("gas", "y")]).to("cm"), - (ad[("gas", "z")]).to("cm"), - ]).T - print('sphcoords:') - print(sphcoords) - print('gridcoords:') - print(gridcoords) - dists = distancematrix(gridcoords, sphcoords, - periodic=(periodic,)*3, - periods=np.array([3., 3., 3.])) - print('dists <= 1:') - print(dists <= 1) - sml = (ad[("gas", "smoothing_length")]).to("cm") - normkern = cubicspline_python(dists / sml.v[np.newaxis, :]) - sphcontr = normkern / sml[np.newaxis, :]**3 * ad[("gas", "mass")] - contsum = np.sum(sphcontr, axis=1) - sphweights = normkern / sml[np.newaxis, :]**3 \ - * ad[("gas", "mass")] / ad[("gas", "density")] - weights = np.sum(sphweights, axis=1) - expected = contsum / weights - expected = expected.reshape((outgridsize, outgridsize)) - expected[np.isnan(expected)] = 0. # convention in the slices - - print('expected:\n', expected.v) - print('recovered:\n', img.v) - assert_rel_equal(expected.v, img.v, 5) - - - -@pytest.mark.parametrize("periodic", [True, False]) -@pytest.mark.parametrize("shiftcenter", [False, True]) -@pytest.mark.parametrize("northvector", [None, (1.e-4, 1., 0.)]) -@pytest.mark.parametrize("zoff", [0., 0.1, 0.5, 1.]) -def test_sph_slice_general_offaxis( - northvector: tuple[float, float, float] | None, - shiftcenter: bool, - zoff: float, - periodic: bool, - ) -> None: - ''' - Same as the on-axis slices, but we rotate the basis vectors - to test whether roations are handled ok. the rotation is chosen to - be small so that in/exclusion of particles within bboxes, etc. - works out the same way. - Particles at [0.5, 1.5, 2.5] (in each coordinate) - smoothing lengths 0.25 - all particles have mass 1., density 1.5, - except the single center particle, with mass 2., density 3. - - Parameters: - ----------- - northvector: tuple - y-axis direction in the final plot (direction vector) - shiftcenter: bool - shift the coordinates to center the projection on. - (The grid is offset to this same center) - zoff: float - offset of the slice plane from the SPH particle center plane - periodic: bool - assume periodic boundary conditions, or not - - Returns: - -------- - None - ''' - if shiftcenter: - center = np.array((0.625, 0.625, 0.625)) # cm - else: - center = np.array((1.5, 1.5, 1.5)) # cm - bbox = unyt.unyt_array(np.array([[0., 3.], [0., 3.], [0., 3.]]), 'cm') - hsml_factor = 0.5 - unitrho = 1.5 - # test correct centering, particle selection - def makemasses(i, j, k): - if i == j == k == 1: - return 2. - else: - return 1. - # try to make sure dl differences from periodic wrapping are small - epsilon = 1e-4 - projaxis = np.array([epsilon, 0.00, np.sqrt(1. - epsilon**2)]) - e1dir = projaxis / np.sqrt(np.sum(projaxis**2)) - if northvector is None: - e2dir = np.array([0., 1., 0.]) - else: - e2dir = np.asarray(northvector) - e2dir = e2dir - np.sum(e1dir * e2dir) * e2dir # orthonormalize - e2dir /= np.sqrt(np.sum(e2dir**2)) - e3dir = np.cross(e2dir, e1dir) - - outgridsize = 10 - width = 2.5 - _center = center.copy() - _center += zoff * e1dir - - ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, - periodic=periodic, - offsets=np.full(3, 0.5), - massgenerator=makemasses, - unitrho=unitrho, - bbox=bbox.v, - recenter=center, - e1hat=e1dir, e2hat=e2dir, e3hat=e3dir) - - source = ds.all_data() - # couple to dataset -> right unit registry - center = ds.arr(center, 'cm') - print('position:\n', source['gas','position']) - slc = yt.SlicePlot(ds, e1dir, ("gas", "density"), - width=(width, "cm"), - buff_size=(outgridsize,) * 2, - center=(_center, "cm"), - north_vector=e2dir) - img = slc.frb.data[('gas', 'density')] - - # center is same in x/y (e3dir/e2dir) - gridcenx = np.dot(_center, e3dir) - 0.5 * width \ - + 0.5 * width / outgridsize \ - + np.arange(outgridsize) * width / outgridsize - gridceny = np.dot(_center, e2dir) - 0.5 * width \ - + 0.5 * width / outgridsize \ - + np.arange(outgridsize) * width / outgridsize - xgrid = np.repeat(gridcenx, outgridsize) - ygrid = np.tile(gridceny, outgridsize) - zgrid = np.full(outgridsize**2, np.dot(_center, e1dir)) - gridcoords = (xgrid[:, np.newaxis] * e3dir[np.newaxis, :] - + ygrid[:, np.newaxis] * e2dir[np.newaxis, :] - + zgrid[:, np.newaxis] * e1dir[np.newaxis, :]) - print('gridcoords:') - print(gridcoords) - ad = ds.all_data() - sphcoords = np.array([(ad[("gas", "x")]).to("cm"), - (ad[("gas", "y")]).to("cm"), - (ad[("gas", "z")]).to("cm"), - ]).T - dists = distancematrix(gridcoords, sphcoords, - periodic=(periodic,)*3, - periods=np.array([3., 3., 3.])) - sml = (ad[("gas", "smoothing_length")]).to("cm") - normkern = cubicspline_python(dists / sml.v[np.newaxis, :]) - sphcontr = normkern / sml[np.newaxis, :]**3 * ad[("gas", "mass")] - contsum = np.sum(sphcontr, axis=1) - sphweights = normkern / sml[np.newaxis, :]**3 \ - * ad[("gas", "mass")] / ad[("gas", "density")] - weights = np.sum(sphweights, axis=1) - expected = contsum / weights - expected = expected.reshape((outgridsize, outgridsize)) - expected = expected.T # transposed for image plotting - expected[np.isnan(expected)] = 0. # convention in the slices - - #print(axis, shiftcenter, depth, periodic, weighted) - print('expected:\n', expected.v) - print('recovered:\n', img.v) - assert_rel_equal(expected.v, img.v, 4) - -# only axis-aligned; testing YTArbitraryGrid, YTCoveringGrid -@pytest.mark.parametrize("periodic", [True, False, (True, True, False)]) -@pytest.mark.parametrize("wholebox", [True, False]) -def test_sph_grid(periodic: bool | tuple[bool, bool, bool], - wholebox: bool): - bbox = np.array([[-1., 3.], [1., 5.2], [-1., 3.]]) - ds = fake_random_sph_ds(50, bbox, periodic=periodic) - - if not hasattr(periodic, "__len__"): - periodic = (periodic,) * 3 - - if wholebox: - left = bbox[:, 0].copy() - level = 2 - ncells = np.array([2**level] * 3) - print('left: ', left) - print('ncells: ', ncells) - resgrid = ds.covering_grid(level, tuple(left), ncells) - right = bbox[:, 1].copy() - xedges = np.linspace(left[0], right[0], ncells[0] + 1) - yedges = np.linspace(left[1], right[1], ncells[1] + 1) - zedges = np.linspace(left[2], right[2], ncells[2] + 1) - else: - left = np.array([-1., 1.8, -1.]) - right = np.array([2.5, 5.2, 2.5]) - ncells = np.array([3, 4, 4]) - resgrid = ds.arbitrary_grid(left, right, dims=ncells) - xedges = np.linspace(left[0], right[0], ncells[0] + 1) - yedges = np.linspace(left[1], right[1], ncells[1] + 1) - zedges = np.linspace(left[2], right[2], ncells[2] + 1) - res = resgrid["gas", "density"] - xcens = 0.5 * (xedges[:-1] + xedges[1:]) - ycens = 0.5 * (yedges[:-1] + yedges[1:]) - zcens = 0.5 * (zedges[:-1] + zedges[1:]) - - ad = ds.all_data() - sphcoords = np.array([(ad[("gas", "x")]).to("cm"), - (ad[("gas", "y")]).to("cm"), - (ad[("gas", "z")]).to("cm"), - ]).T - gridx, gridy, gridz = np.meshgrid(xcens, ycens, zcens, - indexing='ij') - outshape = gridx.shape - gridx = gridx.flatten() - gridy = gridy.flatten() - gridz = gridz.flatten() - gridcoords = np.array([gridx, gridy, gridz]).T - periods = bbox[:, 1] - bbox[:, 0] - dists = distancematrix(gridcoords, sphcoords, - periodic=periodic, - periods=periods) - sml = (ad[("gas", "smoothing_length")]).to("cm") - normkern = cubicspline_python(dists / sml.v[np.newaxis, :]) - sphcontr = normkern / sml[np.newaxis, :]**3 * ad[("gas", "mass")] - contsum = np.sum(sphcontr, axis=1) - sphweights = normkern / sml[np.newaxis, :]**3 \ - * ad[("gas", "mass")] / ad[("gas", "density")] - weights = np.sum(sphweights, axis=1) - expected = contsum / weights - expected = expected.reshape(outshape) - expected[np.isnan(expected)] = 0. # convention in the slices - - #print(axis, shiftcenter, depth, periodic, weighted) - print('expected:\n', expected.v) - print('recovered:\n', res.v) - assert_rel_equal(expected.v, res.v, 4) diff --git a/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py b/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py new file mode 100644 index 0000000000..4d299e2516 --- /dev/null +++ b/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py @@ -0,0 +1,459 @@ +import numpy as np +import pytest +import unyt + +import yt +from yt.data_objects.selection_objects.region import YTRegion +from yt.testing import ( + assert_rel_equal, + cubicspline_python, + distancematrix, + fake_random_sph_ds, + fake_sph_flexible_grid_ds, + integrate_kernel, +) + +@pytest.mark.parametrize("weighted", [True, False]) +@pytest.mark.parametrize("periodic", [True, False]) +@pytest.mark.parametrize("depth", [None, (1., "cm")]) +@pytest.mark.parametrize("shiftcenter", [False, True]) +@pytest.mark.parametrize("axis", [0, 1, 2]) +def test_sph_proj_general_alongaxes(axis: int, + shiftcenter: bool, + depth : float | None, + periodic: bool, + weighted: bool) -> None: + ''' + The previous projection tests were for a specific issue. + Here, we test more functionality of the projections. + We just send lines of sight through pixel centers for convenience. + Particles at [0.5, 1.5, 2.5] (in each coordinate) + smoothing lengths 0.25 + all particles have mass 1., density 1.5, + except the single center particle, with mass 2., density 3. + + Parameters: + ----------- + axis: {0, 1, 2} + projection axis (aligned with sim. axis) + shiftcenter: bool + shift the coordinates to center the projection on. + (The grid is offset to this same center) + depth: float or None + depth of the projection slice + periodic: bool + assume periodic boundary conditions, or not + weighted: bool + make a weighted projection (density-weighted density), or not + + Returns: + -------- + None + ''' + if shiftcenter: + center = unyt.unyt_array(np.array((0.625, 0.625, 0.625)), 'cm') + else: + center = unyt.unyt_array(np.array((1.5, 1.5, 1.5)), 'cm') + bbox = unyt.unyt_array(np.array([[0., 3.], [0., 3.], [0., 3.]]), 'cm') + hsml_factor = 0.5 + unitrho = 1.5 + # test correct centering, particle selection + def makemasses(i, j, k): + if i == j == k == 1: + return 2. + else: + return 1. + # m / rho, factor 1. / hsml**2 is included in the kernel integral + # (density is adjusted, so same for center particle) + prefactor = 1. / unitrho #/ (0.5 * 0.5)**2 + dl_cen = integrate_kernel(cubicspline_python, 0., 0.25) + + # result shouldn't depend explicitly on the center if we re-center + # the data, unless we get cut-offs in the non-periodic case + ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, + periodic=periodic, + offsets=np.full(3, 0.5), + massgenerator=makemasses, + unitrho=unitrho, + bbox=bbox.v, + recenter=center.v) + if depth is None: + source = ds.all_data() + else: + depth = unyt.unyt_quantity(*depth) + le = np.array(ds.domain_left_edge) + re = np.array(ds.domain_right_edge) + le[axis] = center[axis] - 0.5 * depth + re[axis] = center[axis] + 0.5 * depth + cen = 0.5 * (le + re) + reg = YTRegion(center=cen, + left_edge=le, + right_edge=re, + ds=ds) + source = reg + + # we don't actually want a plot, it's just a straightforward, + # common way to get an frb / image array + if weighted: + toweight_field = ("gas", "density") + else: + toweight_field = None + prj = yt.ProjectionPlot(ds, axis, ("gas", "density"), + width=(2.5, "cm"), + weight_field=toweight_field, + buff_size=(5, 5), + center=center, + data_source=source) + img = prj.frb.data[('gas', 'density')] + if weighted: + expected_out = np.zeros((5, 5,), dtype=img.v.dtype) + expected_out[::2, ::2] = unitrho + if depth is None: + ## during shift, particle coords do wrap around edges + #if (not periodic) and shiftcenter: + # # weight 1. for unitrho, 2. for 2. * untrho + # expected_out[2, 2] *= 5. / 3. + #else: + # weight (2 * 1.) for unitrho, (1 * 2.) for 2. * unitrho + expected_out[2, 2] *= 1.5 + else: + # only 2 * unitrho element included + expected_out[2, 2] *= 2. + else: + expected_out = np.zeros((5, 5,), dtype=img.v.dtype) + expected_out[::2, ::2] = dl_cen * prefactor * unitrho + if depth is None: + # 3 particles per l.o.s., including the denser one + expected_out *= 3. + expected_out[2, 2] *= 4. / 3. + else: + # 1 particle per l.o.s., including the denser one + expected_out[2, 2] *= 2. + # grid is shifted to the left -> 'missing' stuff at the left + if (not periodic) and shiftcenter: + expected_out[:1, :] = 0. + expected_out[:, :1] = 0. + #print(axis, shiftcenter, depth, periodic, weighted) + #print(expected_out) + #print(img.v) + assert_rel_equal(expected_out, img.v, 5) + +@pytest.mark.parametrize("periodic", [True, False]) +@pytest.mark.parametrize("shiftcenter", [False, True]) +@pytest.mark.parametrize("zoff", [0., 0.1, 0.5, 1.]) +@pytest.mark.parametrize("axis", [0, 1, 2]) +def test_sph_slice_general_alongaxes(axis: int, + shiftcenter: bool, + periodic: bool, + zoff: float) -> None: + ''' + Particles at [0.5, 1.5, 2.5] (in each coordinate) + smoothing lengths 0.25 + all particles have mass 1., density 1.5, + except the single center particle, with mass 2., density 3. + + Parameters: + ----------- + axis: {0, 1, 2} + projection axis (aligned with sim. axis) + northvector: tuple + y-axis direction in the final plot (direction vector) + shiftcenter: bool + shift the coordinates to center the projection on. + (The grid is offset to this same center) + zoff: float + offset of the slice plane from the SPH particle center plane + periodic: bool + assume periodic boundary conditions, or not + + Returns: + -------- + None + ''' + if shiftcenter: + center = unyt.unyt_array(np.array((0.625, 0.625, 0.625)), 'cm') + else: + center = unyt.unyt_array(np.array((1.5, 1.5, 1.5)), 'cm') + bbox = unyt.unyt_array(np.array([[0., 3.], [0., 3.], [0., 3.]]), 'cm') + hsml_factor = 0.5 + unitrho = 1.5 + # test correct centering, particle selection + def makemasses(i, j, k): + if i == j == k == 1: + return 2. + elif i == j == k == 2: + return 3. + else: + return 1. + + # result shouldn't depend explicitly on the center if we re-center + # the data, unless we get cut-offs in the non-periodic case + ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, + periodic=periodic, + offsets=np.full(3, 0.5), + massgenerator=makemasses, + unitrho=unitrho, + bbox=bbox.v, + recenter=center.v) + ad = ds.all_data() + #print(ad[('gas', 'position')]) + outgridsize = 10 + width = 2.5 + _center = center.to("cm").v.copy() + _center[axis] += zoff + + # we don't actually want a plot, it's just a straightforward, + # common way to get an frb / image array + slc = yt.SlicePlot(ds, axis, ("gas", "density"), + width=(width, "cm"), + buff_size=(outgridsize,) * 2, + center=(_center, "cm")) + img = slc.frb.data[('gas', 'density')] + + # center is same in non-projection coords + if axis == 0: + ci = 1 + else: + ci = 0 + gridcens = _center[ci] - 0.5 * width \ + + 0.5 * width / outgridsize \ + + np.arange(outgridsize) * width / outgridsize + xgrid = np.repeat(gridcens, outgridsize) + ygrid = np.tile(gridcens, outgridsize) + zgrid = np.full(outgridsize**2, _center[axis]) + gridcoords = np.empty((outgridsize**2, 3), dtype=xgrid.dtype) + if axis == 2: + gridcoords[:, 0] = xgrid + gridcoords[:, 1] = ygrid + gridcoords[:, 2] = zgrid + elif axis == 0: + gridcoords[:, 0] = zgrid + gridcoords[:, 1] = xgrid + gridcoords[:, 2] = ygrid + elif axis == 1: + gridcoords[:, 0] = ygrid + gridcoords[:, 1] = zgrid + gridcoords[:, 2] = xgrid + ad = ds.all_data() + sphcoords = np.array([(ad[("gas", "x")]).to("cm"), + (ad[("gas", "y")]).to("cm"), + (ad[("gas", "z")]).to("cm"), + ]).T + print('sphcoords:') + print(sphcoords) + print('gridcoords:') + print(gridcoords) + dists = distancematrix(gridcoords, sphcoords, + periodic=(periodic,)*3, + periods=np.array([3., 3., 3.])) + print('dists <= 1:') + print(dists <= 1) + sml = (ad[("gas", "smoothing_length")]).to("cm") + normkern = cubicspline_python(dists / sml.v[np.newaxis, :]) + sphcontr = normkern / sml[np.newaxis, :]**3 * ad[("gas", "mass")] + contsum = np.sum(sphcontr, axis=1) + sphweights = normkern / sml[np.newaxis, :]**3 \ + * ad[("gas", "mass")] / ad[("gas", "density")] + weights = np.sum(sphweights, axis=1) + expected = contsum / weights + expected = expected.reshape((outgridsize, outgridsize)) + expected[np.isnan(expected)] = 0. # convention in the slices + + print('expected:\n', expected.v) + print('recovered:\n', img.v) + assert_rel_equal(expected.v, img.v, 5) + + + +@pytest.mark.parametrize("periodic", [True, False]) +@pytest.mark.parametrize("shiftcenter", [False, True]) +@pytest.mark.parametrize("northvector", [None, (1.e-4, 1., 0.)]) +@pytest.mark.parametrize("zoff", [0., 0.1, 0.5, 1.]) +def test_sph_slice_general_offaxis( + northvector: tuple[float, float, float] | None, + shiftcenter: bool, + zoff: float, + periodic: bool, + ) -> None: + ''' + Same as the on-axis slices, but we rotate the basis vectors + to test whether roations are handled ok. the rotation is chosen to + be small so that in/exclusion of particles within bboxes, etc. + works out the same way. + Particles at [0.5, 1.5, 2.5] (in each coordinate) + smoothing lengths 0.25 + all particles have mass 1., density 1.5, + except the single center particle, with mass 2., density 3. + + Parameters: + ----------- + northvector: tuple + y-axis direction in the final plot (direction vector) + shiftcenter: bool + shift the coordinates to center the projection on. + (The grid is offset to this same center) + zoff: float + offset of the slice plane from the SPH particle center plane + periodic: bool + assume periodic boundary conditions, or not + + Returns: + -------- + None + ''' + if shiftcenter: + center = np.array((0.625, 0.625, 0.625)) # cm + else: + center = np.array((1.5, 1.5, 1.5)) # cm + bbox = unyt.unyt_array(np.array([[0., 3.], [0., 3.], [0., 3.]]), 'cm') + hsml_factor = 0.5 + unitrho = 1.5 + # test correct centering, particle selection + def makemasses(i, j, k): + if i == j == k == 1: + return 2. + else: + return 1. + # try to make sure dl differences from periodic wrapping are small + epsilon = 1e-4 + projaxis = np.array([epsilon, 0.00, np.sqrt(1. - epsilon**2)]) + e1dir = projaxis / np.sqrt(np.sum(projaxis**2)) + if northvector is None: + e2dir = np.array([0., 1., 0.]) + else: + e2dir = np.asarray(northvector) + e2dir = e2dir - np.sum(e1dir * e2dir) * e2dir # orthonormalize + e2dir /= np.sqrt(np.sum(e2dir**2)) + e3dir = np.cross(e2dir, e1dir) + + outgridsize = 10 + width = 2.5 + _center = center.copy() + _center += zoff * e1dir + + ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, + periodic=periodic, + offsets=np.full(3, 0.5), + massgenerator=makemasses, + unitrho=unitrho, + bbox=bbox.v, + recenter=center, + e1hat=e1dir, e2hat=e2dir, e3hat=e3dir) + + source = ds.all_data() + # couple to dataset -> right unit registry + center = ds.arr(center, 'cm') + print('position:\n', source['gas','position']) + slc = yt.SlicePlot(ds, e1dir, ("gas", "density"), + width=(width, "cm"), + buff_size=(outgridsize,) * 2, + center=(_center, "cm"), + north_vector=e2dir) + img = slc.frb.data[('gas', 'density')] + + # center is same in x/y (e3dir/e2dir) + gridcenx = np.dot(_center, e3dir) - 0.5 * width \ + + 0.5 * width / outgridsize \ + + np.arange(outgridsize) * width / outgridsize + gridceny = np.dot(_center, e2dir) - 0.5 * width \ + + 0.5 * width / outgridsize \ + + np.arange(outgridsize) * width / outgridsize + xgrid = np.repeat(gridcenx, outgridsize) + ygrid = np.tile(gridceny, outgridsize) + zgrid = np.full(outgridsize**2, np.dot(_center, e1dir)) + gridcoords = (xgrid[:, np.newaxis] * e3dir[np.newaxis, :] + + ygrid[:, np.newaxis] * e2dir[np.newaxis, :] + + zgrid[:, np.newaxis] * e1dir[np.newaxis, :]) + print('gridcoords:') + print(gridcoords) + ad = ds.all_data() + sphcoords = np.array([(ad[("gas", "x")]).to("cm"), + (ad[("gas", "y")]).to("cm"), + (ad[("gas", "z")]).to("cm"), + ]).T + dists = distancematrix(gridcoords, sphcoords, + periodic=(periodic,)*3, + periods=np.array([3., 3., 3.])) + sml = (ad[("gas", "smoothing_length")]).to("cm") + normkern = cubicspline_python(dists / sml.v[np.newaxis, :]) + sphcontr = normkern / sml[np.newaxis, :]**3 * ad[("gas", "mass")] + contsum = np.sum(sphcontr, axis=1) + sphweights = normkern / sml[np.newaxis, :]**3 \ + * ad[("gas", "mass")] / ad[("gas", "density")] + weights = np.sum(sphweights, axis=1) + expected = contsum / weights + expected = expected.reshape((outgridsize, outgridsize)) + expected = expected.T # transposed for image plotting + expected[np.isnan(expected)] = 0. # convention in the slices + + #print(axis, shiftcenter, depth, periodic, weighted) + print('expected:\n', expected.v) + print('recovered:\n', img.v) + assert_rel_equal(expected.v, img.v, 4) + +# only axis-aligned; testing YTArbitraryGrid, YTCoveringGrid +@pytest.mark.parametrize("periodic", [True, False, (True, True, False)]) +@pytest.mark.parametrize("wholebox", [True, False]) +def test_sph_grid(periodic: bool | tuple[bool, bool, bool], + wholebox: bool): + bbox = np.array([[-1., 3.], [1., 5.2], [-1., 3.]]) + ds = fake_random_sph_ds(50, bbox, periodic=periodic) + + if not hasattr(periodic, "__len__"): + periodic = (periodic,) * 3 + + if wholebox: + left = bbox[:, 0].copy() + level = 2 + ncells = np.array([2**level] * 3) + print('left: ', left) + print('ncells: ', ncells) + resgrid = ds.covering_grid(level, tuple(left), ncells) + right = bbox[:, 1].copy() + xedges = np.linspace(left[0], right[0], ncells[0] + 1) + yedges = np.linspace(left[1], right[1], ncells[1] + 1) + zedges = np.linspace(left[2], right[2], ncells[2] + 1) + else: + left = np.array([-1., 1.8, -1.]) + right = np.array([2.5, 5.2, 2.5]) + ncells = np.array([3, 4, 4]) + resgrid = ds.arbitrary_grid(left, right, dims=ncells) + xedges = np.linspace(left[0], right[0], ncells[0] + 1) + yedges = np.linspace(left[1], right[1], ncells[1] + 1) + zedges = np.linspace(left[2], right[2], ncells[2] + 1) + res = resgrid["gas", "density"] + xcens = 0.5 * (xedges[:-1] + xedges[1:]) + ycens = 0.5 * (yedges[:-1] + yedges[1:]) + zcens = 0.5 * (zedges[:-1] + zedges[1:]) + + ad = ds.all_data() + sphcoords = np.array([(ad[("gas", "x")]).to("cm"), + (ad[("gas", "y")]).to("cm"), + (ad[("gas", "z")]).to("cm"), + ]).T + gridx, gridy, gridz = np.meshgrid(xcens, ycens, zcens, + indexing='ij') + outshape = gridx.shape + gridx = gridx.flatten() + gridy = gridy.flatten() + gridz = gridz.flatten() + gridcoords = np.array([gridx, gridy, gridz]).T + periods = bbox[:, 1] - bbox[:, 0] + dists = distancematrix(gridcoords, sphcoords, + periodic=periodic, + periods=periods) + sml = (ad[("gas", "smoothing_length")]).to("cm") + normkern = cubicspline_python(dists / sml.v[np.newaxis, :]) + sphcontr = normkern / sml[np.newaxis, :]**3 * ad[("gas", "mass")] + contsum = np.sum(sphcontr, axis=1) + sphweights = normkern / sml[np.newaxis, :]**3 \ + * ad[("gas", "mass")] / ad[("gas", "density")] + weights = np.sum(sphweights, axis=1) + expected = contsum / weights + expected = expected.reshape(outshape) + expected[np.isnan(expected)] = 0. # convention in the slices + + #print(axis, shiftcenter, depth, periodic, weighted) + print('expected:\n', expected.v) + print('recovered:\n', res.v) + assert_rel_equal(expected.v, res.v, 4) diff --git a/yt/visualization/tests/test_offaxisprojection.py b/yt/visualization/tests/test_offaxisprojection.py index 7b7f7e6966..d67fe4c4c5 100644 --- a/yt/visualization/tests/test_offaxisprojection.py +++ b/yt/visualization/tests/test_offaxisprojection.py @@ -5,23 +5,18 @@ import unittest import numpy as np -import pytest -import unyt from numpy.testing import assert_equal from yt.testing import ( assert_fname, assert_rel_equal, - cubicspline_python, fake_octree_ds, fake_random_ds, - fake_sph_flexible_grid_ds, - integrate_kernel, ) from yt.visualization.api import ( OffAxisProjectionPlot, OffAxisSlicePlot, - ProjectionPlot) +) from yt.visualization.image_writer import write_projection from yt.visualization.volume_rendering.api import off_axis_projection @@ -270,138 +265,3 @@ def _vlos_sq(field, data): p1res[setzero] = 0. p2res = p2.frb["gas", "velocity_los"] assert_rel_equal(p1res, p2res, 10) - - -@pytest.mark.parametrize("weighted", [True, False]) -@pytest.mark.parametrize("periodic", [True, False]) -@pytest.mark.parametrize("depth", [None, (1., "cm"), (0.5, "cm")]) -@pytest.mark.parametrize("shiftcenter", [False, True]) -@pytest.mark.parametrize("northvector", [None, (1.e-4, 1., 0.)]) -def test_sph_proj_general_offaxis( - northvector: tuple[float, float, float] | None, - shiftcenter: bool, - depth: tuple[float, str] | None, - periodic: bool, - weighted: bool) -> None: - ''' - Same as the on-axis projections, but we rotate the basis vectors - to test whether roations are handled ok. the rotation is chosen to - be small so that in/exclusion of particles within bboxes, etc. - works out the same way. - We just send lines of sight through pixel centers for convenience. - Particles at [0.5, 1.5, 2.5] (in each coordinate) - smoothing lengths 0.25 - all particles have mass 1., density 1.5, - except the single center particle, with mass 2., density 3. - - Parameters: - ----------- - northvector: tuple - y-axis direction in the final plot (direction vector) - shiftcenter: bool - shift the coordinates to center the projection on. - (The grid is offset to this same center) - depth: float or None - depth of the projection slice - periodic: bool - assume periodic boundary conditions, or not - weighted: bool - make a weighted projection (density-weighted density), or not - - Returns: - -------- - None - ''' - if shiftcenter: - center = np.array((0.625, 0.625, 0.625)) # cm - else: - center = np.array((1.5, 1.5, 1.5)) # cm - bbox = unyt.unyt_array(np.array([[0., 3.], [0., 3.], [0., 3.]]), 'cm') - hsml_factor = 0.5 - unitrho = 1.5 - # test correct centering, particle selection - def makemasses(i, j, k): - if i == j == k == 1: - return 2. - else: - return 1. - #dl_shift = _integrate_kernel(_cubicspline_python, - # np.sqrt(2) * 0.125, 0.25) - - # result shouldn't depend explicitly on the center if we re-center - # the data, unless we get cut-offs in the non-periodic case - # *almost* the z-axis - # try to make sure dl differences from periodic wrapping are small - epsilon = 1e-4 - projaxis = np.array([epsilon, 0.00, np.sqrt(1. - epsilon**2)]) - e1dir = projaxis / np.sqrt(np.sum(projaxis**2)) - # TODO: figure out other (default) axes for basis vectors here - if northvector is None: - e2dir = np.array([0., 1., 0.]) - else: - e2dir = np.asarray(northvector) - e2dir = e2dir - np.sum(e1dir * e2dir) * e2dir # orthonormalize - e2dir /= np.sqrt(np.sum(e2dir**2)) - e3dir = np.cross(e1dir, e2dir) - - ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, - periodic=periodic, - offsets=np.full(3, 0.5), - massgenerator=makemasses, - unitrho=unitrho, - bbox=bbox.v, - recenter=center, - e1hat=e1dir, e2hat=e2dir, e3hat=e3dir) - - source = ds.all_data() - # couple to dataset -> right unit registry - center = ds.arr(center, 'cm') - #print('position:\n', source['gas','position']) - - # m / rho, factor 1. / hsml**2 is included in the kernel integral - # (density is adjusted, so same for center particle) - prefactor = 1. / unitrho #/ (0.5 * 0.5)**2 - dl_cen = integrate_kernel(cubicspline_python, 0., 0.25) - - if weighted: - toweight_field = ("gas", "density") - else: - toweight_field = None - # we don't actually want a plot, it's just a straightforward, - # common way to get an frb / image array - prj = ProjectionPlot(ds, projaxis, ("gas", "density"), - width=(2.5, "cm"), - weight_field=toweight_field, - buff_size=(5, 5), - center=center, - data_source=source, - north_vector=northvector, - depth=depth) - img = prj.frb.data[('gas', 'density')] - if weighted: - # periodic shifts will modify the (relative) dl values a bit - expected_out = np.zeros((5, 5,), dtype=img.v.dtype) - expected_out[::2, ::2] = unitrho - if depth is None: - expected_out[2, 2] *= 1.5 - else: - # only 2 * unitrho element included - expected_out[2, 2] *= 2. - else: - expected_out = np.zeros((5, 5,), dtype=img.v.dtype) - expected_out[::2, ::2] = dl_cen * prefactor * unitrho - if depth is None: - # 3 particles per l.o.s., including the denser one - expected_out *= 3. - expected_out[2, 2] *= 4. / 3. - else: - # 1 particle per l.o.s., including the denser one - expected_out[2, 2] *= 2. - # grid is shifted to the left -> 'missing' stuff at the left - if (not periodic) and shiftcenter: - expected_out[:1, :] = 0. - expected_out[:, :1] = 0. - #print(axis, shiftcenter, depth, periodic, weighted) - print('expected:\n', expected_out) - print('recovered:\n', img.v) - assert_rel_equal(expected_out, img.v, 4) diff --git a/yt/visualization/tests/test_offaxisprojection_pytestonly.py b/yt/visualization/tests/test_offaxisprojection_pytestonly.py new file mode 100644 index 0000000000..1a252314ba --- /dev/null +++ b/yt/visualization/tests/test_offaxisprojection_pytestonly.py @@ -0,0 +1,143 @@ +import numpy as np +import pytest +import unyt + +from yt.testing import ( + assert_rel_equal, + cubicspline_python, + fake_sph_flexible_grid_ds, + integrate_kernel, +) +from yt.visualization.api import ProjectionPlot + +@pytest.mark.parametrize("weighted", [True, False]) +@pytest.mark.parametrize("periodic", [True, False]) +@pytest.mark.parametrize("depth", [None, (1., "cm"), (0.5, "cm")]) +@pytest.mark.parametrize("shiftcenter", [False, True]) +@pytest.mark.parametrize("northvector", [None, (1.e-4, 1., 0.)]) +def test_sph_proj_general_offaxis( + northvector: tuple[float, float, float] | None, + shiftcenter: bool, + depth: tuple[float, str] | None, + periodic: bool, + weighted: bool) -> None: + ''' + Same as the on-axis projections, but we rotate the basis vectors + to test whether roations are handled ok. the rotation is chosen to + be small so that in/exclusion of particles within bboxes, etc. + works out the same way. + We just send lines of sight through pixel centers for convenience. + Particles at [0.5, 1.5, 2.5] (in each coordinate) + smoothing lengths 0.25 + all particles have mass 1., density 1.5, + except the single center particle, with mass 2., density 3. + + Parameters: + ----------- + northvector: tuple + y-axis direction in the final plot (direction vector) + shiftcenter: bool + shift the coordinates to center the projection on. + (The grid is offset to this same center) + depth: float or None + depth of the projection slice + periodic: bool + assume periodic boundary conditions, or not + weighted: bool + make a weighted projection (density-weighted density), or not + + Returns: + -------- + None + ''' + if shiftcenter: + center = np.array((0.625, 0.625, 0.625)) # cm + else: + center = np.array((1.5, 1.5, 1.5)) # cm + bbox = unyt.unyt_array(np.array([[0., 3.], [0., 3.], [0., 3.]]), 'cm') + hsml_factor = 0.5 + unitrho = 1.5 + # test correct centering, particle selection + def makemasses(i, j, k): + if i == j == k == 1: + return 2. + else: + return 1. + + # result shouldn't depend explicitly on the center if we re-center + # the data, unless we get cut-offs in the non-periodic case + # *almost* the z-axis + # try to make sure dl differences from periodic wrapping are small + epsilon = 1e-4 + projaxis = np.array([epsilon, 0.00, np.sqrt(1. - epsilon**2)]) + e1dir = projaxis / np.sqrt(np.sum(projaxis**2)) + # TODO: figure out other (default) axes for basis vectors here + if northvector is None: + e2dir = np.array([0., 1., 0.]) + else: + e2dir = np.asarray(northvector) + e2dir = e2dir - np.sum(e1dir * e2dir) * e2dir # orthonormalize + e2dir /= np.sqrt(np.sum(e2dir**2)) + e3dir = np.cross(e1dir, e2dir) + + ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, + periodic=periodic, + offsets=np.full(3, 0.5), + massgenerator=makemasses, + unitrho=unitrho, + bbox=bbox.v, + recenter=center, + e1hat=e1dir, e2hat=e2dir, e3hat=e3dir) + + source = ds.all_data() + # couple to dataset -> right unit registry + center = ds.arr(center, 'cm') + #print('position:\n', source['gas','position']) + + # m / rho, factor 1. / hsml**2 is included in the kernel integral + # (density is adjusted, so same for center particle) + prefactor = 1. / unitrho #/ (0.5 * 0.5)**2 + dl_cen = integrate_kernel(cubicspline_python, 0., 0.25) + + if weighted: + toweight_field = ("gas", "density") + else: + toweight_field = None + # we don't actually want a plot, it's just a straightforward, + # common way to get an frb / image array + prj = ProjectionPlot(ds, projaxis, ("gas", "density"), + width=(2.5, "cm"), + weight_field=toweight_field, + buff_size=(5, 5), + center=center, + data_source=source, + north_vector=northvector, + depth=depth) + img = prj.frb.data[('gas', 'density')] + if weighted: + # periodic shifts will modify the (relative) dl values a bit + expected_out = np.zeros((5, 5,), dtype=img.v.dtype) + expected_out[::2, ::2] = unitrho + if depth is None: + expected_out[2, 2] *= 1.5 + else: + # only 2 * unitrho element included + expected_out[2, 2] *= 2. + else: + expected_out = np.zeros((5, 5,), dtype=img.v.dtype) + expected_out[::2, ::2] = dl_cen * prefactor * unitrho + if depth is None: + # 3 particles per l.o.s., including the denser one + expected_out *= 3. + expected_out[2, 2] *= 4. / 3. + else: + # 1 particle per l.o.s., including the denser one + expected_out[2, 2] *= 2. + # grid is shifted to the left -> 'missing' stuff at the left + if (not periodic) and shiftcenter: + expected_out[:1, :] = 0. + expected_out[:, :1] = 0. + #print(axis, shiftcenter, depth, periodic, weighted) + print('expected:\n', expected_out) + print('recovered:\n', img.v) + assert_rel_equal(expected_out, img.v, 4) From fa78f5c95d0bf4abeed6c24157ad7d60bf1909b5 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:33:06 -0500 Subject: [PATCH 115/186] remove trailing whitespace --- .../analyzing/generating_processed_data.rst | 4 +- .../coordinates/cartesian_coordinates.py | 28 ++++---- .../tests/test_sph_pixelization.py | 58 ++++++++-------- .../tests/test_sph_pixelization_pytestonly.py | 68 +++++++++---------- yt/testing.py | 52 +++++++------- yt/utilities/lib/pixelization_routines.pyx | 68 +++++++++---------- yt/visualization/plot_window.py | 14 ++-- .../tests/test_offaxisprojection.py | 10 +-- .../test_offaxisprojection_pytestonly.py | 32 ++++----- .../volume_rendering/off_axis_projection.py | 16 ++--- 10 files changed, 175 insertions(+), 175 deletions(-) diff --git a/doc/source/analyzing/generating_processed_data.rst b/doc/source/analyzing/generating_processed_data.rst index 5f7866b4ca..5f748d9e25 100644 --- a/doc/source/analyzing/generating_processed_data.rst +++ b/doc/source/analyzing/generating_processed_data.rst @@ -100,8 +100,8 @@ this, see :ref:`saving-grid-data-containers`. In the FITS case, there is an option for setting the ``units`` of the coordinate system in the file. If you want to overwrite a file with the same name, set ``clobber=True``. -The :class:`~yt.visualization.fixed_resolution.FixedResolutionBuffer` -(and its +The :class:`~yt.visualization.fixed_resolution.FixedResolutionBuffer` +(and its :class:`~yt.visualization.fixed_resolution.OffAxisProjectionFixedResolutionBuffer` subclass) can even be exported as a 2D dataset itself, which may be operated on in the same way as any other dataset in yt: diff --git a/yt/geometry/coordinates/cartesian_coordinates.py b/yt/geometry/coordinates/cartesian_coordinates.py index 96f43e0c4d..bf471b2b58 100644 --- a/yt/geometry/coordinates/cartesian_coordinates.py +++ b/yt/geometry/coordinates/cartesian_coordinates.py @@ -370,14 +370,14 @@ def _ortho_pixelize( ptype = data_source.ds._sph_ptypes[0] px_name = self.axis_name[self.x_axis[dim]] py_name = self.axis_name[self.y_axis[dim]] - # need z coordinates for depth, + # need z coordinates for depth, # but name isn't saved in the handler -> use the 'other one' pz_name = list(set(self.axis_order) - {px_name, py_name})[0] # ignore default True periodic argument - # (not actually supplied by a call from + # (not actually supplied by a call from # FixedResolutionBuffer), and use the dataset periodicity - # instead + # instead xa = self.x_axis[dim] ya = self.y_axis[dim] #axorder = data_source.ds.coordinates.axis_order @@ -422,10 +422,10 @@ def _ortho_pixelize( data_source=data_source.data_source, ) proj_reg.set_field_parameter("axis", data_source.axis) - # need some z bounds for SPH projection + # need some z bounds for SPH projection # -> use source bounds bnds3 = bnds + [le[za], re[za]] - + buff = np.zeros(size, dtype="float64") mask_uint8 = np.zeros_like(buff, dtype="uint8") if weight is None: @@ -537,7 +537,7 @@ def _ortho_pixelize( mask_uint8 = np.zeros_like(buff, dtype="uint8") if normalize: buff_den = np.zeros(size, dtype="float64") - + for chunk in data_source.chunks([], "io"): hsmlname = "smoothing_length" pixelize_sph_kernel_slice( @@ -576,7 +576,7 @@ def _ortho_pixelize( if normalize: normalization_2d_utility(buff, buff_den) - + mask = mask_uint8.astype("bool", copy=False) if smoothing_style == "gather": @@ -681,9 +681,9 @@ def _oblique_pixelize(self, data_source, field, bounds, size, antialias): is_sph_field = _finfo.is_sph_field particle_datasets = (ParticleDataset, StreamParticlesDataset) #finfo = self.ds._get_field_info(field) - - # SPH data - # only for slices: a function in off_axis_projection.py + + # SPH data + # only for slices: a function in off_axis_projection.py # handles projections if isinstance(data_source.ds, particle_datasets) and is_sph_field \ and isinstance(data_source, YTCuttingPlane): @@ -706,9 +706,9 @@ def _oblique_pixelize(self, data_source, field, bounds, size, antialias): if kernel_name is None: kernel_name = "cubic" # data_source should be a YTCuttingPlane object - # dimensionless unyt normal/north + # dimensionless unyt normal/north # -> numpy array cython can deal with - normal_vector = data_source.normal.v + normal_vector = data_source.normal.v north_vector = data_source._y_vec.v center = data_source.center.to("code_length") @@ -716,7 +716,7 @@ def _oblique_pixelize(self, data_source, field, bounds, size, antialias): mask_uint8 = np.zeros_like(buff, dtype="uint8") if normalize: buff_den = np.zeros(size, dtype="float64") - + for chunk in data_source.chunks([], "io"): pixelize_sph_kernel_cutting( buff, @@ -759,7 +759,7 @@ def _oblique_pixelize(self, data_source, field, bounds, size, antialias): # swap axes for image plotting mask = mask.swapaxes(0, 1) buff = buff.swapaxes(0, 1) - + # whatever other data this code could handle before the # SPH option was added else: diff --git a/yt/geometry/coordinates/tests/test_sph_pixelization.py b/yt/geometry/coordinates/tests/test_sph_pixelization.py index da5c9984bf..3ae3e9ee3d 100644 --- a/yt/geometry/coordinates/tests/test_sph_pixelization.py +++ b/yt/geometry/coordinates/tests/test_sph_pixelization.py @@ -2,7 +2,7 @@ import yt from yt.testing import ( - assert_rel_equal, + assert_rel_equal, cubicspline_python, fake_sph_flexible_grid_ds, integrate_kernel, @@ -55,17 +55,17 @@ def test_sph_projection_basic1(): particles at 0.5, 1.5, 2.5 ''' bbox = np.array([[0., 3.]] * 3) - ds = fake_sph_flexible_grid_ds(hsml_factor=1.0, nperside=3, + ds = fake_sph_flexible_grid_ds(hsml_factor=1.0, nperside=3, bbox=bbox) # works, but no depth control (at least without specific filters) - proj = ds.proj(("gas", "density"), 2) - frb = proj.to_frb(width=(2.5, 'cm'), + proj = ds.proj(("gas", "density"), 2) + frb = proj.to_frb(width=(2.5, 'cm'), resolution=(5, 5), height=(2.5, 'cm'), - center=np.array([1.5, 1.5, 1.5]), + center=np.array([1.5, 1.5, 1.5]), periodic=False) out = frb.get_image(('gas', 'density')) - + expected_out = np.zeros((5, 5), dtype=np.float64) dl_1part = integrate_kernel(cubicspline_python, 0., 0.5) linedens_1part = dl_1part * 1. # unit mass, density @@ -84,18 +84,18 @@ def test_sph_projection_basic2(): other pixels are still zero. ''' bbox = np.array([[0., 3.]] * 3) - ds = fake_sph_flexible_grid_ds(hsml_factor=0.5, nperside=3, + ds = fake_sph_flexible_grid_ds(hsml_factor=0.5, nperside=3, bbox=bbox) - proj = ds.proj(("gas", "density"), 2) - frb = proj.to_frb(width=(2.5, 'cm'), + proj = ds.proj(("gas", "density"), 2) + frb = proj.to_frb(width=(2.5, 'cm'), resolution=(5, 5), height=(2.5, 'cm'), - center=np.array([1.375, 1.375, 1.5]), + center=np.array([1.375, 1.375, 1.5]), periodic=False) out = frb.get_image(('gas', 'density')) - + expected_out = np.zeros((5, 5), dtype=np.float64) - dl_1part = integrate_kernel(cubicspline_python, + dl_1part = integrate_kernel(cubicspline_python, np.sqrt(2) * 0.125, 0.25) linedens_1part = dl_1part * 1. # unit mass, density @@ -109,7 +109,7 @@ def test_sph_projection_basic2(): def get_dataset_sphrefine(reflevel: int = 1): ''' - constant density particle grid, + constant density particle grid, with increasing particle sampling ''' lenfact = (1./3.)**(reflevel - 1) @@ -122,42 +122,42 @@ def get_dataset_sphrefine(reflevel: int = 1): hsml_factor = lenfact bbox = np.array([[0., 3.]] * 3) offsets = np.ones(3, dtype=np.float64) * 0.5 # in units of ehat - + def refmass(i: int, j: int, k: int) -> float: return massfact unitrho = 1. / massfact # want density 1 for decreasing mass - ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, + ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=nperside, - periodic=True, + periodic=True, e1hat=e1hat, - e2hat=e2hat, + e2hat=e2hat, e3hat=e3hat, offsets=offsets, massgenerator=refmass, unitrho=unitrho, - bbox=bbox, + bbox=bbox, ) - return ds + return ds def getdata_test_gridproj2(): # initial pixel centers at 0.5, 1., 1.5, 2., 2.5 # particles at 0.5, 1.5, 2.5 - # refine particle grid, check if pixel values remain the + # refine particle grid, check if pixel values remain the # same in the pixels passing through initial particle centers outlist = [] dss = [] for rl in range(1, 4): ds = get_dataset_sphrefine(reflevel=rl) - proj = ds.proj(("gas", "density"), 2) - frb = proj.to_frb(width=(2.5, 'cm'), + proj = ds.proj(("gas", "density"), 2) + frb = proj.to_frb(width=(2.5, 'cm'), resolution=(5, 5), height=(2.5, 'cm'), - center=np.array([1.5, 1.5, 1.5]), + center=np.array([1.5, 1.5, 1.5]), periodic=False) out = frb.get_image(('gas', 'density')) outlist.append(out) - dss.append(ds) + dss.append(ds) return outlist, dss def test_sph_gridproj_reseffect1(): @@ -179,16 +179,16 @@ def test_sph_gridproj_reseffect2(): refine the pixel grid instead of the particle grid ''' ds = get_dataset_sphrefine(reflevel=2) - proj = ds.proj(("gas", "density"), 2) + proj = ds.proj(("gas", "density"), 2) imgs = {} maxrl = 5 for rl in range(1, maxrl + 1): npix = 1 + 2**(rl + 1) margin = 0.5 - 0.5**(rl + 1) - frb = proj.to_frb(width=(3. - 2. * margin, 'cm'), + frb = proj.to_frb(width=(3. - 2. * margin, 'cm'), resolution=(npix, npix), height=(3. - 2. * margin, 'cm'), - center=np.array([1.5, 1.5, 1.5]), + center=np.array([1.5, 1.5, 1.5]), periodic=False) out = frb.get_image(('gas', 'density')) imgs[rl] = out @@ -197,6 +197,6 @@ def test_sph_gridproj_reseffect2(): for rl in imgs: img = imgs[rl] pixspace = 2**(rl) - #print(f'Grid refinement level {rl}:') - assert_rel_equal(img[::pixspace, ::pixspace], + #print(f'Grid refinement level {rl}:') + assert_rel_equal(img[::pixspace, ::pixspace], ref[::pixspace_ref, ::pixspace_ref], 4) diff --git a/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py b/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py index 4d299e2516..af95fde034 100644 --- a/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py +++ b/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py @@ -5,7 +5,7 @@ import yt from yt.data_objects.selection_objects.region import YTRegion from yt.testing import ( - assert_rel_equal, + assert_rel_equal, cubicspline_python, distancematrix, fake_random_sph_ds, @@ -24,20 +24,20 @@ def test_sph_proj_general_alongaxes(axis: int, periodic: bool, weighted: bool) -> None: ''' - The previous projection tests were for a specific issue. + The previous projection tests were for a specific issue. Here, we test more functionality of the projections. We just send lines of sight through pixel centers for convenience. Particles at [0.5, 1.5, 2.5] (in each coordinate) smoothing lengths 0.25 - all particles have mass 1., density 1.5, + all particles have mass 1., density 1.5, except the single center particle, with mass 2., density 3. - + Parameters: ----------- axis: {0, 1, 2} projection axis (aligned with sim. axis) shiftcenter: bool - shift the coordinates to center the projection on. + shift the coordinates to center the projection on. (The grid is offset to this same center) depth: float or None depth of the projection slice @@ -60,7 +60,7 @@ def test_sph_proj_general_alongaxes(axis: int, # test correct centering, particle selection def makemasses(i, j, k): if i == j == k == 1: - return 2. + return 2. else: return 1. # m / rho, factor 1. / hsml**2 is included in the kernel integral @@ -69,13 +69,13 @@ def makemasses(i, j, k): dl_cen = integrate_kernel(cubicspline_python, 0., 0.25) # result shouldn't depend explicitly on the center if we re-center - # the data, unless we get cut-offs in the non-periodic case + # the data, unless we get cut-offs in the non-periodic case ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, - periodic=periodic, + periodic=periodic, offsets=np.full(3, 0.5), massgenerator=makemasses, unitrho=unitrho, - bbox=bbox.v, + bbox=bbox.v, recenter=center.v) if depth is None: source = ds.all_data() @@ -92,9 +92,9 @@ def makemasses(i, j, k): ds=ds) source = reg - # we don't actually want a plot, it's just a straightforward, + # we don't actually want a plot, it's just a straightforward, # common way to get an frb / image array - if weighted: + if weighted: toweight_field = ("gas", "density") else: toweight_field = None @@ -108,7 +108,7 @@ def makemasses(i, j, k): if weighted: expected_out = np.zeros((5, 5,), dtype=img.v.dtype) expected_out[::2, ::2] = unitrho - if depth is None: + if depth is None: ## during shift, particle coords do wrap around edges #if (not periodic) and shiftcenter: # # weight 1. for unitrho, 2. for 2. * untrho @@ -130,7 +130,7 @@ def makemasses(i, j, k): # 1 particle per l.o.s., including the denser one expected_out[2, 2] *= 2. # grid is shifted to the left -> 'missing' stuff at the left - if (not periodic) and shiftcenter: + if (not periodic) and shiftcenter: expected_out[:1, :] = 0. expected_out[:, :1] = 0. #print(axis, shiftcenter, depth, periodic, weighted) @@ -149,9 +149,9 @@ def test_sph_slice_general_alongaxes(axis: int, ''' Particles at [0.5, 1.5, 2.5] (in each coordinate) smoothing lengths 0.25 - all particles have mass 1., density 1.5, + all particles have mass 1., density 1.5, except the single center particle, with mass 2., density 3. - + Parameters: ----------- axis: {0, 1, 2} @@ -159,7 +159,7 @@ def test_sph_slice_general_alongaxes(axis: int, northvector: tuple y-axis direction in the final plot (direction vector) shiftcenter: bool - shift the coordinates to center the projection on. + shift the coordinates to center the projection on. (The grid is offset to this same center) zoff: float offset of the slice plane from the SPH particle center plane @@ -180,36 +180,36 @@ def test_sph_slice_general_alongaxes(axis: int, # test correct centering, particle selection def makemasses(i, j, k): if i == j == k == 1: - return 2. + return 2. elif i == j == k == 2: return 3. else: return 1. # result shouldn't depend explicitly on the center if we re-center - # the data, unless we get cut-offs in the non-periodic case + # the data, unless we get cut-offs in the non-periodic case ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, - periodic=periodic, + periodic=periodic, offsets=np.full(3, 0.5), massgenerator=makemasses, unitrho=unitrho, - bbox=bbox.v, + bbox=bbox.v, recenter=center.v) ad = ds.all_data() #print(ad[('gas', 'position')]) outgridsize = 10 width = 2.5 _center = center.to("cm").v.copy() - _center[axis] += zoff + _center[axis] += zoff - # we don't actually want a plot, it's just a straightforward, + # we don't actually want a plot, it's just a straightforward, # common way to get an frb / image array slc = yt.SlicePlot(ds, axis, ("gas", "density"), width=(width, "cm"), buff_size=(outgridsize,) * 2, center=(_center, "cm")) img = slc.frb.data[('gas', 'density')] - + # center is same in non-projection coords if axis == 0: ci = 1 @@ -278,25 +278,25 @@ def test_sph_slice_general_offaxis( ''' Same as the on-axis slices, but we rotate the basis vectors to test whether roations are handled ok. the rotation is chosen to - be small so that in/exclusion of particles within bboxes, etc. + be small so that in/exclusion of particles within bboxes, etc. works out the same way. Particles at [0.5, 1.5, 2.5] (in each coordinate) smoothing lengths 0.25 - all particles have mass 1., density 1.5, + all particles have mass 1., density 1.5, except the single center particle, with mass 2., density 3. - + Parameters: ----------- northvector: tuple y-axis direction in the final plot (direction vector) shiftcenter: bool - shift the coordinates to center the projection on. + shift the coordinates to center the projection on. (The grid is offset to this same center) zoff: float offset of the slice plane from the SPH particle center plane periodic: bool assume periodic boundary conditions, or not - + Returns: -------- None @@ -311,11 +311,11 @@ def test_sph_slice_general_offaxis( # test correct centering, particle selection def makemasses(i, j, k): if i == j == k == 1: - return 2. + return 2. else: return 1. # try to make sure dl differences from periodic wrapping are small - epsilon = 1e-4 + epsilon = 1e-4 projaxis = np.array([epsilon, 0.00, np.sqrt(1. - epsilon**2)]) e1dir = projaxis / np.sqrt(np.sum(projaxis**2)) if northvector is None: @@ -332,11 +332,11 @@ def makemasses(i, j, k): _center += zoff * e1dir ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, - periodic=periodic, + periodic=periodic, offsets=np.full(3, 0.5), massgenerator=makemasses, unitrho=unitrho, - bbox=bbox.v, + bbox=bbox.v, recenter=center, e1hat=e1dir, e2hat=e2dir, e3hat=e3dir) @@ -361,7 +361,7 @@ def makemasses(i, j, k): xgrid = np.repeat(gridcenx, outgridsize) ygrid = np.tile(gridceny, outgridsize) zgrid = np.full(outgridsize**2, np.dot(_center, e1dir)) - gridcoords = (xgrid[:, np.newaxis] * e3dir[np.newaxis, :] + gridcoords = (xgrid[:, np.newaxis] * e3dir[np.newaxis, :] + ygrid[:, np.newaxis] * e2dir[np.newaxis, :] + zgrid[:, np.newaxis] * e1dir[np.newaxis, :]) print('gridcoords:') @@ -425,7 +425,7 @@ def test_sph_grid(periodic: bool | tuple[bool, bool, bool], xcens = 0.5 * (xedges[:-1] + xedges[1:]) ycens = 0.5 * (yedges[:-1] + yedges[1:]) zcens = 0.5 * (zedges[:-1] + zedges[1:]) - + ad = ds.all_data() sphcoords = np.array([(ad[("gas", "x")]).to("cm"), (ad[("gas", "y")]).to("cm"), diff --git a/yt/testing.py b/yt/testing.py index 68b6cc4619..40b185010e 100644 --- a/yt/testing.py +++ b/yt/testing.py @@ -86,12 +86,12 @@ def assert_rel_equal(a1, a2, decimals, err_msg="", verbose=True): # tested: volume integral is 1. def cubicspline_python(x: float) -> float: ''' - cubic spline SPH kernel function for testing against more + cubic spline SPH kernel function for testing against more effiecient cython methods Parameters ---------- - x: + x: impact parameter / smoothing length [dimenionless] Returns @@ -113,7 +113,7 @@ def integrate_kernel(kernelfunc: Callable[[float], float], """ integrates a kernel function over a line passing entirely through it - + Parameters: ----------- kernelfunc: @@ -122,10 +122,10 @@ def integrate_kernel(kernelfunc: Callable[[float], float], impact parameter hsml: smoothing length [same units as impact parameter] - + Returns: -------- - the integral of the SPH kernel function. + the integral of the SPH kernel function. units: 1 / units of b and hsml """ pre = 1. / hsml**2 @@ -155,9 +155,9 @@ def distancematrix(pos3_i0: np.ndarray[float], for positions along the different cartesian axes pos3_i1: shape (second number of points, 3) as pos3_i0, but for the second set of points - periodic: + periodic: are the positions along each axis periodic (True) or not - periods: + periods: the periods along each axis. Ignored if positions in a given direction are not periodic. @@ -796,14 +796,14 @@ def constantmass(i: int, j: int, k: int) -> float: def fake_sph_flexible_grid_ds( hsml_factor: float = 1.0, nperside: int = 3, - periodic: bool = True, + periodic: bool = True, e1hat: np.ndarray[float] = _xhat, e2hat: np.ndarray[float] = _yhat, e3hat: np.ndarray[float] = _zhat, offsets: np.ndarray[float] = _floathalves, massgenerator: Callable[[int, int, int], float] = constantmass, unitrho: float = 1., - bbox: np.ndarray | None = None, + bbox: np.ndarray | None = None, recenter: np.ndarray | None = None, ) -> StreamParticlesDataset: """Returns an in-memory SPH dataset useful for testing @@ -818,10 +818,10 @@ def fake_sph_flexible_grid_ds( periodic: are the positions taken to be periodic? (applies to all coordinate axes) - e1hat: shape (3,) + e1hat: shape (3,) the first basis vector defining the 3D grid. If the basis vectors are not normalized to 1 or not orthogonal, the spacing - or overlap between SPH particles will be affected, but this is + or overlap between SPH particles will be affected, but this is allowed. e2hat: shape (3,) the second basis vector defining the 3D grid. (See `e1hat`.) @@ -831,27 +831,27 @@ def fake_sph_flexible_grid_ds( the the zero point of the 3D grid along each coordinate axis massgenerator: a function assigning a mass to each particle, as a function of - the e[1-3]hat indices, in order + the e[1-3]hat indices, in order unitrho: defines the density for a particle with mass 1 ('g'), and the standard (uniform) grid `hsml_factor`. bbox: if np.ndarray, shape is (2, 3) the assumed enclosing volume of the particles. Should enclose all the coordinate values. If not specified, a bbox is defined - which encloses all coordinates values with a margin. If + which encloses all coordinates values with a margin. If `periodic`, the size of the `bbox` along each coordinate is also the period along that axis. recenter: if not `None`, after generating the grid, the positions are - periodically shifted to move the old center to this positions. - Useful for testing periodicity handling. + periodically shifted to move the old center to this positions. + Useful for testing periodicity handling. This shift is relative to the halfway positions of the bbox edges. Returns: -------- A `StreamParticlesDataset` object with particle positions, masses, - velocities (zero), smoothing lengths, and densities specified. + velocities (zero), smoothing lengths, and densities specified. Values are in cgs units. """ @@ -878,15 +878,15 @@ def fake_sph_flexible_grid_ds( [np.min(pos[:, 1]) - margin, np.max(pos[:, 1]) + margin], [np.min(pos[:, 2]) - margin, - np.max(pos[:, 2]) + margin], + np.max(pos[:, 2]) + margin], ]) - + if recenter is not None: periods = bbox[:, 1] - bbox[:, 0] # old center -> new position pos += -0.5 * periods[np.newaxis, :] + recenter[np.newaxis, :] # wrap coordinates -> all in [0, boxsize) range - pos %= periods[np.newaxis, :] + pos %= periods[np.newaxis, :] # shift back to original bbox range pos += (bbox[:, 0])[np.newaxis, :] if not periodic: @@ -910,7 +910,7 @@ def fake_sph_flexible_grid_ds( "smoothing_length": (np.ones(npart) * 0.5 * hsml_factor, "cm"), "density": (rho[okinds], "g/cm**3"), } - + ds = load_particles(data=data, bbox=bbox, periodicity=(periodic,) * 3, length_unit=1., mass_unit=1., time_unit=1., @@ -919,7 +919,7 @@ def fake_sph_flexible_grid_ds( return ds -def fake_random_sph_ds(npart: int, bbox: np.ndarray, +def fake_random_sph_ds(npart: int, bbox: np.ndarray, periodic: bool | tuple[bool, bool, bool] = True, massrange: tuple[float, float] = (0.5, 2.), hsmlrange: tuple[float, float] = (0.5, 2.), @@ -931,24 +931,24 @@ def fake_random_sph_ds(npart: int, bbox: np.ndarray, ----------- npart: number of particles to generate - bbox: shape: (3, 2), units: "cm" + bbox: shape: (3, 2), units: "cm" the assumed enclosing volume of the particles. Particle positions are drawn uniformly from these ranges. periodic: - are the positions taken to be periodic? If a single value, + are the positions taken to be periodic? If a single value, that value is applied to all axes massrange: particle masses are drawn uniformly from this range (unit: "g") hsmlrange: units: "cm" particle smoothing lengths are drawn uniformly from this range unitrho: - defines the density for a particle with mass 1 ("g"), and + defines the density for a particle with mass 1 ("g"), and smoothing length 1 ("cm"). Returns: -------- A `StreamParticlesDataset` object with particle positions, masses, - velocities (zero), smoothing lengths, and densities specified. + velocities (zero), smoothing lengths, and densities specified. Values are in cgs units. """ @@ -974,7 +974,7 @@ def fake_random_sph_ds(npart: int, bbox: np.ndarray, "smoothing_length": (hsml, "cm"), "density": (dens, "g/cm**3"), } - + ds = load_particles(data=data, bbox=bbox, periodicity=periodic, length_unit=1., mass_unit=1., time_unit=1., diff --git a/yt/utilities/lib/pixelization_routines.pyx b/yt/utilities/lib/pixelization_routines.pyx index 7469d492af..b1f7e620d5 100644 --- a/yt/utilities/lib/pixelization_routines.pyx +++ b/yt/utilities/lib/pixelization_routines.pyx @@ -1144,7 +1144,7 @@ def pixelize_sph_kernel_projection( cdef np.float64_t[:] _weight_field cdef int * xiter cdef int * yiter - cdef int * ziter + cdef int * ziter cdef np.float64_t * xiterv cdef np.float64_t * yiterv cdef np.float64_t * ziterv @@ -1205,7 +1205,7 @@ def pixelize_sph_kernel_projection( xiterv[0] = yiterv[0] = ziterv[0] = 0.0 for i in range(xsize * ysize): local_buff[i] = 0.0 - + for j in prange(0, posx.shape[0], schedule="dynamic"): if j % 100000 == 0: with gil: @@ -1237,16 +1237,16 @@ def pixelize_sph_kernel_projection( # we set the smoothing length squared with lower limit of the pixel # Nope! that causes weird grid resolution dependences and increases - # total values when resolution elements have hsml < grid spacing + # total values when resolution elements have hsml < grid spacing h_j2 = hsml[j]*hsml[j] ih_j2 = 1.0/h_j2 prefactor_j = pmass[j] / pdens[j] / hsml[j]**2 * quantity_to_smooth[j] if weight_field is not None: prefactor_j *= _weight_field[j] - + # Discussion point: do we want the hsml margin on the z direction? - # it's consistent with Ray and Region selections, I think, + # it's consistent with Ray and Region selections, I think, # but does tend to 'tack on' stuff compared to the nominal depth for kk in range(2): # discard if z is outside bounds @@ -1493,13 +1493,13 @@ def interpolate_sph_grid_gather(np.float64_t[:, :, :] buff, def pixelize_sph_kernel_slice( np.float64_t[:, :] buff, np.uint8_t[:, :] mask, - np.float64_t[:] posx, np.float64_t[:] posy, - np.float64_t[:] posz, + np.float64_t[:] posx, np.float64_t[:] posy, + np.float64_t[:] posz, np.float64_t[:] hsml, np.float64_t[:] pmass, np.float64_t[:] pdens, np.float64_t[:] quantity_to_smooth, bounds, - np.float64_t slicez, + np.float64_t slicez, kernel_name="cubic", _check_period = (1, 1, 1), period=None): @@ -1511,7 +1511,7 @@ def pixelize_sph_kernel_slice( #print(period) #print() # bounds are [x0, x1, y0, y1], slicez is the single coordinate - # of the slice along the normal direction. + # of the slice along the normal direction. # similar method to pixelize_sph_kernel_projection cdef np.intp_t xsize, ysize cdef np.float64_t x_min, x_max, y_min, y_max, prefactor_j @@ -1532,7 +1532,7 @@ def pixelize_sph_kernel_slice( period_z = period[2] for i in range(3): check_period[i] = np.int8(_check_period[i]) - + xsize, ysize = buff.shape[0], buff.shape[1] x_min = bounds[0] x_max = bounds[1] @@ -1639,8 +1639,8 @@ def pixelize_sph_kernel_slice( continue # see equation 4 of the SPLASH paper - q_ij = math.sqrt(posx_diff - + posy_diff + q_ij = math.sqrt(posx_diff + + posy_diff + posz_diff) * ih_j if q_ij >= 1: continue @@ -1675,7 +1675,7 @@ def pixelize_sph_kernel_arbitrary_grid(np.float64_t[:, :, :] buff, cdef np.float64_t x_min, x_max, y_min, y_max, z_min, z_max, prefactor_j cdef np.int64_t xi, yi, zi, x0, x1, y0, y1, z0, z1 cdef np.float64_t q_ij, posx_diff, posy_diff, posz_diff, px, py, pz - cdef np.float64_t x, y, z, dx, dy, dz, idx, idy, idz, h_j2, h_j, ih_j + cdef np.float64_t x, y, z, dx, dy, dz, idx, idy, idz, h_j2, h_j, ih_j # cdef np.float64_t h_j3 cdef int j, ii, jj, kk cdef np.float64_t period_x = 0, period_y = 0, period_z = 0 @@ -1722,16 +1722,16 @@ def pixelize_sph_kernel_arbitrary_grid(np.float64_t[:, :, :] buff, idz = 1.0/dz kernel = get_kernel_func(kernel_name) - - # nogil seems dangerous here, but there are no actual parallel + + # nogil seems dangerous here, but there are no actual parallel # sections (e.g., prange instead of range) used here. # However, for future writers: - # !! the final buff array mutation has no protections against + # !! the final buff array mutation has no protections against # !! race conditions (e.g., OpenMP's atomic read/write), and # !! cython doesn't seem to provide such options. # (other routines in this file use private variable buffer arrays - # and add everything together at the end, but grid arrays can get - # big fast, and having such a large array in each thread could + # and add everything together at the end, but grid arrays can get + # big fast, and having such a large array in each thread could # cause memory use issues.) with nogil: # TODO make this parallel without using too much memory @@ -1830,14 +1830,14 @@ def pixelize_sph_kernel_arbitrary_grid(np.float64_t[:, :, :] buff, continue # see equation 4 of the SPLASH paper - q_ij = math.sqrt(posx_diff + q_ij = math.sqrt(posx_diff + posy_diff + posz_diff) * ih_j if q_ij >= 1: continue # shared variable buff should not # be mutatated in a nogil section - # where different threads may change + # where different threads may change # the same array element buff[xi, yi, zi] += prefactor_j \ * kernel(q_ij) @@ -2000,7 +2000,7 @@ def rotate_particle_coord_pib(np.float64_t[:] px, return px_rotated, py_rotated, rot_bounds_x0, rot_bounds_x1, rot_bounds_y0, rot_bounds_y1 # version intended for SPH off-axis slices/projections -# includes dealing with periodic boundaries, but also +# includes dealing with periodic boundaries, but also # shifts particles so center -> origin. # therefore, don't want to use this in the ParticleImageBuffer, # which expects differently centered coordinates. @@ -2055,14 +2055,14 @@ def rotate_particle_coord(np.float64_t[:] px, coordinate_matrix[0] = px[i] coordinate_matrix[1] = py[i] coordinate_matrix[2] = pz[i] - - # centering: + + # centering: # make sure this also works for centers close to periodic edges # added consequence: the center is placed at the origin # (might as well keep it there in these temporary coordinates) for ax in range(3): # assumed center is zero even if non-periodic - coordinate_matrix[ax] -= center[ax] + coordinate_matrix[ax] -= center[ax] if not periodic[ax]: continue period = bounds[2 * ax + 1] - bounds[2 * ax] # abs. difference between points in the volume is <= period @@ -2077,9 +2077,9 @@ def rotate_particle_coord(np.float64_t[:] px, py_rotated[i] = rotated_coordinates[1] pz_rotated[i] = rotated_coordinates[2] - return (px_rotated, py_rotated, pz_rotated, - rot_bounds_x0, rot_bounds_x1, - rot_bounds_y0, rot_bounds_y1, + return (px_rotated, py_rotated, pz_rotated, + rot_bounds_x0, rot_bounds_x1, + rot_bounds_y0, rot_bounds_y1, rot_bounds_z0, rot_bounds_z1) @@ -2103,7 +2103,7 @@ def off_axis_projection_SPH(np.float64_t[:] px, weight_field=None, depth=None, kernel_name="cubic"): - # periodic: periodicity of the data set: + # periodic: periodicity of the data set: # Do nothing in event of a 0 normal vector if np.allclose(normal_vector, 0.): return @@ -2122,7 +2122,7 @@ def off_axis_projection_SPH(np.float64_t[:] px, normal_vector, north_vector) # check_period=0: assumed to be a small region compared to the box - # size. The rotation already ensures that a center close to a + # size. The rotation already ensures that a center close to a # periodic edge works out fine. # since the simple single-coordinate modulo math periodicity # does not apply to the *rotated* coordinates, the periodicity @@ -2149,8 +2149,8 @@ def off_axis_projection_SPH(np.float64_t[:] px, def pixelize_sph_kernel_cutting( np.float64_t[:, :] buff, np.uint8_t[:, :] mask, - np.float64_t[:] posx, np.float64_t[:] posy, - np.float64_t[:] posz, + np.float64_t[:] posx, np.float64_t[:] posy, + np.float64_t[:] posz, np.float64_t[:] hsml, np.float64_t[:] pmass, np.float64_t[:] pdens, np.float64_t[:] quantity_to_smooth, @@ -2162,7 +2162,7 @@ def pixelize_sph_kernel_cutting( if check_period == 0: periodic = np.zeros(3, dtype=bool) - + posx_rot, posy_rot, posz_rot, \ rot_bounds_x0, rot_bounds_x1, \ rot_bounds_y0, rot_bounds_y1, \ @@ -2172,13 +2172,13 @@ def pixelize_sph_kernel_cutting( widthxy, 0., normal_vector, north_vector) - bounds_rot = np.array([rot_bounds_x0, rot_bounds_x1, + bounds_rot = np.array([rot_bounds_x0, rot_bounds_x1, rot_bounds_y0, rot_bounds_y1]) slicez_rot = rot_bounds_z0 pixelize_sph_kernel_slice(buff, mask, posx_rot, posy_rot, posz_rot, hsml, pmass, pdens, quantity_to_smooth, - bounds_rot, slicez_rot, + bounds_rot, slicez_rot, kernel_name=kernel_name, _check_period=np.zeros(3, dtype="int"), period=None) diff --git a/yt/visualization/plot_window.py b/yt/visualization/plot_window.py index 68db155ad1..5afd02bd09 100644 --- a/yt/visualization/plot_window.py +++ b/yt/visualization/plot_window.py @@ -90,7 +90,7 @@ def get_oblique_window_parameters(normal, center, width, ds, if len(width) == 2: # Transforming to the cutting plane coordinate system - # the original dimensionless center messes up off-axis + # the original dimensionless center messes up off-axis # SPH projections though -> don't use this center there center = ((center - ds.domain_left_edge) / ds.domain_width - 0.5)\ * ds.domain_width @@ -2469,18 +2469,18 @@ def __init__( # units match bounds # for SPH data, we want to input the original center # the cython backend handles centering to this point and - # rotation. + # rotation. # get3bounds gets a depth 0.5 * diagonal + margin in the - # depth=None case. + # depth=None case. (bounds, center_rot) = get_oblique_window_parameters( normal, center, width, ds, depth=depth, get3bounds=True, ) # will probably fail if you try to project an SPH and non-SPH # field in a single call - # checks for SPH fields copied from the + # checks for SPH fields copied from the # _ortho_pixelize method in cartesian_coordinates.py - - ## data_source might be None here + + ## data_source might be None here ## (OffAxisProjectionDummyDataSource gets used later) if data_source is None: data_source = ds.all_data() @@ -2488,7 +2488,7 @@ def __init__( finfo = data_source.ds.field_info[field] is_sph_field = finfo.is_sph_field particle_datasets = (ParticleDataset, StreamParticlesDataset) - + if isinstance(data_source.ds, particle_datasets) and is_sph_field: center_use = parse_center_array(center, ds=data_source.ds, axis=None) diff --git a/yt/visualization/tests/test_offaxisprojection.py b/yt/visualization/tests/test_offaxisprojection.py index d67fe4c4c5..89a610f298 100644 --- a/yt/visualization/tests/test_offaxisprojection.py +++ b/yt/visualization/tests/test_offaxisprojection.py @@ -14,7 +14,7 @@ fake_random_ds, ) from yt.visualization.api import ( - OffAxisProjectionPlot, + OffAxisProjectionPlot, OffAxisSlicePlot, ) from yt.visualization.image_writer import write_projection @@ -242,7 +242,7 @@ def _vlos_sq(field, data): ## marginally < 0, resulting in unmatched NaN values in the ## first assert_rel_equal argument. The compute_stddev_image ## function used in OffAxisProjectionPlot checks for and deals - ## with these cases. + ## with these cases. #assert_rel_equal( # np.sqrt( # p1.frb["gas", "velocity_los_squared"] - p1.frb["gas", "velocity_los"] ** 2 @@ -254,10 +254,10 @@ def _vlos_sq(field, data): p1_sqexp = p1.frb["gas", "velocity_los"] ** 2 p1res = np.sqrt(p1_expsq - p1_sqexp) # set values to zero that have **2 - **2 < 0, but - # the absolute values are much smaller than the smallest - # postive values of **2 and **2 + # the absolute values are much smaller than the smallest + # postive values of **2 and **2 # (i.e., the difference is pretty much zero) - mindiff = 1e-10 * min(np.min(p1_expsq[p1_expsq > 0]), + mindiff = 1e-10 * min(np.min(p1_expsq[p1_expsq > 0]), np.min(p1_sqexp[p1_sqexp > 0])) print(mindiff) setzero = np.logical_and(p1_expsq - p1_sqexp < 0, diff --git a/yt/visualization/tests/test_offaxisprojection_pytestonly.py b/yt/visualization/tests/test_offaxisprojection_pytestonly.py index 1a252314ba..5a1f013313 100644 --- a/yt/visualization/tests/test_offaxisprojection_pytestonly.py +++ b/yt/visualization/tests/test_offaxisprojection_pytestonly.py @@ -19,25 +19,25 @@ def test_sph_proj_general_offaxis( northvector: tuple[float, float, float] | None, shiftcenter: bool, depth: tuple[float, str] | None, - periodic: bool, + periodic: bool, weighted: bool) -> None: ''' Same as the on-axis projections, but we rotate the basis vectors to test whether roations are handled ok. the rotation is chosen to - be small so that in/exclusion of particles within bboxes, etc. - works out the same way. + be small so that in/exclusion of particles within bboxes, etc. + works out the same way. We just send lines of sight through pixel centers for convenience. Particles at [0.5, 1.5, 2.5] (in each coordinate) smoothing lengths 0.25 - all particles have mass 1., density 1.5, + all particles have mass 1., density 1.5, except the single center particle, with mass 2., density 3. - + Parameters: ----------- northvector: tuple y-axis direction in the final plot (direction vector) shiftcenter: bool - shift the coordinates to center the projection on. + shift the coordinates to center the projection on. (The grid is offset to this same center) depth: float or None depth of the projection slice @@ -45,7 +45,7 @@ def test_sph_proj_general_offaxis( assume periodic boundary conditions, or not weighted: bool make a weighted projection (density-weighted density), or not - + Returns: -------- None @@ -60,15 +60,15 @@ def test_sph_proj_general_offaxis( # test correct centering, particle selection def makemasses(i, j, k): if i == j == k == 1: - return 2. + return 2. else: return 1. # result shouldn't depend explicitly on the center if we re-center - # the data, unless we get cut-offs in the non-periodic case + # the data, unless we get cut-offs in the non-periodic case # *almost* the z-axis # try to make sure dl differences from periodic wrapping are small - epsilon = 1e-4 + epsilon = 1e-4 projaxis = np.array([epsilon, 0.00, np.sqrt(1. - epsilon**2)]) e1dir = projaxis / np.sqrt(np.sum(projaxis**2)) # TODO: figure out other (default) axes for basis vectors here @@ -81,11 +81,11 @@ def makemasses(i, j, k): e3dir = np.cross(e1dir, e2dir) ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, - periodic=periodic, + periodic=periodic, offsets=np.full(3, 0.5), massgenerator=makemasses, unitrho=unitrho, - bbox=bbox.v, + bbox=bbox.v, recenter=center, e1hat=e1dir, e2hat=e2dir, e3hat=e3dir) @@ -99,11 +99,11 @@ def makemasses(i, j, k): prefactor = 1. / unitrho #/ (0.5 * 0.5)**2 dl_cen = integrate_kernel(cubicspline_python, 0., 0.25) - if weighted: + if weighted: toweight_field = ("gas", "density") else: toweight_field = None - # we don't actually want a plot, it's just a straightforward, + # we don't actually want a plot, it's just a straightforward, # common way to get an frb / image array prj = ProjectionPlot(ds, projaxis, ("gas", "density"), width=(2.5, "cm"), @@ -118,7 +118,7 @@ def makemasses(i, j, k): # periodic shifts will modify the (relative) dl values a bit expected_out = np.zeros((5, 5,), dtype=img.v.dtype) expected_out[::2, ::2] = unitrho - if depth is None: + if depth is None: expected_out[2, 2] *= 1.5 else: # only 2 * unitrho element included @@ -134,7 +134,7 @@ def makemasses(i, j, k): # 1 particle per l.o.s., including the denser one expected_out[2, 2] *= 2. # grid is shifted to the left -> 'missing' stuff at the left - if (not periodic) and shiftcenter: + if (not periodic) and shiftcenter: expected_out[:1, :] = 0. expected_out[:, :1] = 0. #print(axis, shiftcenter, depth, periodic, weighted) diff --git a/yt/visualization/volume_rendering/off_axis_projection.py b/yt/visualization/volume_rendering/off_axis_projection.py index 44f03ade0c..b3f579b9cd 100644 --- a/yt/visualization/volume_rendering/off_axis_projection.py +++ b/yt/visualization/volume_rendering/off_axis_projection.py @@ -82,8 +82,8 @@ def off_axis_projection( A vector that, if specified, restricts the orientation such that the north vector dotted into the image plane points "up". Useful for rotations depth: float, tuple[float, str], or unyt_array or size 1. - specify the depth of the projection region (size along the - line of sight). If no units are given (unyt_array or second + specify the depth of the projection region (size along the + line of sight). If no units are given (unyt_array or second tuple element), code units are assumed. num_threads: integer, optional, default 1 Use this many OpenMP threads during projection. @@ -151,16 +151,16 @@ def off_axis_projection( if not hasattr(width, "units"): width = data_source.ds.arr(width, "code_length") if depth is not None: - # handle units (intrinsic or as a tuple), + # handle units (intrinsic or as a tuple), # then convert to code length # float -> assumed to be in code units if isinstance(depth, tuple): depth = data_source.ds.arr(np.array([depth[0]]), depth[1]) if hasattr(depth, "units"): depth = depth.to("code_length").d - + #depth = data_source.ds.arr(depth, "code_length") - + if hasattr(data_source.ds, "_sph_ptypes"): if method != "integrate": @@ -218,7 +218,7 @@ def off_axis_projection( # if weight is None: buf = np.zeros((resolution[0], resolution[1]), dtype="float64") mask = np.ones_like(buf, dtype="uint8") - + ## width from fixed_resolution.py is just the size of the domain #x_min = center[0] - width[0] / 2 #x_max = center[0] + width[0] / 2 @@ -226,7 +226,7 @@ def off_axis_projection( #y_max = center[1] + width[1] / 2 #z_min = center[2] - width[2] / 2 #z_max = center[2] + width[2] / 2 - + periodic = data_source.ds.periodicity le = data_source.ds.domain_left_edge.to("code_length").d re = data_source.ds.domain_right_edge.to("code_length").d @@ -234,7 +234,7 @@ def off_axis_projection( x_max, y_max, z_max = re bounds = [x_min, x_max, y_min, y_max, z_min, z_max] # only need (rotated) x/y widths - _width = (width.to("code_length").d)[:2] + _width = (width.to("code_length").d)[:2] finfo = data_source.ds.field_info[item] ounits = finfo.output_units kernel_name = None From 785108afc557361f3bcfd962ce4a32ef766dbadf Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Thu, 18 Jul 2024 15:13:19 -0500 Subject: [PATCH 116/186] add typing option of (array, unit) values --- yt/loaders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/loaders.py b/yt/loaders.py index 54d52e5b83..aa7abea55f 100644 --- a/yt/loaders.py +++ b/yt/loaders.py @@ -694,7 +694,7 @@ def load_amr_grids( def load_particles( - data: dict[AnyFieldKey, np.ndarray], + data: dict[AnyFieldKey, Union[np.ndarray, tuple[np.ndarry, str]]], length_unit=None, bbox=None, sim_time=None, From fec520f88fa225691c471a83a57947dc7b5ba00b Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Thu, 18 Jul 2024 15:13:47 -0500 Subject: [PATCH 117/186] attempted change typing to work with python 3.9 --- yt/testing.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/yt/testing.py b/yt/testing.py index 40b185010e..af4e699827 100644 --- a/yt/testing.py +++ b/yt/testing.py @@ -9,11 +9,12 @@ from functools import wraps from importlib.util import find_spec from shutil import which -from typing import Callable +from typing import Callable, Union from unittest import SkipTest import matplotlib import numpy as np +import numpy.typing as npt from more_itertools import always_iterable from numpy.random import RandomState from unyt.exceptions import UnitOperationError @@ -84,7 +85,8 @@ def assert_rel_equal(a1, a2, decimals, err_msg="", verbose=True): ) # tested: volume integral is 1. -def cubicspline_python(x: float) -> float: +def cubicspline_python(x: Union[float,npt.ndarray[float]], + ) -> npt.ndarray[float]: ''' cubic spline SPH kernel function for testing against more effiecient cython methods @@ -140,11 +142,11 @@ def integrate_kernel(kernelfunc: Callable[[float], float], return pre * integral _zeroperiods = np.array([0., 0., 0.]) -def distancematrix(pos3_i0: np.ndarray[float], - pos3_i1: np.ndarray[float], - periodic: tuple[bool] = (True,) * 3, - periods: np.ndarray = _zeroperiods, - ) -> np.ndarray[float]: +def distancematrix(pos3_i0: npt.ndarray[float], + pos3_i1: npt.ndarray[float], + periodic: tuple[bool, bool, bool] = (True,) * 3, + periods: npt.ndarray[float] = _zeroperiods, + ) -> npt.ndarray[float]: ''' Calculates the distances between two arrays of points. @@ -797,14 +799,14 @@ def fake_sph_flexible_grid_ds( hsml_factor: float = 1.0, nperside: int = 3, periodic: bool = True, - e1hat: np.ndarray[float] = _xhat, - e2hat: np.ndarray[float] = _yhat, - e3hat: np.ndarray[float] = _zhat, - offsets: np.ndarray[float] = _floathalves, + e1hat: npt.ndarray[float] = _xhat, + e2hat: npt.ndarray[float] = _yhat, + e3hat: npt.ndarray[float] = _zhat, + offsets: npt.ndarray[float] = _floathalves, massgenerator: Callable[[int, int, int], float] = constantmass, unitrho: float = 1., - bbox: np.ndarray | None = None, - recenter: np.ndarray | None = None, + bbox: Union[npt.ndarray[float], None] = None, + recenter: Union[npt.ndarray[float], None] = None, ) -> StreamParticlesDataset: """Returns an in-memory SPH dataset useful for testing @@ -919,8 +921,9 @@ def fake_sph_flexible_grid_ds( return ds -def fake_random_sph_ds(npart: int, bbox: np.ndarray, - periodic: bool | tuple[bool, bool, bool] = True, +def fake_random_sph_ds(npart: int, + bbox: npt.ndarray[float], + periodic: Union[bool, tuple[bool, bool, bool]] = True, massrange: tuple[float, float] = (0.5, 2.), hsmlrange: tuple[float, float] = (0.5, 2.), unitrho: float = 1., From f4dd2cc57e26488a486e410b01665cea6a200f01 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 19:36:15 +0000 Subject: [PATCH 118/186] [pre-commit.ci] auto fixes from pre-commit.com hooks preserve auto fixes in merge for more information, see https://pre-commit.ci --- .../coordinates/cartesian_coordinates.py | 50 ++- .../tests/test_sph_pixelization.py | 173 ++++---- .../tests/test_sph_pixelization_pytestonly.py | 414 ++++++++++-------- yt/testing.py | 156 ++++--- yt/visualization/plot_window.py | 38 +- .../tests/test_offaxisprojection.py | 16 +- .../test_offaxisprojection_pytestonly.py | 122 +++--- .../volume_rendering/off_axis_projection.py | 15 +- .../volume_rendering/old_camera.py | 1 + 9 files changed, 566 insertions(+), 419 deletions(-) diff --git a/yt/geometry/coordinates/cartesian_coordinates.py b/yt/geometry/coordinates/cartesian_coordinates.py index bf471b2b58..7c68b8b0fe 100644 --- a/yt/geometry/coordinates/cartesian_coordinates.py +++ b/yt/geometry/coordinates/cartesian_coordinates.py @@ -380,7 +380,7 @@ def _ortho_pixelize( # instead xa = self.x_axis[dim] ya = self.y_axis[dim] - #axorder = data_source.ds.coordinates.axis_order + # axorder = data_source.ds.coordinates.axis_order za = list({0, 1, 2} - {xa, ya})[0] ds_periodic = data_source.ds.periodicity _periodic = np.array(ds_periodic) @@ -395,7 +395,7 @@ def _ortho_pixelize( if kernel_name is None: kernel_name = "cubic" - if isinstance(data_source, YTParticleProj): # projection + if isinstance(data_source, YTParticleProj): # projection weight = data_source.weight_field moment = data_source.moment le, re = data_source.data_source.get_bbox() @@ -444,7 +444,7 @@ def _ortho_pixelize( bnds3, _check_period=_periodic.astype("int"), period=period3, - kernel_name=kernel_name + kernel_name=kernel_name, ) # We use code length here, but to get the path length right # we need to multiply by the conversion factor between @@ -478,7 +478,7 @@ def _ortho_pixelize( _check_period=_periodic.astype("int"), period=period3, weight_field=chunk[weight].in_units(wounits), - kernel_name=kernel_name + kernel_name=kernel_name, ) mylog.info( "Making a fixed resolution buffer of (%s) %d by %d", @@ -501,7 +501,7 @@ def _ortho_pixelize( bnds3, _check_period=_periodic.astype("int"), period=period3, - kernel_name=kernel_name + kernel_name=kernel_name, ) normalization_2d_utility(buff, weight_buff) if moment == 2: @@ -523,7 +523,7 @@ def _ortho_pixelize( _check_period=_periodic.astype("int"), period=period3, weight_field=chunk[weight].in_units(wounits), - kernel_name=kernel_name + kernel_name=kernel_name, ) normalization_2d_utility(buff2, weight_buff) buff = compute_stddev_image(buff2, buff) @@ -554,7 +554,7 @@ def _ortho_pixelize( data_source.coord.to("code_length").v, _check_period=_periodic.astype("int"), period=period3, - kernel_name=kernel_name + kernel_name=kernel_name, ) if normalize: pixelize_sph_kernel_slice( @@ -571,7 +571,7 @@ def _ortho_pixelize( data_source.coord.to("code_length").v, _check_period=_periodic.astype("int"), period=period3, - kernel_name=kernel_name + kernel_name=kernel_name, ) if normalize: @@ -680,13 +680,16 @@ def _oblique_pixelize(self, data_source, field, bounds, size, antialias): _finfo = data_source.ds.field_info[field] is_sph_field = _finfo.is_sph_field particle_datasets = (ParticleDataset, StreamParticlesDataset) - #finfo = self.ds._get_field_info(field) + # finfo = self.ds._get_field_info(field) # SPH data # only for slices: a function in off_axis_projection.py # handles projections - if isinstance(data_source.ds, particle_datasets) and is_sph_field \ - and isinstance(data_source, YTCuttingPlane): + if ( + isinstance(data_source.ds, particle_datasets) + and is_sph_field + and isinstance(data_source, YTCuttingPlane) + ): normalize = getattr(self.ds, "use_sph_normalization", True) le = data_source.ds.domain_left_edge.to("code_length") re = data_source.ds.domain_right_edge.to("code_length") @@ -698,8 +701,7 @@ def _oblique_pixelize(self, data_source, field, bounds, size, antialias): axorder = data_source.ds.coordinates.axis_order ounits = data_source.ds.field_info[field].output_units # input bounds are in code length units already - widthxy = np.array((bounds[1] - bounds[0], - bounds[3] - bounds[2])) + widthxy = np.array((bounds[1] - bounds[0], bounds[3] - bounds[2])) kernel_name = None if hasattr(data_source.ds, "kernel_name"): kernel_name = data_source.ds.kernel_name @@ -728,9 +730,12 @@ def _oblique_pixelize(self, data_source, field, bounds, size, antialias): chunk[ptype, "mass"].to("code_mass"), chunk[ptype, "density"].to("code_density"), chunk[field].in_units(ounits), - center, widthxy, - normal_vector, north_vector, - boxbounds, periodic, + center, + widthxy, + normal_vector, + north_vector, + boxbounds, + periodic, kernel_name=kernel_name, check_period=1, ) @@ -745,11 +750,14 @@ def _oblique_pixelize(self, data_source, field, bounds, size, antialias): chunk[ptype, "mass"].to("code_mass"), chunk[ptype, "density"].to("code_density"), np.ones(chunk[ptype, "density"].shape[0]), - center, widthxy, - normal_vector, north_vector, - boxbounds, periodic, + center, + widthxy, + normal_vector, + north_vector, + boxbounds, + periodic, kernel_name=kernel_name, - check_period=1 + check_period=1, ) if normalize: @@ -763,7 +771,7 @@ def _oblique_pixelize(self, data_source, field, bounds, size, antialias): # whatever other data this code could handle before the # SPH option was added else: - indices = np.argsort(data_source["pdx"])[::-1].astype(np.int_) + indices = np.argsort(data_source["pdx"])[::-1].astype("int64", copy=False) buff = np.full((size[1], size[0]), np.nan, dtype="float64") ftype = "index" if isinstance(data_source.ds, YTSpatialPlotDataset): diff --git a/yt/geometry/coordinates/tests/test_sph_pixelization.py b/yt/geometry/coordinates/tests/test_sph_pixelization.py index 3ae3e9ee3d..1d1c82be5c 100644 --- a/yt/geometry/coordinates/tests/test_sph_pixelization.py +++ b/yt/geometry/coordinates/tests/test_sph_pixelization.py @@ -6,7 +6,8 @@ cubicspline_python, fake_sph_flexible_grid_ds, integrate_kernel, - requires_file) + requires_file, +) from yt.utilities.math_utils import compute_stddev_image ## off-axis projection tests for SPH data are in @@ -48,71 +49,74 @@ def _vysq(field, data): sigy = compute_stddev_image(prj1.frb["gas", "vysq"], prj1.frb["gas", "velocity_y"]) assert_rel_equal(sigy, prj2.frb["gas", "velocity_y"].d, 10) + def test_sph_projection_basic1(): - ''' + """ small, uniform grid: expected values for given dl? pixel centers at 0.5, 1., 1.5, 2., 2.5 particles at 0.5, 1.5, 2.5 - ''' - bbox = np.array([[0., 3.]] * 3) - ds = fake_sph_flexible_grid_ds(hsml_factor=1.0, nperside=3, - bbox=bbox) + """ + bbox = np.array([[0.0, 3.0]] * 3) + ds = fake_sph_flexible_grid_ds(hsml_factor=1.0, nperside=3, bbox=bbox) # works, but no depth control (at least without specific filters) proj = ds.proj(("gas", "density"), 2) - frb = proj.to_frb(width=(2.5, 'cm'), - resolution=(5, 5), - height=(2.5, 'cm'), - center=np.array([1.5, 1.5, 1.5]), - periodic=False) - out = frb.get_image(('gas', 'density')) + frb = proj.to_frb( + width=(2.5, "cm"), + resolution=(5, 5), + height=(2.5, "cm"), + center=np.array([1.5, 1.5, 1.5]), + periodic=False, + ) + out = frb.get_image(("gas", "density")) expected_out = np.zeros((5, 5), dtype=np.float64) - dl_1part = integrate_kernel(cubicspline_python, 0., 0.5) - linedens_1part = dl_1part * 1. # unit mass, density - linedens = 3. * linedens_1part + dl_1part = integrate_kernel(cubicspline_python, 0.0, 0.5) + linedens_1part = dl_1part * 1.0 # unit mass, density + linedens = 3.0 * linedens_1part expected_out[::2, ::2] = linedens assert_rel_equal(expected_out, out.v, 5) - #return out + # return out + def test_sph_projection_basic2(): - ''' + """ small, uniform grid: expected values for given dl? pixel centers at 0.5, 1., 1.5, 2., 2.5 particles at 0.5, 1.5, 2.5 but hsml radii are 0.25 -> try non-zero impact parameters, other pixels are still zero. - ''' - bbox = np.array([[0., 3.]] * 3) - ds = fake_sph_flexible_grid_ds(hsml_factor=0.5, nperside=3, - bbox=bbox) + """ + bbox = np.array([[0.0, 3.0]] * 3) + ds = fake_sph_flexible_grid_ds(hsml_factor=0.5, nperside=3, bbox=bbox) proj = ds.proj(("gas", "density"), 2) - frb = proj.to_frb(width=(2.5, 'cm'), - resolution=(5, 5), - height=(2.5, 'cm'), - center=np.array([1.375, 1.375, 1.5]), - periodic=False) - out = frb.get_image(('gas', 'density')) + frb = proj.to_frb( + width=(2.5, "cm"), + resolution=(5, 5), + height=(2.5, "cm"), + center=np.array([1.375, 1.375, 1.5]), + periodic=False, + ) + out = frb.get_image(("gas", "density")) expected_out = np.zeros((5, 5), dtype=np.float64) - dl_1part = integrate_kernel(cubicspline_python, - np.sqrt(2) * 0.125, - 0.25) - linedens_1part = dl_1part * 1. # unit mass, density - linedens = 3. * linedens_1part + dl_1part = integrate_kernel(cubicspline_python, np.sqrt(2) * 0.125, 0.25) + linedens_1part = dl_1part * 1.0 # unit mass, density + linedens = 3.0 * linedens_1part expected_out[::2, ::2] = linedens - #print(expected_out) - #print(out.v) + # print(expected_out) + # print(out.v) assert_rel_equal(expected_out, out.v, 4) - #return out + # return out + def get_dataset_sphrefine(reflevel: int = 1): - ''' + """ constant density particle grid, with increasing particle sampling - ''' - lenfact = (1./3.)**(reflevel - 1) + """ + lenfact = (1.0 / 3.0) ** (reflevel - 1) massfact = lenfact**3 nperside = 3**reflevel @@ -120,26 +124,29 @@ def get_dataset_sphrefine(reflevel: int = 1): e2hat = np.array([0, lenfact, 0]) e3hat = np.array([0, 0, lenfact]) hsml_factor = lenfact - bbox = np.array([[0., 3.]] * 3) - offsets = np.ones(3, dtype=np.float64) * 0.5 # in units of ehat + bbox = np.array([[0.0, 3.0]] * 3) + offsets = np.ones(3, dtype=np.float64) * 0.5 # in units of ehat def refmass(i: int, j: int, k: int) -> float: return massfact - unitrho = 1. / massfact # want density 1 for decreasing mass - - ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, - nperside=nperside, - periodic=True, - e1hat=e1hat, - e2hat=e2hat, - e3hat=e3hat, - offsets=offsets, - massgenerator=refmass, - unitrho=unitrho, - bbox=bbox, - ) + + unitrho = 1.0 / massfact # want density 1 for decreasing mass + + ds = fake_sph_flexible_grid_ds( + hsml_factor=hsml_factor, + nperside=nperside, + periodic=True, + e1hat=e1hat, + e2hat=e2hat, + e3hat=e3hat, + offsets=offsets, + massgenerator=refmass, + unitrho=unitrho, + bbox=bbox, + ) return ds + def getdata_test_gridproj2(): # initial pixel centers at 0.5, 1., 1.5, 2., 2.5 # particles at 0.5, 1.5, 2.5 @@ -150,53 +157,63 @@ def getdata_test_gridproj2(): for rl in range(1, 4): ds = get_dataset_sphrefine(reflevel=rl) proj = ds.proj(("gas", "density"), 2) - frb = proj.to_frb(width=(2.5, 'cm'), - resolution=(5, 5), - height=(2.5, 'cm'), - center=np.array([1.5, 1.5, 1.5]), - periodic=False) - out = frb.get_image(('gas', 'density')) + frb = proj.to_frb( + width=(2.5, "cm"), + resolution=(5, 5), + height=(2.5, "cm"), + center=np.array([1.5, 1.5, 1.5]), + periodic=False, + ) + out = frb.get_image(("gas", "density")) outlist.append(out) dss.append(ds) return outlist, dss + def test_sph_gridproj_reseffect1(): - ''' + """ Comparing same pixel centers with higher particle resolution. The pixel centers are at x/y coordinates [0.5, 1., 1.5, 2., 2.5] at the first level, the spacing halves at each level. Checking the pixels at [0.5, 1.5, 2.5], which should have the same values at each resolution. - ''' + """ imgs, _ = getdata_test_gridproj2() ref = imgs[-1] for img in imgs: - assert_rel_equal(img[::img.shape[0] // 2, ::img.shape[1] // 2], - ref[::ref.shape[0] // 2, ::ref.shape[1] // 2], 4) + assert_rel_equal( + img[:: img.shape[0] // 2, :: img.shape[1] // 2], + ref[:: ref.shape[0] // 2, :: ref.shape[1] // 2], + 4, + ) + def test_sph_gridproj_reseffect2(): - ''' + """ refine the pixel grid instead of the particle grid - ''' + """ ds = get_dataset_sphrefine(reflevel=2) proj = ds.proj(("gas", "density"), 2) imgs = {} maxrl = 5 for rl in range(1, maxrl + 1): - npix = 1 + 2**(rl + 1) - margin = 0.5 - 0.5**(rl + 1) - frb = proj.to_frb(width=(3. - 2. * margin, 'cm'), - resolution=(npix, npix), - height=(3. - 2. * margin, 'cm'), - center=np.array([1.5, 1.5, 1.5]), - periodic=False) - out = frb.get_image(('gas', 'density')) + npix = 1 + 2 ** (rl + 1) + margin = 0.5 - 0.5 ** (rl + 1) + frb = proj.to_frb( + width=(3.0 - 2.0 * margin, "cm"), + resolution=(npix, npix), + height=(3.0 - 2.0 * margin, "cm"), + center=np.array([1.5, 1.5, 1.5]), + periodic=False, + ) + out = frb.get_image(("gas", "density")) imgs[rl] = out ref = imgs[maxrl] - pixspace_ref = 2**(maxrl) + pixspace_ref = 2 ** (maxrl) for rl in imgs: img = imgs[rl] - pixspace = 2**(rl) - #print(f'Grid refinement level {rl}:') - assert_rel_equal(img[::pixspace, ::pixspace], - ref[::pixspace_ref, ::pixspace_ref], 4) + pixspace = 2 ** (rl) + # print(f'Grid refinement level {rl}:') + assert_rel_equal( + img[::pixspace, ::pixspace], ref[::pixspace_ref, ::pixspace_ref], 4 + ) diff --git a/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py b/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py index af95fde034..945286af0c 100644 --- a/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py +++ b/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py @@ -13,17 +13,16 @@ integrate_kernel, ) + @pytest.mark.parametrize("weighted", [True, False]) @pytest.mark.parametrize("periodic", [True, False]) -@pytest.mark.parametrize("depth", [None, (1., "cm")]) +@pytest.mark.parametrize("depth", [None, (1.0, "cm")]) @pytest.mark.parametrize("shiftcenter", [False, True]) @pytest.mark.parametrize("axis", [0, 1, 2]) -def test_sph_proj_general_alongaxes(axis: int, - shiftcenter: bool, - depth : float | None, - periodic: bool, - weighted: bool) -> None: - ''' +def test_sph_proj_general_alongaxes( + axis: int, shiftcenter: bool, depth: float | None, periodic: bool, weighted: bool +) -> None: + """ The previous projection tests were for a specific issue. Here, we test more functionality of the projections. We just send lines of sight through pixel centers for convenience. @@ -49,34 +48,39 @@ def test_sph_proj_general_alongaxes(axis: int, Returns: -------- None - ''' + """ if shiftcenter: - center = unyt.unyt_array(np.array((0.625, 0.625, 0.625)), 'cm') + center = unyt.unyt_array(np.array((0.625, 0.625, 0.625)), "cm") else: - center = unyt.unyt_array(np.array((1.5, 1.5, 1.5)), 'cm') - bbox = unyt.unyt_array(np.array([[0., 3.], [0., 3.], [0., 3.]]), 'cm') + center = unyt.unyt_array(np.array((1.5, 1.5, 1.5)), "cm") + bbox = unyt.unyt_array(np.array([[0.0, 3.0], [0.0, 3.0], [0.0, 3.0]]), "cm") hsml_factor = 0.5 unitrho = 1.5 + # test correct centering, particle selection def makemasses(i, j, k): if i == j == k == 1: - return 2. + return 2.0 else: - return 1. + return 1.0 + # m / rho, factor 1. / hsml**2 is included in the kernel integral # (density is adjusted, so same for center particle) - prefactor = 1. / unitrho #/ (0.5 * 0.5)**2 - dl_cen = integrate_kernel(cubicspline_python, 0., 0.25) + prefactor = 1.0 / unitrho # / (0.5 * 0.5)**2 + dl_cen = integrate_kernel(cubicspline_python, 0.0, 0.25) # result shouldn't depend explicitly on the center if we re-center # the data, unless we get cut-offs in the non-periodic case - ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, - periodic=periodic, - offsets=np.full(3, 0.5), - massgenerator=makemasses, - unitrho=unitrho, - bbox=bbox.v, - recenter=center.v) + ds = fake_sph_flexible_grid_ds( + hsml_factor=hsml_factor, + nperside=3, + periodic=periodic, + offsets=np.full(3, 0.5), + massgenerator=makemasses, + unitrho=unitrho, + bbox=bbox.v, + recenter=center.v, + ) if depth is None: source = ds.all_data() else: @@ -86,10 +90,7 @@ def makemasses(i, j, k): le[axis] = center[axis] - 0.5 * depth re[axis] = center[axis] + 0.5 * depth cen = 0.5 * (le + re) - reg = YTRegion(center=cen, - left_edge=le, - right_edge=re, - ds=ds) + reg = YTRegion(center=cen, left_edge=le, right_edge=re, ds=ds) source = reg # we don't actually want a plot, it's just a straightforward, @@ -98,55 +99,71 @@ def makemasses(i, j, k): toweight_field = ("gas", "density") else: toweight_field = None - prj = yt.ProjectionPlot(ds, axis, ("gas", "density"), - width=(2.5, "cm"), - weight_field=toweight_field, - buff_size=(5, 5), - center=center, - data_source=source) - img = prj.frb.data[('gas', 'density')] + prj = yt.ProjectionPlot( + ds, + axis, + ("gas", "density"), + width=(2.5, "cm"), + weight_field=toweight_field, + buff_size=(5, 5), + center=center, + data_source=source, + ) + img = prj.frb.data[("gas", "density")] if weighted: - expected_out = np.zeros((5, 5,), dtype=img.v.dtype) + expected_out = np.zeros( + ( + 5, + 5, + ), + dtype=img.v.dtype, + ) expected_out[::2, ::2] = unitrho if depth is None: ## during shift, particle coords do wrap around edges - #if (not periodic) and shiftcenter: + # if (not periodic) and shiftcenter: # # weight 1. for unitrho, 2. for 2. * untrho # expected_out[2, 2] *= 5. / 3. - #else: + # else: # weight (2 * 1.) for unitrho, (1 * 2.) for 2. * unitrho expected_out[2, 2] *= 1.5 else: # only 2 * unitrho element included - expected_out[2, 2] *= 2. + expected_out[2, 2] *= 2.0 else: - expected_out = np.zeros((5, 5,), dtype=img.v.dtype) + expected_out = np.zeros( + ( + 5, + 5, + ), + dtype=img.v.dtype, + ) expected_out[::2, ::2] = dl_cen * prefactor * unitrho if depth is None: # 3 particles per l.o.s., including the denser one - expected_out *= 3. - expected_out[2, 2] *= 4. / 3. + expected_out *= 3.0 + expected_out[2, 2] *= 4.0 / 3.0 else: # 1 particle per l.o.s., including the denser one - expected_out[2, 2] *= 2. + expected_out[2, 2] *= 2.0 # grid is shifted to the left -> 'missing' stuff at the left if (not periodic) and shiftcenter: - expected_out[:1, :] = 0. - expected_out[:, :1] = 0. - #print(axis, shiftcenter, depth, periodic, weighted) - #print(expected_out) - #print(img.v) + expected_out[:1, :] = 0.0 + expected_out[:, :1] = 0.0 + # print(axis, shiftcenter, depth, periodic, weighted) + # print(expected_out) + # print(img.v) assert_rel_equal(expected_out, img.v, 5) + @pytest.mark.parametrize("periodic", [True, False]) @pytest.mark.parametrize("shiftcenter", [False, True]) -@pytest.mark.parametrize("zoff", [0., 0.1, 0.5, 1.]) +@pytest.mark.parametrize("zoff", [0.0, 0.1, 0.5, 1.0]) @pytest.mark.parametrize("axis", [0, 1, 2]) -def test_sph_slice_general_alongaxes(axis: int, - shiftcenter: bool, - periodic: bool, - zoff: float) -> None: - ''' +def test_sph_slice_general_alongaxes( + axis: int, shiftcenter: bool, periodic: bool, zoff: float +) -> None: + """ Particles at [0.5, 1.5, 2.5] (in each coordinate) smoothing lengths 0.25 all particles have mass 1., density 1.5, @@ -169,34 +186,38 @@ def test_sph_slice_general_alongaxes(axis: int, Returns: -------- None - ''' + """ if shiftcenter: - center = unyt.unyt_array(np.array((0.625, 0.625, 0.625)), 'cm') + center = unyt.unyt_array(np.array((0.625, 0.625, 0.625)), "cm") else: - center = unyt.unyt_array(np.array((1.5, 1.5, 1.5)), 'cm') - bbox = unyt.unyt_array(np.array([[0., 3.], [0., 3.], [0., 3.]]), 'cm') + center = unyt.unyt_array(np.array((1.5, 1.5, 1.5)), "cm") + bbox = unyt.unyt_array(np.array([[0.0, 3.0], [0.0, 3.0], [0.0, 3.0]]), "cm") hsml_factor = 0.5 unitrho = 1.5 + # test correct centering, particle selection def makemasses(i, j, k): if i == j == k == 1: - return 2. + return 2.0 elif i == j == k == 2: - return 3. + return 3.0 else: - return 1. + return 1.0 # result shouldn't depend explicitly on the center if we re-center # the data, unless we get cut-offs in the non-periodic case - ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, - periodic=periodic, - offsets=np.full(3, 0.5), - massgenerator=makemasses, - unitrho=unitrho, - bbox=bbox.v, - recenter=center.v) + ds = fake_sph_flexible_grid_ds( + hsml_factor=hsml_factor, + nperside=3, + periodic=periodic, + offsets=np.full(3, 0.5), + massgenerator=makemasses, + unitrho=unitrho, + bbox=bbox.v, + recenter=center.v, + ) ad = ds.all_data() - #print(ad[('gas', 'position')]) + # print(ad[('gas', 'position')]) outgridsize = 10 width = 2.5 _center = center.to("cm").v.copy() @@ -204,20 +225,27 @@ def makemasses(i, j, k): # we don't actually want a plot, it's just a straightforward, # common way to get an frb / image array - slc = yt.SlicePlot(ds, axis, ("gas", "density"), - width=(width, "cm"), - buff_size=(outgridsize,) * 2, - center=(_center, "cm")) - img = slc.frb.data[('gas', 'density')] + slc = yt.SlicePlot( + ds, + axis, + ("gas", "density"), + width=(width, "cm"), + buff_size=(outgridsize,) * 2, + center=(_center, "cm"), + ) + img = slc.frb.data[("gas", "density")] # center is same in non-projection coords if axis == 0: ci = 1 else: ci = 0 - gridcens = _center[ci] - 0.5 * width \ - + 0.5 * width / outgridsize \ - + np.arange(outgridsize) * width / outgridsize + gridcens = ( + _center[ci] + - 0.5 * width + + 0.5 * width / outgridsize + + np.arange(outgridsize) * width / outgridsize + ) xgrid = np.repeat(gridcens, outgridsize) ygrid = np.tile(gridcens, outgridsize) zgrid = np.full(outgridsize**2, _center[axis]) @@ -235,47 +263,56 @@ def makemasses(i, j, k): gridcoords[:, 1] = zgrid gridcoords[:, 2] = xgrid ad = ds.all_data() - sphcoords = np.array([(ad[("gas", "x")]).to("cm"), - (ad[("gas", "y")]).to("cm"), - (ad[("gas", "z")]).to("cm"), - ]).T - print('sphcoords:') + sphcoords = np.array( + [ + (ad[("gas", "x")]).to("cm"), + (ad[("gas", "y")]).to("cm"), + (ad[("gas", "z")]).to("cm"), + ] + ).T + print("sphcoords:") print(sphcoords) - print('gridcoords:') + print("gridcoords:") print(gridcoords) - dists = distancematrix(gridcoords, sphcoords, - periodic=(periodic,)*3, - periods=np.array([3., 3., 3.])) - print('dists <= 1:') + dists = distancematrix( + gridcoords, + sphcoords, + periodic=(periodic,) * 3, + periods=np.array([3.0, 3.0, 3.0]), + ) + print("dists <= 1:") print(dists <= 1) sml = (ad[("gas", "smoothing_length")]).to("cm") normkern = cubicspline_python(dists / sml.v[np.newaxis, :]) - sphcontr = normkern / sml[np.newaxis, :]**3 * ad[("gas", "mass")] + sphcontr = normkern / sml[np.newaxis, :] ** 3 * ad[("gas", "mass")] contsum = np.sum(sphcontr, axis=1) - sphweights = normkern / sml[np.newaxis, :]**3 \ - * ad[("gas", "mass")] / ad[("gas", "density")] + sphweights = ( + normkern + / sml[np.newaxis, :] ** 3 + * ad[("gas", "mass")] + / ad[("gas", "density")] + ) weights = np.sum(sphweights, axis=1) expected = contsum / weights expected = expected.reshape((outgridsize, outgridsize)) - expected[np.isnan(expected)] = 0. # convention in the slices + expected[np.isnan(expected)] = 0.0 # convention in the slices - print('expected:\n', expected.v) - print('recovered:\n', img.v) + print("expected:\n", expected.v) + print("recovered:\n", img.v) assert_rel_equal(expected.v, img.v, 5) - @pytest.mark.parametrize("periodic", [True, False]) @pytest.mark.parametrize("shiftcenter", [False, True]) -@pytest.mark.parametrize("northvector", [None, (1.e-4, 1., 0.)]) -@pytest.mark.parametrize("zoff", [0., 0.1, 0.5, 1.]) +@pytest.mark.parametrize("northvector", [None, (1.0e-4, 1.0, 0.0)]) +@pytest.mark.parametrize("zoff", [0.0, 0.1, 0.5, 1.0]) def test_sph_slice_general_offaxis( - northvector: tuple[float, float, float] | None, - shiftcenter: bool, - zoff: float, - periodic: bool, - ) -> None: - ''' + northvector: tuple[float, float, float] | None, + shiftcenter: bool, + zoff: float, + periodic: bool, +) -> None: + """ Same as the on-axis slices, but we rotate the basis vectors to test whether roations are handled ok. the rotation is chosen to be small so that in/exclusion of particles within bboxes, etc. @@ -300,29 +337,31 @@ def test_sph_slice_general_offaxis( Returns: -------- None - ''' + """ if shiftcenter: - center = np.array((0.625, 0.625, 0.625)) # cm + center = np.array((0.625, 0.625, 0.625)) # cm else: - center = np.array((1.5, 1.5, 1.5)) # cm - bbox = unyt.unyt_array(np.array([[0., 3.], [0., 3.], [0., 3.]]), 'cm') + center = np.array((1.5, 1.5, 1.5)) # cm + bbox = unyt.unyt_array(np.array([[0.0, 3.0], [0.0, 3.0], [0.0, 3.0]]), "cm") hsml_factor = 0.5 unitrho = 1.5 + # test correct centering, particle selection def makemasses(i, j, k): if i == j == k == 1: - return 2. + return 2.0 else: - return 1. + return 1.0 + # try to make sure dl differences from periodic wrapping are small epsilon = 1e-4 - projaxis = np.array([epsilon, 0.00, np.sqrt(1. - epsilon**2)]) + projaxis = np.array([epsilon, 0.00, np.sqrt(1.0 - epsilon**2)]) e1dir = projaxis / np.sqrt(np.sum(projaxis**2)) if northvector is None: - e2dir = np.array([0., 1., 0.]) + e2dir = np.array([0.0, 1.0, 0.0]) else: e2dir = np.asarray(northvector) - e2dir = e2dir - np.sum(e1dir * e2dir) * e2dir # orthonormalize + e2dir = e2dir - np.sum(e1dir * e2dir) * e2dir # orthonormalize e2dir /= np.sqrt(np.sum(e2dir**2)) e3dir = np.cross(e2dir, e1dir) @@ -331,72 +370,99 @@ def makemasses(i, j, k): _center = center.copy() _center += zoff * e1dir - ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, - periodic=periodic, - offsets=np.full(3, 0.5), - massgenerator=makemasses, - unitrho=unitrho, - bbox=bbox.v, - recenter=center, - e1hat=e1dir, e2hat=e2dir, e3hat=e3dir) + ds = fake_sph_flexible_grid_ds( + hsml_factor=hsml_factor, + nperside=3, + periodic=periodic, + offsets=np.full(3, 0.5), + massgenerator=makemasses, + unitrho=unitrho, + bbox=bbox.v, + recenter=center, + e1hat=e1dir, + e2hat=e2dir, + e3hat=e3dir, + ) source = ds.all_data() # couple to dataset -> right unit registry - center = ds.arr(center, 'cm') - print('position:\n', source['gas','position']) - slc = yt.SlicePlot(ds, e1dir, ("gas", "density"), - width=(width, "cm"), - buff_size=(outgridsize,) * 2, - center=(_center, "cm"), - north_vector=e2dir) - img = slc.frb.data[('gas', 'density')] + center = ds.arr(center, "cm") + print("position:\n", source["gas", "position"]) + slc = yt.SlicePlot( + ds, + e1dir, + ("gas", "density"), + width=(width, "cm"), + buff_size=(outgridsize,) * 2, + center=(_center, "cm"), + north_vector=e2dir, + ) + img = slc.frb.data[("gas", "density")] # center is same in x/y (e3dir/e2dir) - gridcenx = np.dot(_center, e3dir) - 0.5 * width \ - + 0.5 * width / outgridsize \ - + np.arange(outgridsize) * width / outgridsize - gridceny = np.dot(_center, e2dir) - 0.5 * width \ - + 0.5 * width / outgridsize \ - + np.arange(outgridsize) * width / outgridsize + gridcenx = ( + np.dot(_center, e3dir) + - 0.5 * width + + 0.5 * width / outgridsize + + np.arange(outgridsize) * width / outgridsize + ) + gridceny = ( + np.dot(_center, e2dir) + - 0.5 * width + + 0.5 * width / outgridsize + + np.arange(outgridsize) * width / outgridsize + ) xgrid = np.repeat(gridcenx, outgridsize) ygrid = np.tile(gridceny, outgridsize) zgrid = np.full(outgridsize**2, np.dot(_center, e1dir)) - gridcoords = (xgrid[:, np.newaxis] * e3dir[np.newaxis, :] - + ygrid[:, np.newaxis] * e2dir[np.newaxis, :] - + zgrid[:, np.newaxis] * e1dir[np.newaxis, :]) - print('gridcoords:') + gridcoords = ( + xgrid[:, np.newaxis] * e3dir[np.newaxis, :] + + ygrid[:, np.newaxis] * e2dir[np.newaxis, :] + + zgrid[:, np.newaxis] * e1dir[np.newaxis, :] + ) + print("gridcoords:") print(gridcoords) ad = ds.all_data() - sphcoords = np.array([(ad[("gas", "x")]).to("cm"), - (ad[("gas", "y")]).to("cm"), - (ad[("gas", "z")]).to("cm"), - ]).T - dists = distancematrix(gridcoords, sphcoords, - periodic=(periodic,)*3, - periods=np.array([3., 3., 3.])) + sphcoords = np.array( + [ + (ad[("gas", "x")]).to("cm"), + (ad[("gas", "y")]).to("cm"), + (ad[("gas", "z")]).to("cm"), + ] + ).T + dists = distancematrix( + gridcoords, + sphcoords, + periodic=(periodic,) * 3, + periods=np.array([3.0, 3.0, 3.0]), + ) sml = (ad[("gas", "smoothing_length")]).to("cm") normkern = cubicspline_python(dists / sml.v[np.newaxis, :]) - sphcontr = normkern / sml[np.newaxis, :]**3 * ad[("gas", "mass")] + sphcontr = normkern / sml[np.newaxis, :] ** 3 * ad[("gas", "mass")] contsum = np.sum(sphcontr, axis=1) - sphweights = normkern / sml[np.newaxis, :]**3 \ - * ad[("gas", "mass")] / ad[("gas", "density")] + sphweights = ( + normkern + / sml[np.newaxis, :] ** 3 + * ad[("gas", "mass")] + / ad[("gas", "density")] + ) weights = np.sum(sphweights, axis=1) expected = contsum / weights expected = expected.reshape((outgridsize, outgridsize)) - expected = expected.T # transposed for image plotting - expected[np.isnan(expected)] = 0. # convention in the slices + expected = expected.T # transposed for image plotting + expected[np.isnan(expected)] = 0.0 # convention in the slices - #print(axis, shiftcenter, depth, periodic, weighted) - print('expected:\n', expected.v) - print('recovered:\n', img.v) + # print(axis, shiftcenter, depth, periodic, weighted) + print("expected:\n", expected.v) + print("recovered:\n", img.v) assert_rel_equal(expected.v, img.v, 4) + # only axis-aligned; testing YTArbitraryGrid, YTCoveringGrid @pytest.mark.parametrize("periodic", [True, False, (True, True, False)]) @pytest.mark.parametrize("wholebox", [True, False]) -def test_sph_grid(periodic: bool | tuple[bool, bool, bool], - wholebox: bool): - bbox = np.array([[-1., 3.], [1., 5.2], [-1., 3.]]) +def test_sph_grid(periodic: bool | tuple[bool, bool, bool], wholebox: bool): + bbox = np.array([[-1.0, 3.0], [1.0, 5.2], [-1.0, 3.0]]) ds = fake_random_sph_ds(50, bbox, periodic=periodic) if not hasattr(periodic, "__len__"): @@ -406,15 +472,15 @@ def test_sph_grid(periodic: bool | tuple[bool, bool, bool], left = bbox[:, 0].copy() level = 2 ncells = np.array([2**level] * 3) - print('left: ', left) - print('ncells: ', ncells) + print("left: ", left) + print("ncells: ", ncells) resgrid = ds.covering_grid(level, tuple(left), ncells) right = bbox[:, 1].copy() xedges = np.linspace(left[0], right[0], ncells[0] + 1) yedges = np.linspace(left[1], right[1], ncells[1] + 1) zedges = np.linspace(left[2], right[2], ncells[2] + 1) else: - left = np.array([-1., 1.8, -1.]) + left = np.array([-1.0, 1.8, -1.0]) right = np.array([2.5, 5.2, 2.5]) ncells = np.array([3, 4, 4]) resgrid = ds.arbitrary_grid(left, right, dims=ncells) @@ -427,33 +493,37 @@ def test_sph_grid(periodic: bool | tuple[bool, bool, bool], zcens = 0.5 * (zedges[:-1] + zedges[1:]) ad = ds.all_data() - sphcoords = np.array([(ad[("gas", "x")]).to("cm"), - (ad[("gas", "y")]).to("cm"), - (ad[("gas", "z")]).to("cm"), - ]).T - gridx, gridy, gridz = np.meshgrid(xcens, ycens, zcens, - indexing='ij') + sphcoords = np.array( + [ + (ad[("gas", "x")]).to("cm"), + (ad[("gas", "y")]).to("cm"), + (ad[("gas", "z")]).to("cm"), + ] + ).T + gridx, gridy, gridz = np.meshgrid(xcens, ycens, zcens, indexing="ij") outshape = gridx.shape gridx = gridx.flatten() gridy = gridy.flatten() gridz = gridz.flatten() gridcoords = np.array([gridx, gridy, gridz]).T periods = bbox[:, 1] - bbox[:, 0] - dists = distancematrix(gridcoords, sphcoords, - periodic=periodic, - periods=periods) + dists = distancematrix(gridcoords, sphcoords, periodic=periodic, periods=periods) sml = (ad[("gas", "smoothing_length")]).to("cm") normkern = cubicspline_python(dists / sml.v[np.newaxis, :]) - sphcontr = normkern / sml[np.newaxis, :]**3 * ad[("gas", "mass")] + sphcontr = normkern / sml[np.newaxis, :] ** 3 * ad[("gas", "mass")] contsum = np.sum(sphcontr, axis=1) - sphweights = normkern / sml[np.newaxis, :]**3 \ - * ad[("gas", "mass")] / ad[("gas", "density")] + sphweights = ( + normkern + / sml[np.newaxis, :] ** 3 + * ad[("gas", "mass")] + / ad[("gas", "density")] + ) weights = np.sum(sphweights, axis=1) expected = contsum / weights expected = expected.reshape(outshape) - expected[np.isnan(expected)] = 0. # convention in the slices + expected[np.isnan(expected)] = 0.0 # convention in the slices - #print(axis, shiftcenter, depth, periodic, weighted) - print('expected:\n', expected.v) - print('recovered:\n', res.v) + # print(axis, shiftcenter, depth, periodic, weighted) + print("expected:\n", expected.v) + print("recovered:\n", res.v) assert_rel_equal(expected.v, res.v, 4) diff --git a/yt/testing.py b/yt/testing.py index af4e699827..43f4ba3e60 100644 --- a/yt/testing.py +++ b/yt/testing.py @@ -26,8 +26,6 @@ from yt.loaders import load, load_particles from yt.units.yt_array import YTArray, YTQuantity - - ANSWER_TEST_TAG = "answer_test" @@ -84,10 +82,12 @@ def assert_rel_equal(a1, a2, decimals, err_msg="", verbose=True): np.array(a1) / np.array(a2), 1.0, decimals, err_msg=err_msg, verbose=verbose ) + # tested: volume integral is 1. + def cubicspline_python(x: Union[float,npt.ndarray[float]], ) -> npt.ndarray[float]: - ''' + """ cubic spline SPH kernel function for testing against more effiecient cython methods @@ -99,19 +99,21 @@ def cubicspline_python(x: Union[float,npt.ndarray[float]], Returns ------- value of the kernel function - ''' + """ # C is 8/pi - _c = 8. / np.pi + _c = 8.0 / np.pi x = np.asarray(x) kernel = np.zeros(x.shape, dtype=x.dtype) - half1 = np.where(np.logical_and(x >=0., x <= 0.5)) - kernel[half1] = 1. - 6. * x[half1]**2 * (1. - x[half1]) + half1 = np.where(np.logical_and(x >= 0.0, x <= 0.5)) + kernel[half1] = 1.0 - 6.0 * x[half1] ** 2 * (1.0 - x[half1]) half2 = np.where(np.logical_and(x > 0.5, x <= 1.0)) - kernel[half2] = 2. * (1. - x[half2])**3 + kernel[half2] = 2.0 * (1.0 - x[half2]) ** 3 return kernel * _c -def integrate_kernel(kernelfunc: Callable[[float], float], - b: float, hsml: float) -> float: + +def integrate_kernel( + kernelfunc: Callable[[float], float], b: float, hsml: float +) -> float: """ integrates a kernel function over a line passing entirely through it @@ -130,24 +132,27 @@ def integrate_kernel(kernelfunc: Callable[[float], float], the integral of the SPH kernel function. units: 1 / units of b and hsml """ - pre = 1. / hsml**2 + pre = 1.0 / hsml**2 x = b / hsml - xmax = np.sqrt(1. - x**2) - xmin = -1. * xmax - xe = np.linspace(xmin, xmax, 500) # shape: 500, x.shape + xmax = np.sqrt(1.0 - x**2) + xmin = -1.0 * xmax + xe = np.linspace(xmin, xmax, 500) # shape: 500, x.shape xc = 0.5 * (xe[:-1, ...] + xe[1:, ...]) dx = np.diff(xe, axis=0) spv = kernelfunc(np.sqrt(xc**2 + x**2)) integral = np.sum(spv * dx, axis=0) return pre * integral -_zeroperiods = np.array([0., 0., 0.]) -def distancematrix(pos3_i0: npt.ndarray[float], - pos3_i1: npt.ndarray[float], - periodic: tuple[bool, bool, bool] = (True,) * 3, - periods: npt.ndarray[float] = _zeroperiods, - ) -> npt.ndarray[float]: - ''' +_zeroperiods = np.array([0.0, 0.0, 0.0]) + + +def distancematrix( + pos3_i0: np.ndarray[float], + pos3_i1: np.ndarray[float], + periodic: tuple[bool] = (True,) * 3, + periods: np.ndarray = _zeroperiods, +) -> np.ndarray[float]: + """ Calculates the distances between two arrays of points. Parameters: @@ -168,16 +173,16 @@ def distancematrix(pos3_i0: npt.ndarray[float], a 2D-array of distances between postions `pos3_i0` (changes along index 0) and `pos3_i1` (changes along index 1) - ''' + """ d2 = np.zeros((len(pos3_i0), len(pos3_i1)), dtype=pos3_i0.dtype) for ax in range(3): # 'center on' pos3_i1 _d = pos3_i0[:, ax, np.newaxis] - pos3_i1[np.newaxis, :, ax] if periodic[ax]: _period = periods[ax] - _d += 0.5 * _period # center on half box size - _d %= _period # wrap coordinate to 0 -- boxsize range - _d -= 0.5 * _period # center back to zero + _d += 0.5 * _period # center on half box size + _d %= _period # wrap coordinate to 0 -- boxsize range + _d -= 0.5 * _period # center back to zero d2 += _d**2 return np.sqrt(d2) @@ -789,25 +794,28 @@ def fake_sph_grid_ds(hsml_factor=1.0): def constantmass(i: int, j: int, k: int) -> float: - return 1. + return 1.0 + _xhat = np.array([1, 0, 0]) _yhat = np.array([0, 1, 0]) _zhat = np.array([0, 0, 1]) _floathalves = 0.5 * np.ones((3,), dtype=np.float64) + + def fake_sph_flexible_grid_ds( - hsml_factor: float = 1.0, - nperside: int = 3, - periodic: bool = True, - e1hat: npt.ndarray[float] = _xhat, - e2hat: npt.ndarray[float] = _yhat, - e3hat: npt.ndarray[float] = _zhat, - offsets: npt.ndarray[float] = _floathalves, - massgenerator: Callable[[int, int, int], float] = constantmass, - unitrho: float = 1., - bbox: Union[npt.ndarray[float], None] = None, - recenter: Union[npt.ndarray[float], None] = None, - ) -> StreamParticlesDataset: + hsml_factor: float = 1.0, + nperside: int = 3, + periodic: bool = True, + e1hat: np.ndarray[float] = _xhat, + e2hat: np.ndarray[float] = _yhat, + e3hat: np.ndarray[float] = _zhat, + offsets: np.ndarray[float] = _floathalves, + massgenerator: Callable[[int, int, int], float] = constantmass, + unitrho: float = 1.0, + bbox: np.ndarray | None = None, + recenter: np.ndarray | None = None, +) -> StreamParticlesDataset: """Returns an in-memory SPH dataset useful for testing Parameters: @@ -864,9 +872,11 @@ def fake_sph_flexible_grid_ds( for i in range(0, nperside): for j in range(0, nperside): for k in range(0, nperside): - _pos = (offsets[0] + i) * e1hat \ - + (offsets[1] + j) * e2hat \ - + (offsets[2] + k) * e3hat + _pos = ( + (offsets[0] + i) * e1hat + + (offsets[1] + j) * e2hat + + (offsets[2] + k) * e3hat + ) ind = nperside**2 * i + nperside * j + k pos[ind, :] = _pos mass[ind] = massgenerator(i, j, k) @@ -874,14 +884,14 @@ def fake_sph_flexible_grid_ds( if bbox is None: eps = 1e-3 - margin = (1. + eps) * hsml_factor - bbox = np.array([[np.min(pos[:, 0]) - margin, - np.max(pos[:, 0]) + margin], - [np.min(pos[:, 1]) - margin, - np.max(pos[:, 1]) + margin], - [np.min(pos[:, 2]) - margin, - np.max(pos[:, 2]) + margin], - ]) + margin = (1.0 + eps) * hsml_factor + bbox = np.array( + [ + [np.min(pos[:, 0]) - margin, np.max(pos[:, 0]) + margin], + [np.min(pos[:, 1]) - margin, np.max(pos[:, 1]) + margin], + [np.min(pos[:, 2]) - margin, np.max(pos[:, 2]) + margin], + ] + ) if recenter is not None: periods = bbox[:, 1] - bbox[:, 0] @@ -913,21 +923,26 @@ def fake_sph_flexible_grid_ds( "density": (rho[okinds], "g/cm**3"), } - ds = load_particles(data=data, - bbox=bbox, periodicity=(periodic,) * 3, - length_unit=1., mass_unit=1., time_unit=1., - velocity_unit=1.) - ds.kernel_name = 'cubic' + ds = load_particles( + data=data, + bbox=bbox, + periodicity=(periodic,) * 3, + length_unit=1.0, + mass_unit=1.0, + time_unit=1.0, + velocity_unit=1.0, + ) + ds.kernel_name = "cubic" return ds - -def fake_random_sph_ds(npart: int, - bbox: npt.ndarray[float], - periodic: Union[bool, tuple[bool, bool, bool]] = True, - massrange: tuple[float, float] = (0.5, 2.), - hsmlrange: tuple[float, float] = (0.5, 2.), - unitrho: float = 1., - ) -> StreamParticlesDataset: +def fake_random_sph_ds( + npart: int, + bbox: np.ndarray, + periodic: bool | tuple[bool, bool, bool] = True, + massrange: tuple[float, float] = (0.5, 2.0), + hsmlrange: tuple[float, float] = (0.5, 2.0), + unitrho: float = 1.0, +) -> StreamParticlesDataset: """Returns an in-memory SPH dataset useful for testing Parameters: @@ -956,7 +971,7 @@ def fake_random_sph_ds(npart: int, """ if not hasattr(periodic, "__len__"): - periodic = (periodic, ) * 3 + periodic = (periodic,) * 3 gen = np.random.default_rng(seed=0) posx = gen.uniform(low=bbox[0][0], high=bbox[0][1], size=npart) @@ -978,11 +993,16 @@ def fake_random_sph_ds(npart: int, "density": (dens, "g/cm**3"), } - ds = load_particles(data=data, - bbox=bbox, periodicity=periodic, - length_unit=1., mass_unit=1., time_unit=1., - velocity_unit=1.) - ds.kernel_name = 'cubic' + ds = load_particles( + data=data, + bbox=bbox, + periodicity=periodic, + length_unit=1.0, + mass_unit=1.0, + time_unit=1.0, + velocity_unit=1.0, + ) + ds.kernel_name = "cubic" return ds diff --git a/yt/visualization/plot_window.py b/yt/visualization/plot_window.py index 5afd02bd09..8176065f47 100644 --- a/yt/visualization/plot_window.py +++ b/yt/visualization/plot_window.py @@ -83,8 +83,9 @@ def get_window_parameters(axis, center, width, ds): return (bounds, center, display_center) -def get_oblique_window_parameters(normal, center, width, ds, - depth=None, get3bounds=False): +def get_oblique_window_parameters( + normal, center, width, ds, depth=None, get3bounds=False +): center, display_center = ds.coordinates.sanitize_center(center, axis=None) width = ds.coordinates.sanitize_width(normal, width, depth) @@ -92,8 +93,9 @@ def get_oblique_window_parameters(normal, center, width, ds, # Transforming to the cutting plane coordinate system # the original dimensionless center messes up off-axis # SPH projections though -> don't use this center there - center = ((center - ds.domain_left_edge) / ds.domain_width - 0.5)\ - * ds.domain_width + center = ( + (center - ds.domain_left_edge) / ds.domain_width - 0.5 + ) * ds.domain_width (normal, perp1, perp2) = ortho_find(normal) mat = np.transpose(np.column_stack((perp1, perp2, normal))) center = np.dot(mat, center) @@ -103,9 +105,9 @@ def get_oblique_window_parameters(normal, center, width, ds, if get3bounds and depth is None: # off-axis projection, depth not specified # -> set 'large enough' depth using half the box diagonal + margin - d2 = ds.domain_width[0].in_units("code_length")**2 - d2 += ds.domain_width[1].in_units("code_length")**2 - d2 += ds.domain_width[2].in_units("code_length")**2 + d2 = ds.domain_width[0].in_units("code_length") ** 2 + d2 += ds.domain_width[1].in_units("code_length") ** 2 + d2 += ds.domain_width[2].in_units("code_length") ** 2 diag = np.sqrt(d2) bounds = bounds + (-0.51 * diag, 0.51 * diag) return (bounds, center) @@ -1832,12 +1834,12 @@ def __init__( normal = self.sanitize_normal_vector(ds, normal) # this will handle time series data and controllers axis = fix_axis(normal, ds) - #print('center at SlicePlot init: ', center) - #print('current domain left edge: ', ds.domain_left_edge) + # print('center at SlicePlot init: ', center) + # print('current domain left edge: ', ds.domain_left_edge) (bounds, center, display_center) = get_window_parameters( axis, center, width, ds ) - # print('center after get_window_parameters: ', center) + # print('center after get_window_parameters: ', center) if field_parameters is None: field_parameters = {} @@ -2473,7 +2475,12 @@ def __init__( # get3bounds gets a depth 0.5 * diagonal + margin in the # depth=None case. (bounds, center_rot) = get_oblique_window_parameters( - normal, center, width, ds, depth=depth, get3bounds=True, + normal, + center, + width, + ds, + depth=depth, + get3bounds=True, ) # will probably fail if you try to project an SPH and non-SPH # field in a single call @@ -2489,16 +2496,15 @@ def __init__( is_sph_field = finfo.is_sph_field particle_datasets = (ParticleDataset, StreamParticlesDataset) - if isinstance(data_source.ds, particle_datasets) and is_sph_field: - center_use = parse_center_array(center, ds=data_source.ds, - axis=None) + if isinstance(data_source.ds, particle_datasets) and is_sph_field: + center_use = parse_center_array(center, ds=data_source.ds, axis=None) else: center_use = center_rot fields = list(iter_fields(fields))[:] - #oap_width = ds.arr( + # oap_width = ds.arr( # (bounds[1] - bounds[0], # bounds[3] - bounds[2]) - #) + # ) OffAxisProj = OffAxisProjectionDummyDataSource( center_use, ds, diff --git a/yt/visualization/tests/test_offaxisprojection.py b/yt/visualization/tests/test_offaxisprojection.py index 89a610f298..34dfbf49aa 100644 --- a/yt/visualization/tests/test_offaxisprojection.py +++ b/yt/visualization/tests/test_offaxisprojection.py @@ -243,13 +243,13 @@ def _vlos_sq(field, data): ## first assert_rel_equal argument. The compute_stddev_image ## function used in OffAxisProjectionPlot checks for and deals ## with these cases. - #assert_rel_equal( + # assert_rel_equal( # np.sqrt( # p1.frb["gas", "velocity_los_squared"] - p1.frb["gas", "velocity_los"] ** 2 # ), # p2.frb["gas", "velocity_los"], # 10, - #) + # ) p1_expsq = p1.frb["gas", "velocity_los_squared"] p1_sqexp = p1.frb["gas", "velocity_los"] ** 2 p1res = np.sqrt(p1_expsq - p1_sqexp) @@ -257,11 +257,13 @@ def _vlos_sq(field, data): # the absolute values are much smaller than the smallest # postive values of **2 and **2 # (i.e., the difference is pretty much zero) - mindiff = 1e-10 * min(np.min(p1_expsq[p1_expsq > 0]), - np.min(p1_sqexp[p1_sqexp > 0])) + mindiff = 1e-10 * min( + np.min(p1_expsq[p1_expsq > 0]), np.min(p1_sqexp[p1_sqexp > 0]) + ) print(mindiff) - setzero = np.logical_and(p1_expsq - p1_sqexp < 0, - p1_expsq - p1_sqexp > -1. * mindiff) - p1res[setzero] = 0. + setzero = np.logical_and( + p1_expsq - p1_sqexp < 0, p1_expsq - p1_sqexp > -1.0 * mindiff + ) + p1res[setzero] = 0.0 p2res = p2.frb["gas", "velocity_los"] assert_rel_equal(p1res, p2res, 10) diff --git a/yt/visualization/tests/test_offaxisprojection_pytestonly.py b/yt/visualization/tests/test_offaxisprojection_pytestonly.py index 5a1f013313..67db76db37 100644 --- a/yt/visualization/tests/test_offaxisprojection_pytestonly.py +++ b/yt/visualization/tests/test_offaxisprojection_pytestonly.py @@ -10,18 +10,20 @@ ) from yt.visualization.api import ProjectionPlot + @pytest.mark.parametrize("weighted", [True, False]) @pytest.mark.parametrize("periodic", [True, False]) -@pytest.mark.parametrize("depth", [None, (1., "cm"), (0.5, "cm")]) +@pytest.mark.parametrize("depth", [None, (1.0, "cm"), (0.5, "cm")]) @pytest.mark.parametrize("shiftcenter", [False, True]) -@pytest.mark.parametrize("northvector", [None, (1.e-4, 1., 0.)]) +@pytest.mark.parametrize("northvector", [None, (1.0e-4, 1.0, 0.0)]) def test_sph_proj_general_offaxis( - northvector: tuple[float, float, float] | None, - shiftcenter: bool, - depth: tuple[float, str] | None, - periodic: bool, - weighted: bool) -> None: - ''' + northvector: tuple[float, float, float] | None, + shiftcenter: bool, + depth: tuple[float, str] | None, + periodic: bool, + weighted: bool, +) -> None: + """ Same as the on-axis projections, but we rotate the basis vectors to test whether roations are handled ok. the rotation is chosen to be small so that in/exclusion of particles within bboxes, etc. @@ -49,55 +51,61 @@ def test_sph_proj_general_offaxis( Returns: -------- None - ''' + """ if shiftcenter: - center = np.array((0.625, 0.625, 0.625)) # cm + center = np.array((0.625, 0.625, 0.625)) # cm else: - center = np.array((1.5, 1.5, 1.5)) # cm - bbox = unyt.unyt_array(np.array([[0., 3.], [0., 3.], [0., 3.]]), 'cm') + center = np.array((1.5, 1.5, 1.5)) # cm + bbox = unyt.unyt_array(np.array([[0.0, 3.0], [0.0, 3.0], [0.0, 3.0]]), "cm") hsml_factor = 0.5 unitrho = 1.5 + # test correct centering, particle selection def makemasses(i, j, k): if i == j == k == 1: - return 2. + return 2.0 else: - return 1. + return 1.0 # result shouldn't depend explicitly on the center if we re-center # the data, unless we get cut-offs in the non-periodic case # *almost* the z-axis # try to make sure dl differences from periodic wrapping are small epsilon = 1e-4 - projaxis = np.array([epsilon, 0.00, np.sqrt(1. - epsilon**2)]) + projaxis = np.array([epsilon, 0.00, np.sqrt(1.0 - epsilon**2)]) e1dir = projaxis / np.sqrt(np.sum(projaxis**2)) # TODO: figure out other (default) axes for basis vectors here if northvector is None: - e2dir = np.array([0., 1., 0.]) + e2dir = np.array([0.0, 1.0, 0.0]) else: e2dir = np.asarray(northvector) - e2dir = e2dir - np.sum(e1dir * e2dir) * e2dir # orthonormalize + e2dir = e2dir - np.sum(e1dir * e2dir) * e2dir # orthonormalize e2dir /= np.sqrt(np.sum(e2dir**2)) e3dir = np.cross(e1dir, e2dir) - ds = fake_sph_flexible_grid_ds(hsml_factor=hsml_factor, nperside=3, - periodic=periodic, - offsets=np.full(3, 0.5), - massgenerator=makemasses, - unitrho=unitrho, - bbox=bbox.v, - recenter=center, - e1hat=e1dir, e2hat=e2dir, e3hat=e3dir) + ds = fake_sph_flexible_grid_ds( + hsml_factor=hsml_factor, + nperside=3, + periodic=periodic, + offsets=np.full(3, 0.5), + massgenerator=makemasses, + unitrho=unitrho, + bbox=bbox.v, + recenter=center, + e1hat=e1dir, + e2hat=e2dir, + e3hat=e3dir, + ) source = ds.all_data() # couple to dataset -> right unit registry - center = ds.arr(center, 'cm') - #print('position:\n', source['gas','position']) + center = ds.arr(center, "cm") + # print('position:\n', source['gas','position']) # m / rho, factor 1. / hsml**2 is included in the kernel integral # (density is adjusted, so same for center particle) - prefactor = 1. / unitrho #/ (0.5 * 0.5)**2 - dl_cen = integrate_kernel(cubicspline_python, 0., 0.25) + prefactor = 1.0 / unitrho # / (0.5 * 0.5)**2 + dl_cen = integrate_kernel(cubicspline_python, 0.0, 0.25) if weighted: toweight_field = ("gas", "density") @@ -105,39 +113,55 @@ def makemasses(i, j, k): toweight_field = None # we don't actually want a plot, it's just a straightforward, # common way to get an frb / image array - prj = ProjectionPlot(ds, projaxis, ("gas", "density"), - width=(2.5, "cm"), - weight_field=toweight_field, - buff_size=(5, 5), - center=center, - data_source=source, - north_vector=northvector, - depth=depth) - img = prj.frb.data[('gas', 'density')] + prj = ProjectionPlot( + ds, + projaxis, + ("gas", "density"), + width=(2.5, "cm"), + weight_field=toweight_field, + buff_size=(5, 5), + center=center, + data_source=source, + north_vector=northvector, + depth=depth, + ) + img = prj.frb.data[("gas", "density")] if weighted: # periodic shifts will modify the (relative) dl values a bit - expected_out = np.zeros((5, 5,), dtype=img.v.dtype) + expected_out = np.zeros( + ( + 5, + 5, + ), + dtype=img.v.dtype, + ) expected_out[::2, ::2] = unitrho if depth is None: expected_out[2, 2] *= 1.5 else: # only 2 * unitrho element included - expected_out[2, 2] *= 2. + expected_out[2, 2] *= 2.0 else: - expected_out = np.zeros((5, 5,), dtype=img.v.dtype) + expected_out = np.zeros( + ( + 5, + 5, + ), + dtype=img.v.dtype, + ) expected_out[::2, ::2] = dl_cen * prefactor * unitrho if depth is None: # 3 particles per l.o.s., including the denser one - expected_out *= 3. - expected_out[2, 2] *= 4. / 3. + expected_out *= 3.0 + expected_out[2, 2] *= 4.0 / 3.0 else: # 1 particle per l.o.s., including the denser one - expected_out[2, 2] *= 2. + expected_out[2, 2] *= 2.0 # grid is shifted to the left -> 'missing' stuff at the left if (not periodic) and shiftcenter: - expected_out[:1, :] = 0. - expected_out[:, :1] = 0. - #print(axis, shiftcenter, depth, periodic, weighted) - print('expected:\n', expected_out) - print('recovered:\n', img.v) + expected_out[:1, :] = 0.0 + expected_out[:, :1] = 0.0 + # print(axis, shiftcenter, depth, periodic, weighted) + print("expected:\n", expected_out) + print("recovered:\n", img.v) assert_rel_equal(expected_out, img.v, 4) diff --git a/yt/visualization/volume_rendering/off_axis_projection.py b/yt/visualization/volume_rendering/off_axis_projection.py index b3f579b9cd..d3765222c9 100644 --- a/yt/visualization/volume_rendering/off_axis_projection.py +++ b/yt/visualization/volume_rendering/off_axis_projection.py @@ -159,8 +159,7 @@ def off_axis_projection( if hasattr(depth, "units"): depth = depth.to("code_length").d - #depth = data_source.ds.arr(depth, "code_length") - + # depth = data_source.ds.arr(depth, "code_length") if hasattr(data_source.ds, "_sph_ptypes"): if method != "integrate": @@ -220,12 +219,12 @@ def off_axis_projection( mask = np.ones_like(buf, dtype="uint8") ## width from fixed_resolution.py is just the size of the domain - #x_min = center[0] - width[0] / 2 - #x_max = center[0] + width[0] / 2 - #y_min = center[1] - width[1] / 2 - #y_max = center[1] + width[1] / 2 - #z_min = center[2] - width[2] / 2 - #z_max = center[2] + width[2] / 2 + # x_min = center[0] - width[0] / 2 + # x_max = center[0] + width[0] / 2 + # y_min = center[1] - width[1] / 2 + # y_max = center[1] + width[1] / 2 + # z_min = center[2] - width[2] / 2 + # z_max = center[2] + width[2] / 2 periodic = data_source.ds.periodicity le = data_source.ds.domain_left_edge.to("code_length").d diff --git a/yt/visualization/volume_rendering/old_camera.py b/yt/visualization/volume_rendering/old_camera.py index 0ca8489814..fd171dbc80 100644 --- a/yt/visualization/volume_rendering/old_camera.py +++ b/yt/visualization/volume_rendering/old_camera.py @@ -2438,6 +2438,7 @@ def _render(self, double_check, num_threads, image, sampler, msg): data_object_registry["stereospherical_camera"] = StereoSphericalCamera + # replaced in volume_rendering API by the function of the same name in # yt/visualization/volume_rendering/off_axis_projection def off_axis_projection( From d4e5bfb9728e49211162a079facef04dccd90736 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 20:21:29 +0000 Subject: [PATCH 119/186] [pre-commit.ci] auto fixes from pre-commit.com hooks keep those fixes in merge for more information, see https://pre-commit.ci --- yt/testing.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/yt/testing.py b/yt/testing.py index 43f4ba3e60..b4d00c7875 100644 --- a/yt/testing.py +++ b/yt/testing.py @@ -84,9 +84,9 @@ def assert_rel_equal(a1, a2, decimals, err_msg="", verbose=True): # tested: volume integral is 1. - -def cubicspline_python(x: Union[float,npt.ndarray[float]], - ) -> npt.ndarray[float]: +def cubicspline_python( + x: Union[float, npt.ndarray[float]], +) -> npt.ndarray[float]: """ cubic spline SPH kernel function for testing against more effiecient cython methods @@ -143,6 +143,7 @@ def integrate_kernel( integral = np.sum(spv * dx, axis=0) return pre * integral + _zeroperiods = np.array([0.0, 0.0, 0.0]) @@ -813,8 +814,8 @@ def fake_sph_flexible_grid_ds( offsets: np.ndarray[float] = _floathalves, massgenerator: Callable[[int, int, int], float] = constantmass, unitrho: float = 1.0, - bbox: np.ndarray | None = None, - recenter: np.ndarray | None = None, + bbox: Union[npt.ndarray[float], None] = None, + recenter: Union[npt.ndarray[float], None] = None, ) -> StreamParticlesDataset: """Returns an in-memory SPH dataset useful for testing @@ -937,8 +938,8 @@ def fake_sph_flexible_grid_ds( def fake_random_sph_ds( npart: int, - bbox: np.ndarray, - periodic: bool | tuple[bool, bool, bool] = True, + bbox: npt.ndarray[float], + periodic: Union[bool, tuple[bool, bool, bool]] = True, massrange: tuple[float, float] = (0.5, 2.0), hsmlrange: tuple[float, float] = (0.5, 2.0), unitrho: float = 1.0, From b4f493587add9a1c53e3ed541a49630c4e4ea08c Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Thu, 18 Jul 2024 15:44:36 -0500 Subject: [PATCH 120/186] attempting to pass type-checking --- .../tests/test_sph_pixelization_pytestonly.py | 20 ++++++++--- yt/loaders.py | 2 +- yt/testing.py | 34 +++++++++---------- .../test_offaxisprojection_pytestonly.py | 6 ++-- 4 files changed, 38 insertions(+), 24 deletions(-) diff --git a/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py b/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py index 945286af0c..44983c1120 100644 --- a/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py +++ b/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py @@ -1,3 +1,5 @@ +from typing import Union + import numpy as np import pytest import unyt @@ -20,7 +22,11 @@ @pytest.mark.parametrize("shiftcenter", [False, True]) @pytest.mark.parametrize("axis", [0, 1, 2]) def test_sph_proj_general_alongaxes( - axis: int, shiftcenter: bool, depth: float | None, periodic: bool, weighted: bool + axis: int, + shiftcenter: bool, + depth: Union[float, None], + periodic: bool, + weighted: bool, ) -> None: """ The previous projection tests were for a specific issue. @@ -161,7 +167,10 @@ def makemasses(i, j, k): @pytest.mark.parametrize("zoff", [0.0, 0.1, 0.5, 1.0]) @pytest.mark.parametrize("axis", [0, 1, 2]) def test_sph_slice_general_alongaxes( - axis: int, shiftcenter: bool, periodic: bool, zoff: float + axis: int, + shiftcenter: bool, + periodic: bool, + zoff: float, ) -> None: """ Particles at [0.5, 1.5, 2.5] (in each coordinate) @@ -307,7 +316,7 @@ def makemasses(i, j, k): @pytest.mark.parametrize("northvector", [None, (1.0e-4, 1.0, 0.0)]) @pytest.mark.parametrize("zoff", [0.0, 0.1, 0.5, 1.0]) def test_sph_slice_general_offaxis( - northvector: tuple[float, float, float] | None, + northvector: Union[tuple[float, float, float], None], shiftcenter: bool, zoff: float, periodic: bool, @@ -461,7 +470,10 @@ def makemasses(i, j, k): # only axis-aligned; testing YTArbitraryGrid, YTCoveringGrid @pytest.mark.parametrize("periodic", [True, False, (True, True, False)]) @pytest.mark.parametrize("wholebox", [True, False]) -def test_sph_grid(periodic: bool | tuple[bool, bool, bool], wholebox: bool): +def test_sph_grid( + periodic: Union[bool, tuple[bool, bool, bool]], + wholebox: bool, +) -> None: bbox = np.array([[-1.0, 3.0], [1.0, 5.2], [-1.0, 3.0]]) ds = fake_random_sph_ds(50, bbox, periodic=periodic) diff --git a/yt/loaders.py b/yt/loaders.py index aa7abea55f..1a19c5583a 100644 --- a/yt/loaders.py +++ b/yt/loaders.py @@ -694,7 +694,7 @@ def load_amr_grids( def load_particles( - data: dict[AnyFieldKey, Union[np.ndarray, tuple[np.ndarry, str]]], + data: dict[AnyFieldKey, Union[np.ndarray, tuple[np.ndarray, str]]], length_unit=None, bbox=None, sim_time=None, diff --git a/yt/testing.py b/yt/testing.py index b4d00c7875..b809835310 100644 --- a/yt/testing.py +++ b/yt/testing.py @@ -9,7 +9,7 @@ from functools import wraps from importlib.util import find_spec from shutil import which -from typing import Callable, Union +from typing import Any, Callable, Union from unittest import SkipTest import matplotlib @@ -85,8 +85,8 @@ def assert_rel_equal(a1, a2, decimals, err_msg="", verbose=True): # tested: volume integral is 1. def cubicspline_python( - x: Union[float, npt.ndarray[float]], -) -> npt.ndarray[float]: + x: Union[float, np.ndarray], +) -> np.ndarray: """ cubic spline SPH kernel function for testing against more effiecient cython methods @@ -148,11 +148,11 @@ def integrate_kernel( def distancematrix( - pos3_i0: np.ndarray[float], - pos3_i1: np.ndarray[float], - periodic: tuple[bool] = (True,) * 3, + pos3_i0: np.ndarray, + pos3_i1: np.ndarray, + periodic: tuple[bool, bool, bool] = (True,) * 3, periods: np.ndarray = _zeroperiods, -) -> np.ndarray[float]: +) -> np.ndarray: """ Calculates the distances between two arrays of points. @@ -808,14 +808,14 @@ def fake_sph_flexible_grid_ds( hsml_factor: float = 1.0, nperside: int = 3, periodic: bool = True, - e1hat: np.ndarray[float] = _xhat, - e2hat: np.ndarray[float] = _yhat, - e3hat: np.ndarray[float] = _zhat, - offsets: np.ndarray[float] = _floathalves, + e1hat: np.ndarray = _xhat, + e2hat: np.ndarray = _yhat, + e3hat: np.ndarray = _zhat, + offsets: np.ndarray = _floathalves, massgenerator: Callable[[int, int, int], float] = constantmass, unitrho: float = 1.0, - bbox: Union[npt.ndarray[float], None] = None, - recenter: Union[npt.ndarray[float], None] = None, + bbox: Union[np.ndarray, None] = None, + recenter: Union[np.ndarray, None] = None, ) -> StreamParticlesDataset: """Returns an in-memory SPH dataset useful for testing @@ -913,9 +913,9 @@ def fake_sph_flexible_grid_ds( okinds = slice(None, None, None) data = { - "particle_position_x": (np.copy(pos[okinds, 0]), "cm"), - "particle_position_y": (np.copy(pos[okinds, 1]), "cm"), - "particle_position_z": (np.copy(pos[okinds, 2]), "cm"), + "particle_position_x": (np.copy(pos[:, 0][okinds]), "cm"), + "particle_position_y": (np.copy(pos[:, 1][okinds]), "cm"), + "particle_position_z": (np.copy(pos[:, 2][okinds]), "cm"), "particle_mass": (mass[okinds], "g"), "particle_velocity_x": (np.zeros(npart), "cm/s"), "particle_velocity_y": (np.zeros(npart), "cm/s"), @@ -938,7 +938,7 @@ def fake_sph_flexible_grid_ds( def fake_random_sph_ds( npart: int, - bbox: npt.ndarray[float], + bbox: np.ndarray, periodic: Union[bool, tuple[bool, bool, bool]] = True, massrange: tuple[float, float] = (0.5, 2.0), hsmlrange: tuple[float, float] = (0.5, 2.0), diff --git a/yt/visualization/tests/test_offaxisprojection_pytestonly.py b/yt/visualization/tests/test_offaxisprojection_pytestonly.py index 67db76db37..6c9010b8a5 100644 --- a/yt/visualization/tests/test_offaxisprojection_pytestonly.py +++ b/yt/visualization/tests/test_offaxisprojection_pytestonly.py @@ -1,3 +1,5 @@ +from typing import Union + import numpy as np import pytest import unyt @@ -17,9 +19,9 @@ @pytest.mark.parametrize("shiftcenter", [False, True]) @pytest.mark.parametrize("northvector", [None, (1.0e-4, 1.0, 0.0)]) def test_sph_proj_general_offaxis( - northvector: tuple[float, float, float] | None, + northvector: Union[tuple[float, float, float], None], shiftcenter: bool, - depth: tuple[float, str] | None, + depth: Union[tuple[float, str], None], periodic: bool, weighted: bool, ) -> None: From ebb357cdbada870627eedf600124af75c1f82b28 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Thu, 18 Jul 2024 15:46:17 -0500 Subject: [PATCH 121/186] ruff fix --- yt/testing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/yt/testing.py b/yt/testing.py index b809835310..678e769f65 100644 --- a/yt/testing.py +++ b/yt/testing.py @@ -9,12 +9,11 @@ from functools import wraps from importlib.util import find_spec from shutil import which -from typing import Any, Callable, Union +from typing import Callable, Union from unittest import SkipTest import matplotlib import numpy as np -import numpy.typing as npt from more_itertools import always_iterable from numpy.random import RandomState from unyt.exceptions import UnitOperationError From f09851c3eec132c65f78ef00841a9aae2518d305 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Thu, 18 Jul 2024 21:52:36 -0500 Subject: [PATCH 122/186] more attempted type check fixes --- yt/loaders.py | 4 ++-- yt/testing.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/yt/loaders.py b/yt/loaders.py index 1a19c5583a..df0b32ce6d 100644 --- a/yt/loaders.py +++ b/yt/loaders.py @@ -10,7 +10,7 @@ import types import warnings from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Mapping, Optional, Union, cast from urllib.parse import urlsplit import numpy as np @@ -694,7 +694,7 @@ def load_amr_grids( def load_particles( - data: dict[AnyFieldKey, Union[np.ndarray, tuple[np.ndarray, str]]], + data: Mapping[AnyFieldKey, Union[np.ndarray, tuple[np.ndarray, str]]], length_unit=None, bbox=None, sim_time=None, diff --git a/yt/testing.py b/yt/testing.py index 678e769f65..53b4689e65 100644 --- a/yt/testing.py +++ b/yt/testing.py @@ -909,12 +909,12 @@ def fake_sph_flexible_grid_ds( okinds &= pos[:, ax] >= bbox[ax, 0] npart = sum(okinds) else: - okinds = slice(None, None, None) + okinds: slice = slice(None, None, None) data = { - "particle_position_x": (np.copy(pos[:, 0][okinds]), "cm"), - "particle_position_y": (np.copy(pos[:, 1][okinds]), "cm"), - "particle_position_z": (np.copy(pos[:, 2][okinds]), "cm"), + "particle_position_x": (np.copy(pos[okinds, 0]), "cm"), + "particle_position_y": (np.copy(pos[okinds, 1]), "cm"), + "particle_position_z": (np.copy(pos[okinds, 2]), "cm"), "particle_mass": (mass[okinds], "g"), "particle_velocity_x": (np.zeros(npart), "cm/s"), "particle_velocity_y": (np.zeros(npart), "cm/s"), From 987e50e434b00bbf1e18329f7524a75249f8121e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 19 Jul 2024 03:01:41 +0000 Subject: [PATCH 123/186] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- yt/loaders.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yt/loaders.py b/yt/loaders.py index df0b32ce6d..32adc89fae 100644 --- a/yt/loaders.py +++ b/yt/loaders.py @@ -9,8 +9,9 @@ import time import types import warnings +from collections.abc import Mapping from pathlib import Path -from typing import TYPE_CHECKING, Any, Mapping, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Optional, Union, cast from urllib.parse import urlsplit import numpy as np From 422e7161f7988ef2b746e44fc3e2a8463e790695 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Thu, 18 Jul 2024 23:10:21 -0500 Subject: [PATCH 124/186] more attempted type-checking fixes --- yt/loaders.py | 2 +- yt/testing.py | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/yt/loaders.py b/yt/loaders.py index 32adc89fae..8494649549 100644 --- a/yt/loaders.py +++ b/yt/loaders.py @@ -834,7 +834,7 @@ def parse_unit(unit, dimension): field_units, data, _ = process_data(data) sfh = StreamDictFieldHandler() - pdata: dict[AnyFieldKey, np.ndarray] = {} + pdata: dict[AnyFieldKey, Union[np.ndarray, tuple[np.ndarray, str]]] = {} for key in data.keys(): field: FieldKey if not isinstance(key, tuple): diff --git a/yt/testing.py b/yt/testing.py index 53b4689e65..869f695958 100644 --- a/yt/testing.py +++ b/yt/testing.py @@ -6,6 +6,7 @@ import sys import tempfile import unittest +from collections.abc import Mapping from functools import wraps from importlib.util import find_spec from shutil import which @@ -19,6 +20,7 @@ from unyt.exceptions import UnitOperationError from yt._maintenance.deprecation import issue_deprecation_warning +from yt._typing import AnyFieldKey from yt.config import ytcfg from yt.frontends.stream.data_structures import StreamParticlesDataset from yt.funcs import is_sequence @@ -909,18 +911,18 @@ def fake_sph_flexible_grid_ds( okinds &= pos[:, ax] >= bbox[ax, 0] npart = sum(okinds) else: - okinds: slice = slice(None, None, None) + okinds = np.ones((npart,), dtype=bool) - data = { + data: Mapping[AnyFieldKey, tuple[np.ndarray, str]] = { "particle_position_x": (np.copy(pos[okinds, 0]), "cm"), "particle_position_y": (np.copy(pos[okinds, 1]), "cm"), "particle_position_z": (np.copy(pos[okinds, 2]), "cm"), - "particle_mass": (mass[okinds], "g"), + "particle_mass": (np.copy(mass[okinds]), "g"), "particle_velocity_x": (np.zeros(npart), "cm/s"), "particle_velocity_y": (np.zeros(npart), "cm/s"), "particle_velocity_z": (np.zeros(npart), "cm/s"), "smoothing_length": (np.ones(npart) * 0.5 * hsml_factor, "cm"), - "density": (rho[okinds], "g/cm**3"), + "density": (np.copy(rho[okinds]), "g/cm**3"), } ds = load_particles( @@ -981,7 +983,7 @@ def fake_random_sph_ds( hsml = gen.uniform(low=hsmlrange[0], high=hsmlrange[1], size=npart) dens = mass / hsml**3 * unitrho - data = { + data: Mapping[AnyFieldKey, tuple[np.ndarray, str]] = { "particle_position_x": (posx, "cm"), "particle_position_y": (posy, "cm"), "particle_position_z": (posz, "cm"), From 02011d04817a3c8ceda79864f0647eede4f9a102 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Fri, 19 Jul 2024 15:06:39 -0500 Subject: [PATCH 125/186] avoid test failures from divide by zero errors --- .../tests/test_sph_pixelization_pytestonly.py | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py b/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py index 44983c1120..6010644790 100644 --- a/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py +++ b/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py @@ -279,18 +279,18 @@ def makemasses(i, j, k): (ad[("gas", "z")]).to("cm"), ] ).T - print("sphcoords:") - print(sphcoords) - print("gridcoords:") - print(gridcoords) + #print("sphcoords:") + #print(sphcoords) + #print("gridcoords:") + #print(gridcoords) dists = distancematrix( gridcoords, sphcoords, periodic=(periodic,) * 3, periods=np.array([3.0, 3.0, 3.0]), ) - print("dists <= 1:") - print(dists <= 1) + #print("dists <= 1:") + #print(dists <= 1) sml = (ad[("gas", "smoothing_length")]).to("cm") normkern = cubicspline_python(dists / sml.v[np.newaxis, :]) sphcontr = normkern / sml[np.newaxis, :] ** 3 * ad[("gas", "mass")] @@ -302,9 +302,11 @@ def makemasses(i, j, k): / ad[("gas", "density")] ) weights = np.sum(sphweights, axis=1) - expected = contsum / weights + nzeromask = np.logical_not(weights == 0) + expected = np.zeros(weights.shape, weights.dtype) + expected[nzeromask] = contsum[nzeromask] / weights[nzeromask] expected = expected.reshape((outgridsize, outgridsize)) - expected[np.isnan(expected)] = 0.0 # convention in the slices + #expected[np.isnan(expected)] = 0.0 # convention in the slices print("expected:\n", expected.v) print("recovered:\n", img.v) @@ -396,7 +398,7 @@ def makemasses(i, j, k): source = ds.all_data() # couple to dataset -> right unit registry center = ds.arr(center, "cm") - print("position:\n", source["gas", "position"]) + #print("position:\n", source["gas", "position"]) slc = yt.SlicePlot( ds, e1dir, @@ -429,8 +431,8 @@ def makemasses(i, j, k): + ygrid[:, np.newaxis] * e2dir[np.newaxis, :] + zgrid[:, np.newaxis] * e1dir[np.newaxis, :] ) - print("gridcoords:") - print(gridcoords) + #print("gridcoords:") + #print(gridcoords) ad = ds.all_data() sphcoords = np.array( [ @@ -456,10 +458,12 @@ def makemasses(i, j, k): / ad[("gas", "density")] ) weights = np.sum(sphweights, axis=1) - expected = contsum / weights + nzeromask = np.logical_not(weights == 0) + expected = np.zeros(weights.shape, weights.dtype) + expected[nzeromask] = contsum[nzeromask] / weights[nzeromask] expected = expected.reshape((outgridsize, outgridsize)) expected = expected.T # transposed for image plotting - expected[np.isnan(expected)] = 0.0 # convention in the slices + # expected[np.isnan(expected)] = 0.0 # convention in the slices # print(axis, shiftcenter, depth, periodic, weighted) print("expected:\n", expected.v) @@ -484,8 +488,8 @@ def test_sph_grid( left = bbox[:, 0].copy() level = 2 ncells = np.array([2**level] * 3) - print("left: ", left) - print("ncells: ", ncells) + #print("left: ", left) + #print("ncells: ", ncells) resgrid = ds.covering_grid(level, tuple(left), ncells) right = bbox[:, 1].copy() xedges = np.linspace(left[0], right[0], ncells[0] + 1) @@ -531,9 +535,11 @@ def test_sph_grid( / ad[("gas", "density")] ) weights = np.sum(sphweights, axis=1) - expected = contsum / weights + nzeromask = np.logical_not(weights == 0) + expected = np.zeros(weights.shape, weights.dtype) + expected[nzeromask] = contsum[nzeromask] / weights[nzeromask] expected = expected.reshape(outshape) - expected[np.isnan(expected)] = 0.0 # convention in the slices + #expected[np.isnan(expected)] = 0.0 # convention in the slices # print(axis, shiftcenter, depth, periodic, weighted) print("expected:\n", expected.v) From be087cee7eca4865315c8727f6c78899eaaea4e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Fri, 19 Jul 2024 10:02:50 +0200 Subject: [PATCH 126/186] BUG: fix a defect in CartesianCoordinateHandler._ortho_pixelize (ValueError: setting an array element with a sequence.) --- yt/geometry/coordinates/cartesian_coordinates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/geometry/coordinates/cartesian_coordinates.py b/yt/geometry/coordinates/cartesian_coordinates.py index 7c68b8b0fe..4fed1977c4 100644 --- a/yt/geometry/coordinates/cartesian_coordinates.py +++ b/yt/geometry/coordinates/cartesian_coordinates.py @@ -332,7 +332,7 @@ def _ortho_pixelize( period3 = self.period[:].copy() # dummy here period3[0] = self.period[self.x_axis[dim]] period3[1] = self.period[self.y_axis[dim]] - zax = list({0, 1, 2} - {self.x_axis[dim], self.y_axis[dim]}) + zax = list({0, 1, 2} - {self.x_axis[dim], self.y_axis[dim]})[0] period3[2] = self.period[zax] if hasattr(period2, "in_units"): period2 = period2.in_units("code_length").d From 374a758e3fe1ce565c1c4be56ac41958782563c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Fri, 19 Jul 2024 11:00:49 +0200 Subject: [PATCH 127/186] BUG: fix a defect in OffAxisProjectionFixedResolutionBuffer._generate_image_and_mask --- yt/visualization/fixed_resolution.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yt/visualization/fixed_resolution.py b/yt/visualization/fixed_resolution.py index f597393a5d..000c2dba8b 100644 --- a/yt/visualization/fixed_resolution.py +++ b/yt/visualization/fixed_resolution.py @@ -639,6 +639,7 @@ def _generate_image_and_mask(self, item) -> None: self.bounds[5] - self.bounds[4], ) ) + depth = dd.depth[0] if dd.depth is not None else None buff = off_axis_projection( dd.dd, dd.center, @@ -651,7 +652,7 @@ def _generate_image_and_mask(self, item) -> None: no_ghost=dd.no_ghost, interpolated=dd.interpolated, north_vector=dd.north_vector, - depth=dd.depth, + depth=depth, method=dd.method, ) if self.data_source.moment == 2: From e8a0371fb25804cb45162cdff133c66e35b6373a Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Fri, 19 Jul 2024 15:17:20 -0500 Subject: [PATCH 128/186] ruff fix --- .../coordinates/tests/test_sph_pixelization_pytestonly.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py b/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py index 6010644790..476a1bc9c7 100644 --- a/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py +++ b/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py @@ -395,10 +395,10 @@ def makemasses(i, j, k): e3hat=e3dir, ) - source = ds.all_data() + # source = ds.all_data() # couple to dataset -> right unit registry center = ds.arr(center, "cm") - #print("position:\n", source["gas", "position"]) + # print("position:\n", source["gas", "position"]) slc = yt.SlicePlot( ds, e1dir, From 7f129c1bfc0cadc67e6bdcec9fe72c181f505758 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Fri, 19 Jul 2024 15:20:05 -0500 Subject: [PATCH 129/186] avoid 'divide by zero' errors --- yt/visualization/volume_rendering/off_axis_projection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yt/visualization/volume_rendering/off_axis_projection.py b/yt/visualization/volume_rendering/off_axis_projection.py index d3765222c9..5c547f61c9 100644 --- a/yt/visualization/volume_rendering/off_axis_projection.py +++ b/yt/visualization/volume_rendering/off_axis_projection.py @@ -525,7 +525,8 @@ def temp_weightfield(field, data): image *= dl else: mask = image[:, :, 1] == 0 - image[:, :, 0] /= image[:, :, 1] + nmask = np.logical_not(mask) + image[:, :, 0][nmask] /= image[:, :, 1][nmask] image[mask] = 0 return image[:, :, 0] From 6f57981a49cb4cfd69d123ccf85206425aa54fff Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 19 Jul 2024 21:02:46 +0000 Subject: [PATCH 130/186] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../tests/test_sph_pixelization_pytestonly.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py b/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py index 476a1bc9c7..21e4f8d17d 100644 --- a/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py +++ b/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py @@ -279,18 +279,18 @@ def makemasses(i, j, k): (ad[("gas", "z")]).to("cm"), ] ).T - #print("sphcoords:") - #print(sphcoords) - #print("gridcoords:") - #print(gridcoords) + # print("sphcoords:") + # print(sphcoords) + # print("gridcoords:") + # print(gridcoords) dists = distancematrix( gridcoords, sphcoords, periodic=(periodic,) * 3, periods=np.array([3.0, 3.0, 3.0]), ) - #print("dists <= 1:") - #print(dists <= 1) + # print("dists <= 1:") + # print(dists <= 1) sml = (ad[("gas", "smoothing_length")]).to("cm") normkern = cubicspline_python(dists / sml.v[np.newaxis, :]) sphcontr = normkern / sml[np.newaxis, :] ** 3 * ad[("gas", "mass")] @@ -306,7 +306,7 @@ def makemasses(i, j, k): expected = np.zeros(weights.shape, weights.dtype) expected[nzeromask] = contsum[nzeromask] / weights[nzeromask] expected = expected.reshape((outgridsize, outgridsize)) - #expected[np.isnan(expected)] = 0.0 # convention in the slices + # expected[np.isnan(expected)] = 0.0 # convention in the slices print("expected:\n", expected.v) print("recovered:\n", img.v) @@ -431,8 +431,8 @@ def makemasses(i, j, k): + ygrid[:, np.newaxis] * e2dir[np.newaxis, :] + zgrid[:, np.newaxis] * e1dir[np.newaxis, :] ) - #print("gridcoords:") - #print(gridcoords) + # print("gridcoords:") + # print(gridcoords) ad = ds.all_data() sphcoords = np.array( [ @@ -488,8 +488,8 @@ def test_sph_grid( left = bbox[:, 0].copy() level = 2 ncells = np.array([2**level] * 3) - #print("left: ", left) - #print("ncells: ", ncells) + # print("left: ", left) + # print("ncells: ", ncells) resgrid = ds.covering_grid(level, tuple(left), ncells) right = bbox[:, 1].copy() xedges = np.linspace(left[0], right[0], ncells[0] + 1) @@ -539,7 +539,7 @@ def test_sph_grid( expected = np.zeros(weights.shape, weights.dtype) expected[nzeromask] = contsum[nzeromask] / weights[nzeromask] expected = expected.reshape(outshape) - #expected[np.isnan(expected)] = 0.0 # convention in the slices + # expected[np.isnan(expected)] = 0.0 # convention in the slices # print(axis, shiftcenter, depth, periodic, weighted) print("expected:\n", expected.v) From 45574d62753e1dddf067c10c802a7c812edc8e6b Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Fri, 19 Jul 2024 17:16:06 -0500 Subject: [PATCH 131/186] fix .v on numpy array problem --- .../tests/test_sph_pixelization_pytestonly.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py b/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py index 21e4f8d17d..1ec291d5e6 100644 --- a/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py +++ b/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py @@ -308,9 +308,9 @@ def makemasses(i, j, k): expected = expected.reshape((outgridsize, outgridsize)) # expected[np.isnan(expected)] = 0.0 # convention in the slices - print("expected:\n", expected.v) + print("expected:\n", expected) print("recovered:\n", img.v) - assert_rel_equal(expected.v, img.v, 5) + assert_rel_equal(expected, img.v, 5) @pytest.mark.parametrize("periodic", [True, False]) @@ -466,9 +466,9 @@ def makemasses(i, j, k): # expected[np.isnan(expected)] = 0.0 # convention in the slices # print(axis, shiftcenter, depth, periodic, weighted) - print("expected:\n", expected.v) + print("expected:\n", expected) print("recovered:\n", img.v) - assert_rel_equal(expected.v, img.v, 4) + assert_rel_equal(expected, img.v, 4) # only axis-aligned; testing YTArbitraryGrid, YTCoveringGrid @@ -542,6 +542,6 @@ def test_sph_grid( # expected[np.isnan(expected)] = 0.0 # convention in the slices # print(axis, shiftcenter, depth, periodic, weighted) - print("expected:\n", expected.v) + print("expected:\n", expected) print("recovered:\n", res.v) - assert_rel_equal(expected.v, res.v, 4) + assert_rel_equal(expected, res.v, 4) From c653a4f5e97c7ad2cb2c60b0a5653e2434876c67 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Sat, 20 Jul 2024 14:38:19 -0500 Subject: [PATCH 132/186] remove printing in tests ('clogs up' test logs) --- .../tests/test_sph_pixelization_pytestonly.py | 12 ++++++------ .../tests/test_offaxisprojection_pytestonly.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py b/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py index 1ec291d5e6..75c7175e06 100644 --- a/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py +++ b/yt/geometry/coordinates/tests/test_sph_pixelization_pytestonly.py @@ -308,8 +308,8 @@ def makemasses(i, j, k): expected = expected.reshape((outgridsize, outgridsize)) # expected[np.isnan(expected)] = 0.0 # convention in the slices - print("expected:\n", expected) - print("recovered:\n", img.v) + # print("expected:\n", expected) + # print("recovered:\n", img.v) assert_rel_equal(expected, img.v, 5) @@ -466,8 +466,8 @@ def makemasses(i, j, k): # expected[np.isnan(expected)] = 0.0 # convention in the slices # print(axis, shiftcenter, depth, periodic, weighted) - print("expected:\n", expected) - print("recovered:\n", img.v) + # print("expected:\n", expected) + # print("recovered:\n", img.v) assert_rel_equal(expected, img.v, 4) @@ -542,6 +542,6 @@ def test_sph_grid( # expected[np.isnan(expected)] = 0.0 # convention in the slices # print(axis, shiftcenter, depth, periodic, weighted) - print("expected:\n", expected) - print("recovered:\n", res.v) + # print("expected:\n", expected) + # print("recovered:\n", res.v) assert_rel_equal(expected, res.v, 4) diff --git a/yt/visualization/tests/test_offaxisprojection_pytestonly.py b/yt/visualization/tests/test_offaxisprojection_pytestonly.py index 6c9010b8a5..f63ac924c3 100644 --- a/yt/visualization/tests/test_offaxisprojection_pytestonly.py +++ b/yt/visualization/tests/test_offaxisprojection_pytestonly.py @@ -164,6 +164,6 @@ def makemasses(i, j, k): expected_out[:1, :] = 0.0 expected_out[:, :1] = 0.0 # print(axis, shiftcenter, depth, periodic, weighted) - print("expected:\n", expected_out) - print("recovered:\n", img.v) + # print("expected:\n", expected_out) + # print("recovered:\n", img.v) assert_rel_equal(expected_out, img.v, 4) From 92a7127df7b1e07058c66c6bc2f41e6d8371b95e Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Sat, 20 Jul 2024 14:59:38 -0500 Subject: [PATCH 133/186] attempted fix test error from sqrt(< 0) --- yt/visualization/tests/test_offaxisprojection.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/yt/visualization/tests/test_offaxisprojection.py b/yt/visualization/tests/test_offaxisprojection.py index 34dfbf49aa..f72c4a4e63 100644 --- a/yt/visualization/tests/test_offaxisprojection.py +++ b/yt/visualization/tests/test_offaxisprojection.py @@ -252,7 +252,6 @@ def _vlos_sq(field, data): # ) p1_expsq = p1.frb["gas", "velocity_los_squared"] p1_sqexp = p1.frb["gas", "velocity_los"] ** 2 - p1res = np.sqrt(p1_expsq - p1_sqexp) # set values to zero that have **2 - **2 < 0, but # the absolute values are much smaller than the smallest # postive values of **2 and **2 @@ -260,10 +259,13 @@ def _vlos_sq(field, data): mindiff = 1e-10 * min( np.min(p1_expsq[p1_expsq > 0]), np.min(p1_sqexp[p1_sqexp > 0]) ) - print(mindiff) - setzero = np.logical_and( + # print(mindiff) + safeorbad = np.logical_not(np.logical_and( p1_expsq - p1_sqexp < 0, p1_expsq - p1_sqexp > -1.0 * mindiff - ) - p1res[setzero] = 0.0 + )) + # avoid errors from sqrt(negative) + # sqrt in zeros_like insures correct units + p1res = np.zeros_like(np.sqrt(p1_expsq)) + p1res[safeorbad] = np.sqrt(p1_expsq[safeorbad] - p1_sqexp[safeorbad]) p2res = p2.frb["gas", "velocity_los"] assert_rel_equal(p1res, p2res, 10) From 5587ff320e3df53606d9a4ea62fcc28ddd49659a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 20 Jul 2024 20:09:00 +0000 Subject: [PATCH 134/186] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- yt/visualization/tests/test_offaxisprojection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yt/visualization/tests/test_offaxisprojection.py b/yt/visualization/tests/test_offaxisprojection.py index f72c4a4e63..739e48d7ea 100644 --- a/yt/visualization/tests/test_offaxisprojection.py +++ b/yt/visualization/tests/test_offaxisprojection.py @@ -260,9 +260,9 @@ def _vlos_sq(field, data): np.min(p1_expsq[p1_expsq > 0]), np.min(p1_sqexp[p1_sqexp > 0]) ) # print(mindiff) - safeorbad = np.logical_not(np.logical_and( - p1_expsq - p1_sqexp < 0, p1_expsq - p1_sqexp > -1.0 * mindiff - )) + safeorbad = np.logical_not( + np.logical_and(p1_expsq - p1_sqexp < 0, p1_expsq - p1_sqexp > -1.0 * mindiff) + ) # avoid errors from sqrt(negative) # sqrt in zeros_like insures correct units p1res = np.zeros_like(np.sqrt(p1_expsq)) From d3bd38a9b46868f1b7ee71db8f89e39d209a5ec6 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Tue, 23 Jul 2024 16:13:09 -0500 Subject: [PATCH 135/186] docstring typo fix --- yt/visualization/volume_rendering/off_axis_projection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/visualization/volume_rendering/off_axis_projection.py b/yt/visualization/volume_rendering/off_axis_projection.py index 5c547f61c9..3c2359a3f8 100644 --- a/yt/visualization/volume_rendering/off_axis_projection.py +++ b/yt/visualization/volume_rendering/off_axis_projection.py @@ -81,7 +81,7 @@ def off_axis_projection( north_vector : optional, array_like, default None A vector that, if specified, restricts the orientation such that the north vector dotted into the image plane points "up". Useful for rotations - depth: float, tuple[float, str], or unyt_array or size 1. + depth: float, tuple[float, str], or unyt_array of size 1. specify the depth of the projection region (size along the line of sight). If no units are given (unyt_array or second tuple element), code units are assumed. From 726bdf3fecb4369bd3335ebad78a04499c389400 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Tue, 23 Jul 2024 16:22:45 -0500 Subject: [PATCH 136/186] pass depth kwarg to off_axis_projection in FITSOffAxisProjection --- yt/visualization/fits_image.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yt/visualization/fits_image.py b/yt/visualization/fits_image.py index 85a098fa30..c740d91287 100644 --- a/yt/visualization/fits_image.py +++ b/yt/visualization/fits_image.py @@ -1648,6 +1648,7 @@ def __init__( north_vector=north_vector, method=method, weight=weight_field, + depth=depth, ).swapaxes(0, 1) if moment == 2: @@ -1675,6 +1676,7 @@ def _sq_field(field, data, item: FieldKey): north_vector=north_vector, method=method, weight=weight_field, + depth=depth, ).swapaxes(0, 1) buf[key] = compute_stddev_image(buff2, buf[key]) From 8ea95e30052b50948e0ee6f107cbb4719813116b Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 24 Jul 2024 11:13:06 -0500 Subject: [PATCH 137/186] now passing dataset kernel name to SPH gridding backend --- yt/data_objects/construction_data_containers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/yt/data_objects/construction_data_containers.py b/yt/data_objects/construction_data_containers.py index b6dfd4dd8b..5f0c23ad58 100644 --- a/yt/data_objects/construction_data_containers.py +++ b/yt/data_objects/construction_data_containers.py @@ -997,6 +997,7 @@ def _fill_sph_particles(self, fields): smoothing_style = getattr(self.ds, "sph_smoothing_style", "scatter") normalize = getattr(self.ds, "use_sph_normalization", True) + kernel_name = getattr(self.ds, "kernel_name", "cubic") bounds, size = self._get_grid_bounds_size() @@ -1038,6 +1039,7 @@ def _fill_sph_particles(self, fields): pbar=pbar, check_period=is_periodic, period=period, + kernel_name=kernel_name, ) if normalize: pixelize_sph_kernel_arbitrary_grid( @@ -1053,6 +1055,7 @@ def _fill_sph_particles(self, fields): pbar=pbar, check_period=is_periodic, period=period, + kernel_name=kernel_name, ) if normalize: From 22cac2ba239df9a4e7fda9f2b1ca120ff709d218 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 24 Jul 2024 11:13:37 -0500 Subject: [PATCH 138/186] remove hsml margin in line of sight direction for SPH projections --- yt/utilities/lib/pixelization_routines.pyx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/yt/utilities/lib/pixelization_routines.pyx b/yt/utilities/lib/pixelization_routines.pyx index b1f7e620d5..a207473e9e 100644 --- a/yt/utilities/lib/pixelization_routines.pyx +++ b/yt/utilities/lib/pixelization_routines.pyx @@ -1252,7 +1252,11 @@ def pixelize_sph_kernel_projection( # discard if z is outside bounds if ziter[kk] == 999: continue pz = posz[j] + ziterv[kk] - if (pz + hsml[j] < z_min) or (pz - hsml[j] > z_max): continue + ## removed hsml 'margin' in the projection direction to avoid + ## double-counting particles near periodic edges + ## and adding extra 'depth' to projections + #if (pz + hsml[j] < z_min) or (pz - hsml[j] > z_max): continue + if (pz < z_min) or (pz > z_max): continue for ii in range(2): if xiter[ii] == 999: continue From 5362365917849024b18248d84c040b59cfec85c6 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 24 Jul 2024 12:00:42 -0500 Subject: [PATCH 139/186] add prefixes marking old and new test images --- yt/utilities/answer_testing/framework.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt/utilities/answer_testing/framework.py b/yt/utilities/answer_testing/framework.py index 37358573dd..ad2db01c4f 100644 --- a/yt/utilities/answer_testing/framework.py +++ b/yt/utilities/answer_testing/framework.py @@ -776,9 +776,9 @@ def compare(self, new_result, old_result): def dump_images(new_result, old_result, decimals=10): - tmpfd, old_image = tempfile.mkstemp(suffix=".png") + tmpfd, old_image = tempfile.mkstemp(prefix="baseline_", suffix=".png") os.close(tmpfd) - tmpfd, new_image = tempfile.mkstemp(suffix=".png") + tmpfd, new_image = tempfile.mkstemp(prefix="thisPR_", suffix=".png") os.close(tmpfd) image_writer.write_projection(new_result, new_image) image_writer.write_projection(old_result, old_image) From ec2c7d1c579e3d77287cb37782fd3d68837b9a73 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 24 Jul 2024 10:17:31 -0500 Subject: [PATCH 140/186] docs: added link to older docs for nose --- doc/source/developing/testing.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/source/developing/testing.rst b/doc/source/developing/testing.rst index ea571a66d6..1005a78285 100644 --- a/doc/source/developing/testing.rst +++ b/doc/source/developing/testing.rst @@ -155,7 +155,12 @@ Answer Testing .. note:: This section documents answer tests run with ``pytest``. The plan is to switch to using ``pytest`` for answer tests at some point in the future, - but currently (July 2024), answer tests are still implemented and run with ``nose``. + but currently (July 2024), answer tests are still implemented and run with + ``nose``. We generally encourage developers to use ``pytest`` for any new + tests, but if you need to change or update one of the older ``nose`` + tests, or are, e.g., writing a new frontend, + an `older version of this documentation `_ + decribes how the ``nose`` tests work. What Do Answer Tests Do ^^^^^^^^^^^^^^^^^^^^^^^ From 0580c692ad36035a97fa3561cd34f891ad02b7c3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 24 Jul 2024 18:49:07 +0000 Subject: [PATCH 141/186] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- yt/testing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/yt/testing.py b/yt/testing.py index 869f695958..a5e6f10d83 100644 --- a/yt/testing.py +++ b/yt/testing.py @@ -937,6 +937,7 @@ def fake_sph_flexible_grid_ds( ds.kernel_name = "cubic" return ds + def fake_random_sph_ds( npart: int, bbox: np.ndarray, From 6aa7a6ca36a9435e30c6a90a38cd1577ccf4ae81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Thu, 25 Jul 2024 16:13:36 +0200 Subject: [PATCH 142/186] DEP: exclude numpy 2.0.1 on macOS arm64 --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index f136d7b0f9..0a7c07c8a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,8 @@ dependencies = [ "matplotlib>=3.5", "more-itertools>=8.4", "numpy>=1.19.3, <3", # keep minimal requirement in sync with NPY_TARGET_VERSION + # https://github.com/numpy/numpy/issues/27037 + "numpy!=2.0.1 ; platform_machine=='arm64' and platform_system=='Darwin'", "packaging>=20.9", "pillow>=8.0.0", "tomli-w>=0.4.0", From e5f93d6e9d3caa11ef79b7da63dab48653e7dcb4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 24 Jul 2024 17:12:24 +0000 Subject: [PATCH 143/186] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/source/developing/testing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/developing/testing.rst b/doc/source/developing/testing.rst index 1005a78285..96ddfad842 100644 --- a/doc/source/developing/testing.rst +++ b/doc/source/developing/testing.rst @@ -159,7 +159,7 @@ Answer Testing ``nose``. We generally encourage developers to use ``pytest`` for any new tests, but if you need to change or update one of the older ``nose`` tests, or are, e.g., writing a new frontend, - an `older version of this documentation `_ + an `older version of this documentation `_ decribes how the ``nose`` tests work. What Do Answer Tests Do From 05e745e08505f2f69a37139c89a57e12a75a2f49 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Aug 2024 05:28:17 +0000 Subject: [PATCH 144/186] Bump pypa/cibuildwheel in /.github/workflows in the actions group Bumps the actions group in /.github/workflows with 1 update: [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel). Updates `pypa/cibuildwheel` from 2.19.1 to 2.19.2 - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.19.1...v2.19.2) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/wheels.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index 7bc049aaeb..62242d73d8 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -32,7 +32,7 @@ jobs: uses: actions/checkout@v4 - name: Build wheels for CPython - uses: pypa/cibuildwheel@v2.19.1 + uses: pypa/cibuildwheel@v2.19.2 with: output-dir: dist From 724583ce2673e711782dad9e35c7bf69e6c42467 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Aug 2024 17:22:15 +0000 Subject: [PATCH 145/186] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.5.0 → v0.5.6](https://github.com/astral-sh/ruff-pre-commit/compare/v0.5.0...v0.5.6) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 37b92c3559..dada61e6f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,7 +36,7 @@ repos: additional_dependencies: [black==24.3.0] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.0 + rev: v0.5.6 hooks: - id: ruff-format types_or: [ python, pyi, jupyter ] From 5c0410e99e8ca746a389a74637c67aea80093257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Mon, 5 Aug 2024 21:26:50 +0200 Subject: [PATCH 146/186] RFC: manual fix for E721 --- yt/frontends/swift/tests/test_outputs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/frontends/swift/tests/test_outputs.py b/yt/frontends/swift/tests/test_outputs.py index 1aa61284dc..d6dccb41e2 100644 --- a/yt/frontends/swift/tests/test_outputs.py +++ b/yt/frontends/swift/tests/test_outputs.py @@ -64,7 +64,7 @@ def test_non_cosmo_dataset_selection(): @requires_file(EAGLE_6) def test_cosmo_dataset(): ds = load(EAGLE_6) - assert type(ds) == SwiftDataset + assert type(ds) is SwiftDataset field = ("gas", "density") ad = ds.all_data() From 0a3cefcb4ef1ba3e042dd3c47108f4c9660cecb7 Mon Sep 17 00:00:00 2001 From: Chris Havlin Date: Thu, 15 Aug 2024 10:43:11 -0500 Subject: [PATCH 147/186] bug: tipsy, reading just smoothing length --- yt/frontends/tipsy/io.py | 5 ++++- yt/frontends/tipsy/tests/test_outputs.py | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/yt/frontends/tipsy/io.py b/yt/frontends/tipsy/io.py index 29809570aa..044932fa58 100644 --- a/yt/frontends/tipsy/io.py +++ b/yt/frontends/tipsy/io.py @@ -58,7 +58,10 @@ def _fill_fields(self, fields, vals, hsml, mask, data_file): if mask is None: size = 0 elif isinstance(mask, slice): - size = vals[fields[0]].size + if fields[0] == "smoothing_length": + size = hsml.size + else: + size = vals[fields[0]].size else: size = mask.sum() rv = {} diff --git a/yt/frontends/tipsy/tests/test_outputs.py b/yt/frontends/tipsy/tests/test_outputs.py index 7b7667b21f..73e8d6201b 100644 --- a/yt/frontends/tipsy/tests/test_outputs.py +++ b/yt/frontends/tipsy/tests/test_outputs.py @@ -112,3 +112,9 @@ def test_tipsy_index(): ds = data_dir_load(tipsy_gal) sl = ds.slice("z", 0.0) assert sl["gas", "density"].shape[0] != 0 + + +@requires_file(tipsy_gal) +def test_tipsy_smoothing_length(): + ds = data_dir_load(tipsy_gal) + _ = ds.all_data()["Gas", "smoothing_length"] From 50771dbe76083b1be3632c6ed3c5c257474ad968 Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Fri, 16 Aug 2024 09:29:01 -0500 Subject: [PATCH 148/186] explicitly disallow imageio 2.35.0 in testing deps --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 5609e22afe..1557f6fe79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -230,6 +230,7 @@ test = [ "nose~=1.3.7; python_version < '3.10'", "nose-exclude; python_version < '3.10'", "nose-timer~=1.0.0; python_version < '3.10'", + "imageio!=2.35.0", # see https://github.com/yt-project/yt/issues/4966 ] typecheck = [ "mypy==1.8.0", From 2deec34a9958f4e20491a9bfae13f2bfb4a62668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Thu, 15 Aug 2024 19:29:08 +0200 Subject: [PATCH 149/186] MNT: bump ruff to v0.6 --- .pre-commit-config.yaml | 4 +--- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dada61e6f3..d42ab2e32b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,12 +36,10 @@ repos: additional_dependencies: [black==24.3.0] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.6 + rev: v0.6.1 hooks: - id: ruff-format - types_or: [ python, pyi, jupyter ] - id: ruff - types_or: [ python, pyi, jupyter ] args: [ --fix, --show-fixes, diff --git a/pyproject.toml b/pyproject.toml index 1557f6fe79..f0f13146b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -272,7 +272,6 @@ exclude = ''' ''' [tool.ruff] -extend-include = ["*.ipynb"] exclude = [ "doc", "benchmarks", From d000cd532adc9ae14341e14500d0d7221c20713f Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Mon, 19 Aug 2024 15:16:32 -0500 Subject: [PATCH 150/186] increase numbers for frontend tests affect by SPH backend change --- tests/tests.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/tests.yaml b/tests/tests.yaml index d7de9b9650..4f3dc77c95 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -37,7 +37,7 @@ answer_tests: - yt/frontends/amrvac/tests/test_outputs.py:test_riemann_cartesian_175D - yt/frontends/amrvac/tests/test_outputs.py:test_rmi_cartesian_dust_2D - local_arepo_011: # PR 4419 + local_arepo_012: # PR 4419; to 012: PR 4939 - yt/frontends/arepo/tests/test_outputs.py:test_arepo_bullet - yt/frontends/arepo/tests/test_outputs.py:test_arepo_tng59 - yt/frontends/arepo/tests/test_outputs.py:test_arepo_cr @@ -92,7 +92,7 @@ answer_tests: - yt/frontends/flash/tests/test_outputs.py:test_wind_tunnel - yt/frontends/flash/tests/test_outputs.py:test_fid_1to3_b1 - local_gadget_009: # PR 3258 + local_gadget_010: # PR 3258; to 010: PR 4939 - yt/frontends/gadget/tests/test_outputs.py:test_iso_collapse - yt/frontends/gadget/tests/test_outputs.py:test_pid_uniqueness - yt/frontends/gadget/tests/test_outputs.py:test_bigendian_field_access @@ -108,7 +108,7 @@ answer_tests: local_gdf_002: - yt/frontends/gdf/tests/test_outputs_nose.py:test_sedov_tunnel - local_gizmo_008: # PR 2909 + local_gizmo_008: # PR 2909; to 009: PR 4939 - yt/frontends/gizmo/tests/test_outputs.py:test_gizmo_64 local_halos_012: # PR 3325 @@ -118,7 +118,7 @@ answer_tests: - yt/frontends/gadget_fof/tests/test_outputs.py:test_fields_g5 - yt/frontends/gadget_fof/tests/test_outputs.py:test_fields_g42 - local_owls_008: # PR 2909 + local_owls_008: # PR 2909; to 009: PR 4939 - yt/frontends/owls/tests/test_outputs.py:test_snapshot_033 - yt/frontends/owls/tests/test_outputs.py:test_OWLS_particlefilter @@ -130,7 +130,7 @@ answer_tests: - yt/visualization/tests/test_particle_plot.py:test_particle_phase_answers - yt/visualization/tests/test_callbacks.py:test_axis_manipulations - local_tipsy_009: # PR 2909 + local_tipsy_010: # PR 2909; to 010: PR 4939 - yt/frontends/tipsy/tests/test_outputs.py:test_pkdgrav - yt/frontends/tipsy/tests/test_outputs.py:test_gasoline_dmonly - yt/frontends/tipsy/tests/test_outputs.py:test_tipsy_galaxy From f46649424b4dbcb3152611e106c0c2119b9b77fe Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Mon, 19 Aug 2024 15:36:55 -0500 Subject: [PATCH 151/186] comments update --- tests/tests.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/tests.yaml b/tests/tests.yaml index 4f3dc77c95..8396312e1c 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -37,7 +37,7 @@ answer_tests: - yt/frontends/amrvac/tests/test_outputs.py:test_riemann_cartesian_175D - yt/frontends/amrvac/tests/test_outputs.py:test_rmi_cartesian_dust_2D - local_arepo_012: # PR 4419; to 012: PR 4939 + local_arepo_012: # PR 4939 - yt/frontends/arepo/tests/test_outputs.py:test_arepo_bullet - yt/frontends/arepo/tests/test_outputs.py:test_arepo_tng59 - yt/frontends/arepo/tests/test_outputs.py:test_arepo_cr @@ -92,7 +92,7 @@ answer_tests: - yt/frontends/flash/tests/test_outputs.py:test_wind_tunnel - yt/frontends/flash/tests/test_outputs.py:test_fid_1to3_b1 - local_gadget_010: # PR 3258; to 010: PR 4939 + local_gadget_010: # PR 4939 - yt/frontends/gadget/tests/test_outputs.py:test_iso_collapse - yt/frontends/gadget/tests/test_outputs.py:test_pid_uniqueness - yt/frontends/gadget/tests/test_outputs.py:test_bigendian_field_access @@ -108,7 +108,7 @@ answer_tests: local_gdf_002: - yt/frontends/gdf/tests/test_outputs_nose.py:test_sedov_tunnel - local_gizmo_008: # PR 2909; to 009: PR 4939 + local_gizmo_008: # PR 4939 - yt/frontends/gizmo/tests/test_outputs.py:test_gizmo_64 local_halos_012: # PR 3325 @@ -118,7 +118,7 @@ answer_tests: - yt/frontends/gadget_fof/tests/test_outputs.py:test_fields_g5 - yt/frontends/gadget_fof/tests/test_outputs.py:test_fields_g42 - local_owls_008: # PR 2909; to 009: PR 4939 + local_owls_008: # PR 4939 - yt/frontends/owls/tests/test_outputs.py:test_snapshot_033 - yt/frontends/owls/tests/test_outputs.py:test_OWLS_particlefilter @@ -130,7 +130,7 @@ answer_tests: - yt/visualization/tests/test_particle_plot.py:test_particle_phase_answers - yt/visualization/tests/test_callbacks.py:test_axis_manipulations - local_tipsy_010: # PR 2909; to 010: PR 4939 + local_tipsy_010: # PR 4939 - yt/frontends/tipsy/tests/test_outputs.py:test_pkdgrav - yt/frontends/tipsy/tests/test_outputs.py:test_gasoline_dmonly - yt/frontends/tipsy/tests/test_outputs.py:test_tipsy_galaxy From f8949c24b3688d729dbf210ebe59bd206a46ce5b Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Mon, 19 Aug 2024 19:37:27 -0500 Subject: [PATCH 152/186] actually update gizmo, owls test numbers --- tests/tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests.yaml b/tests/tests.yaml index 8396312e1c..e6804b57c8 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -108,7 +108,7 @@ answer_tests: local_gdf_002: - yt/frontends/gdf/tests/test_outputs_nose.py:test_sedov_tunnel - local_gizmo_008: # PR 4939 + local_gizmo_009: # PR 4939 - yt/frontends/gizmo/tests/test_outputs.py:test_gizmo_64 local_halos_012: # PR 3325 @@ -118,7 +118,7 @@ answer_tests: - yt/frontends/gadget_fof/tests/test_outputs.py:test_fields_g5 - yt/frontends/gadget_fof/tests/test_outputs.py:test_fields_g42 - local_owls_008: # PR 4939 + local_owls_009: # PR 4939 - yt/frontends/owls/tests/test_outputs.py:test_snapshot_033 - yt/frontends/owls/tests/test_outputs.py:test_OWLS_particlefilter From ac30d156c15903a485f159e78c21849c3f3fcc45 Mon Sep 17 00:00:00 2001 From: nastasha-w <35771073+nastasha-w@users.noreply.github.com> Date: Wed, 21 Aug 2024 17:50:21 -0500 Subject: [PATCH 153/186] bumpy arepo test number: avoid missing files error? --- tests/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests.yaml b/tests/tests.yaml index e6804b57c8..40836b3381 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -37,7 +37,7 @@ answer_tests: - yt/frontends/amrvac/tests/test_outputs.py:test_riemann_cartesian_175D - yt/frontends/amrvac/tests/test_outputs.py:test_rmi_cartesian_dust_2D - local_arepo_012: # PR 4939 + local_arepo_013: # PR 4939 - yt/frontends/arepo/tests/test_outputs.py:test_arepo_bullet - yt/frontends/arepo/tests/test_outputs.py:test_arepo_tng59 - yt/frontends/arepo/tests/test_outputs.py:test_arepo_cr From af49fcd0c3317c774ca5035df33d436ea1fb4065 Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Tue, 27 Aug 2024 15:42:22 -0500 Subject: [PATCH 154/186] fix indexing error in fill_region_float --- yt/utilities/lib/misc_utilities.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/utilities/lib/misc_utilities.pyx b/yt/utilities/lib/misc_utilities.pyx index 88e1d7e326..2a5e61efa0 100644 --- a/yt/utilities/lib/misc_utilities.pyx +++ b/yt/utilities/lib/misc_utilities.pyx @@ -921,7 +921,7 @@ def fill_region_float(np.ndarray[np.float64_t, ndim=2] fcoords, if (sp[1] + odsp[1] < LE[1]) or (sp[1] - odsp[1] > RE[1]): continue for zi in range(2): if diter[2][zi] == 999: continue - sp[2] = osp[2] + diterv[2][yi] + sp[2] = osp[2] + diterv[2][zi] if (sp[2] + odsp[2] < LE[2]) or (sp[2] - odsp[2] > RE[2]): continue for i in range(3): ld[i] = fmax(((sp[i]-odsp[i]-LE[i])*box_idds[i]),0) From 9385a01d10f4a150a3a93f87de9cb361192cc140 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Sep 2024 05:15:46 +0000 Subject: [PATCH 155/186] Bump the actions group in /.github/workflows with 2 updates Bumps the actions group in /.github/workflows with 2 updates: [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) and [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `pypa/cibuildwheel` from 2.19.2 to 2.20.0 - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.19.2...v2.20.0) Updates `pypa/gh-action-pypi-publish` from 1.9.0 to 1.10.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.9.0...v1.10.0) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/wheels.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index 62242d73d8..a0686440ed 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -32,7 +32,7 @@ jobs: uses: actions/checkout@v4 - name: Build wheels for CPython - uses: pypa/cibuildwheel@v2.19.2 + uses: pypa/cibuildwheel@v2.20.0 with: output-dir: dist @@ -124,7 +124,7 @@ jobs: merge-multiple: true - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.9.0 + uses: pypa/gh-action-pypi-publish@v1.10.0 with: user: __token__ password: ${{ secrets.pypi_token }} From 124e9ebfa644fd4336904d2ec92605213f659967 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 17:11:32 +0000 Subject: [PATCH 156/186] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.6.1 → v0.6.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.6.1...v0.6.3) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d42ab2e32b..1b39e022ad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,7 +36,7 @@ repos: additional_dependencies: [black==24.3.0] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.1 + rev: v0.6.3 hooks: - id: ruff-format - id: ruff From 2a1a1888077da66012b9604ab31b8fda1aa8786f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Mon, 8 Jul 2024 11:18:20 +0200 Subject: [PATCH 157/186] DEP: make ipywidgets/IPython optional dependencies again --- pyproject.toml | 5 +- yt/_maintenance/ipython_compat.py | 31 ++++++++ yt/fields/field_type_container.py | 119 ++++++++++++++++-------------- 3 files changed, 98 insertions(+), 57 deletions(-) create mode 100644 yt/_maintenance/ipython_compat.py diff --git a/pyproject.toml b/pyproject.toml index f0f13146b5..86bdc5de23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,6 @@ requires-python = ">=3.9.2" dependencies = [ "cmyt>=1.1.2", "ewah-bool-utils>=1.2.0", - "ipywidgets>=8.0.0", "matplotlib>=3.5", "more-itertools>=8.4", "numpy>=1.19.3, <3", # keep minimal requirement in sync with NPY_TARGET_VERSION @@ -136,7 +135,8 @@ full = [ "cartopy>=0.22.0", "firefly>=3.2.0", "glueviz>=0.13.3", - "ipython>=2.0.0", + "ipython>=7.16.2", + "ipywidgets>=8.0.0", "miniballcpp>=0.2.1", "mpi4py>=3.0.3", "pandas>=1.1.2", @@ -210,7 +210,6 @@ mapserver = [ minimal = [ "cmyt==1.1.2", "ewah-bool-utils==1.2.0", - "ipywidgets==8.0.0", "matplotlib==3.5", "more-itertools==8.4", "numpy==1.19.3", diff --git a/yt/_maintenance/ipython_compat.py b/yt/_maintenance/ipython_compat.py new file mode 100644 index 0000000000..e405f44b3b --- /dev/null +++ b/yt/_maintenance/ipython_compat.py @@ -0,0 +1,31 @@ +from importlib.metadata import version +from importlib.util import find_spec + +from packaging.version import Version + +__all__ = [ + "IS_IPYTHON", + "IPYWIDGETS_ENABLED", +] + +IS_IPYTHON: bool +HAS_IPYWIDGETS_GE_8: bool +IPYWIDGETS_ENABLED: bool + + +try: + # this name is only defined if running within ipython/jupyter + __IPYTHON__ # type: ignore [name-defined] # noqa: B018 +except NameError: + IS_IPYTHON = False +else: + IS_IPYTHON = True + + +HAS_IPYWIDGETS_GE_8 = ( + Version(version("ipywidgets")) >= Version("8.0.0") + if find_spec("ipywidgets") is not None + else False +) + +IPYWIDGETS_ENABLED = IS_IPYTHON and HAS_IPYWIDGETS_GE_8 diff --git a/yt/fields/field_type_container.py b/yt/fields/field_type_container.py index 9a21be7222..ce6103aba1 100644 --- a/yt/fields/field_type_container.py +++ b/yt/fields/field_type_container.py @@ -7,6 +7,7 @@ import weakref from functools import cached_property +from yt._maintenance.ipython_compat import IPYWIDGETS_ENABLED from yt.fields.derived_field import DerivedField @@ -59,22 +60,24 @@ def __contains__(self, obj): return ob in self.field_types - def _ipython_display_(self): - import ipywidgets - from IPython.display import display + if IPYWIDGETS_ENABLED: - fnames = [] - children = [] - for ftype in sorted(self.field_types): - fnc = getattr(self, ftype) - children.append(ipywidgets.Output()) - with children[-1]: - display(fnc) - fnames.append(ftype) - tabs = ipywidgets.Tab(children=children) - for i, n in enumerate(fnames): - tabs.set_title(i, n) - display(tabs) + def _ipython_display_(self): + import ipywidgets + from IPython.display import display + + fnames = [] + children = [] + for ftype in sorted(self.field_types): + fnc = getattr(self, ftype) + children.append(ipywidgets.Output()) + with children[-1]: + display(fnc) + fnames.append(ftype) + tabs = ipywidgets.Tab(children=children) + for i, n in enumerate(fnames): + tabs.set_title(i, n) + display(tabs) class FieldNameContainer: @@ -112,46 +115,54 @@ def __contains__(self, obj): return True return False - def _ipython_display_(self): - import ipywidgets - from IPython.display import Markdown, display - - names = dir(self) - names.sort() - - def change_field(_ftype, _box, _var_window): - def _change_field(event): - fobj = getattr(_ftype, event["new"]) - _box.clear_output() - with _box: - display( - Markdown( - data="```python\n" - + textwrap.dedent(fobj.get_source()) - + "\n```" + if IPYWIDGETS_ENABLED: + + def _ipython_display_(self): + import ipywidgets + from IPython.display import Markdown, display + + names = dir(self) + names.sort() + + def change_field(_ftype, _box, _var_window): + def _change_field(event): + fobj = getattr(_ftype, event["new"]) + _box.clear_output() + with _box: + display( + Markdown( + data="```python\n" + + textwrap.dedent(fobj.get_source()) + + "\n```" + ) ) - ) - values = inspect.getclosurevars(fobj._function).nonlocals - _var_window.value = _fill_values(values) + values = inspect.getclosurevars(fobj._function).nonlocals + _var_window.value = _fill_values(values) - return _change_field + return _change_field - flist = ipywidgets.Select(options=names, layout=ipywidgets.Layout(height="95%")) - source = ipywidgets.Output(layout=ipywidgets.Layout(width="100%", height="9em")) - var_window = ipywidgets.HTML(value="Empty") - var_box = ipywidgets.Box( - layout=ipywidgets.Layout(width="100%", height="100%", overflow_y="scroll") - ) - var_box.children = [var_window] - ftype_tabs = ipywidgets.Tab( - children=[source, var_box], - layout=ipywidgets.Layout(flex="2 1 auto", width="auto", height="95%"), - ) - ftype_tabs.set_title(0, "Source") - ftype_tabs.set_title(1, "Variables") - flist.observe(change_field(self, source, var_window), "value") - display( - ipywidgets.HBox( - [flist, ftype_tabs], layout=ipywidgets.Layout(height="14em") + flist = ipywidgets.Select( + options=names, layout=ipywidgets.Layout(height="95%") + ) + source = ipywidgets.Output( + layout=ipywidgets.Layout(width="100%", height="9em") + ) + var_window = ipywidgets.HTML(value="Empty") + var_box = ipywidgets.Box( + layout=ipywidgets.Layout( + width="100%", height="100%", overflow_y="scroll" + ) + ) + var_box.children = [var_window] + ftype_tabs = ipywidgets.Tab( + children=[source, var_box], + layout=ipywidgets.Layout(flex="2 1 auto", width="auto", height="95%"), + ) + ftype_tabs.set_title(0, "Source") + ftype_tabs.set_title(1, "Variables") + flist.observe(change_field(self, source, var_window), "value") + display( + ipywidgets.HBox( + [flist, ftype_tabs], layout=ipywidgets.Layout(height="14em") + ) ) - ) From 4c929b2440acc9fe8dfc69e43e77bb30e3d14984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Mon, 8 Jul 2024 11:34:23 +0200 Subject: [PATCH 158/186] RFC: uniformize IPython+ipywidgets runtime detection --- yt/data_objects/static_output.py | 6 ++---- yt/funcs.py | 4 ++-- yt/visualization/image_writer.py | 5 ++--- yt/visualization/plot_container.py | 4 ++-- yt/visualization/profile_plotter.py | 4 ++-- yt/visualization/volume_rendering/old_camera.py | 4 ++-- yt/visualization/volume_rendering/scene.py | 4 ++-- 7 files changed, 14 insertions(+), 17 deletions(-) diff --git a/yt/data_objects/static_output.py b/yt/data_objects/static_output.py index 57f6507dcd..dd5c452474 100644 --- a/yt/data_objects/static_output.py +++ b/yt/data_objects/static_output.py @@ -23,6 +23,7 @@ from unyt.exceptions import UnitConversionError, UnitParseError from yt._maintenance.deprecation import issue_deprecation_warning +from yt._maintenance.ipython_compat import IPYWIDGETS_ENABLED from yt._typing import ( AnyFieldKey, AxisOrder, @@ -119,10 +120,7 @@ def __init__(self, display_array=False): # We can assume that ipywidgets will not be *added* to the system # during the course of execution, and if it is, we will not wrap the # array. - if display_array and find_spec("ipywidgets") is not None: - self.display_array = True - else: - self.display_array = False + self.display_array = display_array and IPYWIDGETS_ENABLED def __get__(self, instance, owner): return self.data.get(instance, None) diff --git a/yt/funcs.py b/yt/funcs.py index 7209b8c72a..fa0b89abaa 100644 --- a/yt/funcs.py +++ b/yt/funcs.py @@ -1,5 +1,4 @@ import base64 -import builtins import contextlib import copy import errno @@ -24,6 +23,7 @@ from more_itertools import always_iterable, collapse, first from yt._maintenance.deprecation import issue_deprecation_warning +from yt._maintenance.ipython_compat import IS_IPYTHON from yt.config import ytcfg from yt.units import YTArray, YTQuantity from yt.utilities.exceptions import YTFieldNotFound, YTInvalidWidthError @@ -1023,7 +1023,7 @@ def toggle_interactivity(): global interactivity interactivity = not interactivity if interactivity: - if "__IPYTHON__" in dir(builtins): + if IS_IPYTHON: import IPython shell = IPython.get_ipython() diff --git a/yt/visualization/image_writer.py b/yt/visualization/image_writer.py index 87bd9189e9..1dcd7490fe 100644 --- a/yt/visualization/image_writer.py +++ b/yt/visualization/image_writer.py @@ -1,7 +1,6 @@ -import builtins - import numpy as np +from yt._maintenance.ipython_compat import IS_IPYTHON from yt.config import ytcfg from yt.funcs import mylog from yt.units.yt_array import YTQuantity @@ -418,7 +417,7 @@ def display_in_notebook(image, max_val=None): three channels. """ - if "__IPYTHON__" in dir(builtins): + if IS_IPYTHON: from IPython.core.displaypub import publish_display_data data = write_bitmap(image, None, max_val=max_val) diff --git a/yt/visualization/plot_container.py b/yt/visualization/plot_container.py index 9cdb39df11..c8327a2b55 100644 --- a/yt/visualization/plot_container.py +++ b/yt/visualization/plot_container.py @@ -1,6 +1,5 @@ import abc import base64 -import builtins import os import warnings from collections import defaultdict @@ -12,6 +11,7 @@ from unyt.dimensions import length from yt._maintenance.deprecation import issue_deprecation_warning +from yt._maintenance.ipython_compat import IS_IPYTHON from yt._typing import FieldKey, Quantity from yt.config import ytcfg from yt.data_objects.time_series import DatasetSeries @@ -634,7 +634,7 @@ def show(self): for v in sorted(self.plots.values()): v.show() else: - if "__IPYTHON__" in dir(builtins): + if IS_IPYTHON: from IPython.display import display display(self) diff --git a/yt/visualization/profile_plotter.py b/yt/visualization/profile_plotter.py index 004352cf90..afaecea03c 100644 --- a/yt/visualization/profile_plotter.py +++ b/yt/visualization/profile_plotter.py @@ -1,5 +1,4 @@ import base64 -import builtins import os from collections.abc import Iterable from functools import wraps @@ -9,6 +8,7 @@ import numpy as np from more_itertools.more import always_iterable, unzip +from yt._maintenance.ipython_compat import IS_IPYTHON from yt._typing import FieldKey from yt.data_objects.profiles import create_profile, sanitize_field_tuple_keys from yt.data_objects.static_output import Dataset @@ -342,7 +342,7 @@ def show(self): >>> pp.show() """ - if "__IPYTHON__" in dir(builtins): + if IS_IPYTHON: from IPython.display import display display(self) diff --git a/yt/visualization/volume_rendering/old_camera.py b/yt/visualization/volume_rendering/old_camera.py index 6a7913e370..7e7117f720 100644 --- a/yt/visualization/volume_rendering/old_camera.py +++ b/yt/visualization/volume_rendering/old_camera.py @@ -1,9 +1,9 @@ -import builtins import sys from copy import deepcopy import numpy as np +from yt._maintenance.ipython_compat import IS_IPYTHON from yt.config import ytcfg from yt.data_objects.api import ImageArray from yt.funcs import ensure_numpy_array, get_num_threads, get_pbar, is_sequence, mylog @@ -900,7 +900,7 @@ def show(self, clip_ratio=None): >>> cam.show() """ - if "__IPYTHON__" in dir(builtins): + if IS_IPYTHON: from IPython.core.displaypub import publish_display_data image = self.snapshot()[:, :, :3] diff --git a/yt/visualization/volume_rendering/scene.py b/yt/visualization/volume_rendering/scene.py index ca8207cfbe..ae30153a42 100644 --- a/yt/visualization/volume_rendering/scene.py +++ b/yt/visualization/volume_rendering/scene.py @@ -1,10 +1,10 @@ -import builtins import functools from collections import OrderedDict from typing import Optional, Union import numpy as np +from yt._maintenance.ipython_compat import IS_IPYTHON from yt.config import ytcfg from yt.funcs import mylog from yt.units.dimensions import length # type: ignore @@ -909,7 +909,7 @@ def show(self, sigma_clip=None): >>> sc.show() """ - if "__IPYTHON__" in dir(builtins): + if IS_IPYTHON: from IPython.display import display self._sigma_clip = sigma_clip From c508a452465dc06958f46f95454ef32bd4a2797b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Fri, 12 Jul 2024 16:47:29 +0200 Subject: [PATCH 159/186] link discussion in a comment Co-authored-by: Chris Havlin --- yt/fields/field_type_container.py | 1 + 1 file changed, 1 insertion(+) diff --git a/yt/fields/field_type_container.py b/yt/fields/field_type_container.py index ce6103aba1..3353138d31 100644 --- a/yt/fields/field_type_container.py +++ b/yt/fields/field_type_container.py @@ -116,6 +116,7 @@ def __contains__(self, obj): return False if IPYWIDGETS_ENABLED: + # for discussion of this class-level conditional: https://github.com/yt-project/yt/pull/4941 def _ipython_display_(self): import ipywidgets From 0bfea2f20c09b60e3c0bcdb951990392f90a3338 Mon Sep 17 00:00:00 2001 From: chavlin Date: Wed, 4 Sep 2024 14:38:46 -0500 Subject: [PATCH 160/186] adjust platform_dep.h for recent MSVS versions --- yt/utilities/lib/platform_dep.h | 54 ++------------------------------- 1 file changed, 2 insertions(+), 52 deletions(-) diff --git a/yt/utilities/lib/platform_dep.h b/yt/utilities/lib/platform_dep.h index a0227d60be..e1d3020b90 100644 --- a/yt/utilities/lib/platform_dep.h +++ b/yt/utilities/lib/platform_dep.h @@ -1,59 +1,9 @@ #include #ifdef MS_WIN32 #include "malloc.h" +// note: the following implicitly requires _MSC_VER >= 1928 (VS 2019, 16.8) #include -typedef int int32_t; -typedef long long int64_t; -/* Taken from http://siliconandlithium.blogspot.com/2014/05/msvc-c99-mathh-header.html */ -#define isnormal(x) ((_fpclass(x) == _FPCLASS_NN) || (_fpclass(x) == _FPCLASS_PN)) -static __inline double rint(double x){ - const double two_to_52 = 4.5035996273704960e+15; - double fa = fabs(x); - if(fa >= two_to_52){ - return x; - } else{ - return copysign(two_to_52 + fa - two_to_52, x); - } -} -#if _MSC_VER < 1928 -static __inline long int lrint(double x){ - return (long)rint(x); -} -#endif -static __inline double fmax(double x, double y){ - return (x > y) ? x : y; -} -static __inline double fmin(double x, double y){ - return (x < y) ? x : y; -} - -/* adapted from http://www.johndcook.com/blog/cpp_erf/ - code is under public domain license */ - -double erf(double x) -{ - /* constants */ - double a1 = 0.254829592; - double a2 = -0.284496736; - double a3 = 1.421413741; - double a4 = -1.453152027; - double a5 = 1.061405429; - double p = 0.3275911; - double t; - double y; - - /* Save the sign of x */ - int sign = 1; - if (x < 0) - sign = -1; - x = fabs(x); - - /* A&S formula 7.1.26 */ - t = 1.0/(1.0 + p*x); - y = 1.0 - (((((a5*t + a4)*t) + a3)*t + a2)*t + a1)*t*exp(-x*x); - - return sign*y; -} +#include #elif defined(__FreeBSD__) #include #include From 4c7541ac37afe9ab596f4ba0c3777628c28a2ba8 Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Thu, 5 Sep 2024 10:04:11 -0500 Subject: [PATCH 161/186] window wheels build with windows-2022 image --- .github/workflows/wheels.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index a0686440ed..cd1f65b437 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -21,7 +21,7 @@ jobs: matrix: os: [ ubuntu-20.04, - windows-2019, + windows-2022, macos-13, # x86_64 macos-14, # arm64 ] From b5932b637ffaa66946a4245bf08f002d3f1b4641 Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Thu, 5 Sep 2024 10:07:19 -0500 Subject: [PATCH 162/186] update windows platform_dep comment --- yt/utilities/lib/platform_dep.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/yt/utilities/lib/platform_dep.h b/yt/utilities/lib/platform_dep.h index e1d3020b90..dbd5736ece 100644 --- a/yt/utilities/lib/platform_dep.h +++ b/yt/utilities/lib/platform_dep.h @@ -1,7 +1,12 @@ #include #ifdef MS_WIN32 #include "malloc.h" -// note: the following implicitly requires _MSC_VER >= 1928 (VS 2019, 16.8) +/* +note: the following implicitly sets a mininum VS version: conservative +minumum is _MSC_VER >= 1928 (VS 2019, 16.8), but may work for VS 2015 +but that has not been tested. see https://github.com/yt-project/yt/pull/4980 +and https://learn.microsoft.com/en-us/cpp/overview/visual-cpp-language-conformance +*/ #include #include #elif defined(__FreeBSD__) From 851d57e418c1288569d21e1e374877057421cfcd Mon Sep 17 00:00:00 2001 From: Chris Havlin Date: Thu, 5 Sep 2024 11:08:27 -0500 Subject: [PATCH 163/186] fix typo in platform_dep.h comment --- yt/utilities/lib/platform_dep.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/utilities/lib/platform_dep.h b/yt/utilities/lib/platform_dep.h index dbd5736ece..d2157040d5 100644 --- a/yt/utilities/lib/platform_dep.h +++ b/yt/utilities/lib/platform_dep.h @@ -3,7 +3,7 @@ #include "malloc.h" /* note: the following implicitly sets a mininum VS version: conservative -minumum is _MSC_VER >= 1928 (VS 2019, 16.8), but may work for VS 2015 +minimum is _MSC_VER >= 1928 (VS 2019, 16.8), but may work for VS 2015 but that has not been tested. see https://github.com/yt-project/yt/pull/4980 and https://learn.microsoft.com/en-us/cpp/overview/visual-cpp-language-conformance */ From a29a6d5f504d99548404afb0484f4c0096242979 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sun, 1 Sep 2024 08:25:48 +0200 Subject: [PATCH 164/186] WHL: enable cp313 wheels --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 86bdc5de23..f7dece5e16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Scientific/Engineering :: Astronomy", "Topic :: Scientific/Engineering :: Physics", "Topic :: Scientific/Engineering :: Visualization", @@ -496,7 +497,7 @@ show_error_context = true exclude = "(test_*|lodgeit)" [tool.cibuildwheel] -build = "cp39-* cp310-* cp311-* cp312-*" +build = "cp39-* cp310-* cp311-* cp312-* cp313-*" build-verbosity = 1 test-skip = "*-musllinux*" test-extras = "test" From 249da76c715d8e600d004ec820c2954a643d5c60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sun, 1 Sep 2024 08:46:08 +0200 Subject: [PATCH 165/186] add whitespace to trigger wheels.yaml --- .github/workflows/wheels.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index cd1f65b437..5fafb3add8 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -13,6 +13,7 @@ on: - MANIFEST.in workflow_dispatch: + jobs: build_wheels: name: Build wheels on ${{ matrix.os }} From 7c03a6d9fa9b7656bee17ec9f0f4f61c2dc2d57c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Mon, 2 Sep 2024 14:12:48 +0200 Subject: [PATCH 166/186] RFC: enable flake8-type-checking ruleset --- pyproject.toml | 1 + yt/data_objects/static_output.py | 6 ++++-- yt/frontends/ramses/io.py | 6 ++++-- yt/visualization/fixed_resolution.py | 2 +- yt/visualization/plot_window.py | 6 ++++-- yt/visualization/profile_plotter.py | 8 ++++++-- 6 files changed, 20 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 86bdc5de23..b528353eae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -294,6 +294,7 @@ select = [ "C4", # flake8-comprehensions "B", # flake8-bugbear "G", # flake8-logging-format + "TCH", # flake8-type-checking "YTT", # flake8-2020 "UP", # pyupgrade "I", # isort diff --git a/yt/data_objects/static_output.py b/yt/data_objects/static_output.py index dd5c452474..80a7c5981d 100644 --- a/yt/data_objects/static_output.py +++ b/yt/data_objects/static_output.py @@ -13,12 +13,11 @@ from functools import cached_property from importlib.util import find_spec from stat import ST_CTIME -from typing import Any, Literal, Optional, Union +from typing import TYPE_CHECKING, Any, Literal, Optional, Union import numpy as np import unyt as un from more_itertools import unzip -from sympy import Symbol from unyt import Unit, UnitSystem, unyt_quantity from unyt.exceptions import UnitConversionError, UnitParseError @@ -75,6 +74,9 @@ from yt.utilities.parallel_tools.parallel_analysis_interface import parallel_root_only from yt.utilities.parameter_file_storage import NoParameterShelf, ParameterFileStore +if TYPE_CHECKING: + from sympy import Symbol + if sys.version_info >= (3, 11): from typing import assert_never else: diff --git a/yt/frontends/ramses/io.py b/yt/frontends/ramses/io.py index f39dd84b22..546ba333f5 100644 --- a/yt/frontends/ramses/io.py +++ b/yt/frontends/ramses/io.py @@ -1,7 +1,6 @@ -import os from collections import defaultdict from functools import lru_cache -from typing import Union +from typing import TYPE_CHECKING, Union import numpy as np @@ -17,6 +16,9 @@ from yt.utilities.logger import ytLogger as mylog from yt.utilities.physical_ratios import cm_per_km, cm_per_mpc +if TYPE_CHECKING: + import os + def convert_ramses_ages(ds, conformal_ages): issue_deprecation_warning( diff --git a/yt/visualization/fixed_resolution.py b/yt/visualization/fixed_resolution.py index d2da6a29e1..8029f7d8ab 100644 --- a/yt/visualization/fixed_resolution.py +++ b/yt/visualization/fixed_resolution.py @@ -134,7 +134,7 @@ def __init__( # the filter methods for the present class are defined only when # fixed_resolution_filters is imported, so we need to guarantee # that it happens no later than instantiation - from yt.visualization.fixed_resolution_filters import ( + from yt.visualization.fixed_resolution_filters import ( # noqa FixedResolutionBufferFilter, ) diff --git a/yt/visualization/plot_window.py b/yt/visualization/plot_window.py index c67a3c6125..fa9220320e 100644 --- a/yt/visualization/plot_window.py +++ b/yt/visualization/plot_window.py @@ -2,7 +2,7 @@ import sys from collections import defaultdict from numbers import Number -from typing import Optional, Union +from typing import TYPE_CHECKING, Optional, Union import matplotlib import numpy as np @@ -55,6 +55,9 @@ invalidate_plot, ) +if TYPE_CHECKING: + from yt.visualization.plot_modifications import PlotCallback + if sys.version_info < (3, 10): from yt._maintenance.backports import zip @@ -866,7 +869,6 @@ def __init__(self, *args, **kwargs) -> None: # the filter methods for the present class are defined only when # fixed_resolution_filters is imported, so we need to guarantee # that it happens no later than instantiation - from yt.visualization.plot_modifications import PlotCallback self._callbacks: list[PlotCallback] = [] diff --git a/yt/visualization/profile_plotter.py b/yt/visualization/profile_plotter.py index afaecea03c..6f354ee529 100644 --- a/yt/visualization/profile_plotter.py +++ b/yt/visualization/profile_plotter.py @@ -1,8 +1,7 @@ import base64 import os -from collections.abc import Iterable from functools import wraps -from typing import Any, Optional, Union +from typing import TYPE_CHECKING, Any, Optional, Union import matplotlib import numpy as np @@ -28,6 +27,11 @@ validate_plot, ) +if TYPE_CHECKING: + from collections.abc import Iterable + + from yt._typing import FieldKey + def invalidate_profile(f): @wraps(f) From 5b6b6025bac989a596c581c0ea0dba3f4f8f0b3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Tue, 3 Sep 2024 15:11:51 +0200 Subject: [PATCH 167/186] MNT: add pep723 metadata to test helper script --- tests/unpin_requirements.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unpin_requirements.py b/tests/unpin_requirements.py index 0da23061dc..932c11192e 100644 --- a/tests/unpin_requirements.py +++ b/tests/unpin_requirements.py @@ -1,3 +1,10 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "tomli ; python_full_version < '3.11'", +# "tomli-w", +# ] +# /// import re import sys From 5ad93c4109fe441f0994e2ccf340b1c6ffa8ea3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Tue, 10 Sep 2024 10:14:56 +0200 Subject: [PATCH 168/186] DEP: move dev-only dependencies to requirements files --- .github/dependabot.yml | 9 +++ .github/workflows/type-checking.yaml | 2 +- MANIFEST.in | 1 + doc/source/developing/building_the_docs.rst | 2 +- pyproject.toml | 37 +---------- requirements/docs.txt | 12 ++++ requirements/minimal_env.txt | 69 +++++++++++++++++++++ requirements/typecheck.txt | 5 ++ tests/ci_install.sh | 2 +- 9 files changed, 101 insertions(+), 38 deletions(-) create mode 100644 requirements/docs.txt create mode 100644 requirements/minimal_env.txt create mode 100644 requirements/typecheck.txt diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e9767227a1..2ed6f7807c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,5 +1,14 @@ version: 2 updates: +- package-ecosystem: pip + directory: /requirements + schedule: + interval: monthly + groups: + actions: + patterns: + - '*' + - package-ecosystem: github-actions directory: /.github/workflows schedule: diff --git a/.github/workflows/type-checking.yaml b/.github/workflows/type-checking.yaml index 6931f4a45e..3c7f7307d2 100644 --- a/.github/workflows/type-checking.yaml +++ b/.github/workflows/type-checking.yaml @@ -37,7 +37,7 @@ jobs: - name: Build run: | python3 -m pip install --upgrade pip - python3 -m pip install -e .[typecheck] + python3 -m pip install -e . -r requirements/typecheck.txt - run: python -m pip list diff --git a/MANIFEST.in b/MANIFEST.in index e1e1eb29d5..442aa58d9e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -70,5 +70,6 @@ include yt/default.mplstyle prune yt/frontends/_skeleton recursive-include yt/frontends/amrvac *.par +recursive-exclude requirements *.txt exclude .codecov.yml .coveragerc .git-blame-ignore-revs .gitmodules .hgchurn .mailmap exclude .pre-commit-config.yaml clean.sh nose_answer.cfg nose_unit.cfg nose_ignores.txt diff --git a/doc/source/developing/building_the_docs.rst b/doc/source/developing/building_the_docs.rst index e1b5629802..ef8d08b8c6 100644 --- a/doc/source/developing/building_the_docs.rst +++ b/doc/source/developing/building_the_docs.rst @@ -90,7 +90,7 @@ the top level of a local copy, run .. code-block:: bash - $ python -m pip install -e ".[doc]" + $ python -m pip install -e . -r requirements/docs.txt Quick versus Full Documentation Builds ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/pyproject.toml b/pyproject.toml index 86bdc5de23..85f9189d2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,8 @@ keywords = [ "astronomy astrophysics visualization amr adaptivemeshrefinement", ] requires-python = ">=3.9.2" + +# keep in sync with requirements/minimal_env.txt dependencies = [ "cmyt>=1.1.2", "ewah-bool-utils>=1.2.0", @@ -190,37 +192,9 @@ full = [ ] # dev-only extra targets -doc = [ - "alabaster>=0.7.13", - "bottle>=0.12.25", - "ipykernel>=6.29.4", - "jinja2<3.1.0", # see https://github.com/readthedocs/readthedocs.org/issues/9037 - "jupyter-client>=8.3.1", - "nbsphinx>=0.9.3", - "nose~=1.3.7; python_version < '3.10'", - "pytest>=6.1", - "pyx>=0.15", - "sphinx>=7.2.5", - "sphinx-bootstrap-theme>=0.8.1", - "sphinx-rtd-theme>=1.3.0", -] mapserver = [ "bottle", ] -minimal = [ - "cmyt==1.1.2", - "ewah-bool-utils==1.2.0", - "matplotlib==3.5", - "more-itertools==8.4", - "numpy==1.19.3", - "packaging==20.9", - "pillow==8.0.0", - "tomli-w==0.4.0", - "tqdm==3.4.0", - "unyt==2.9.2", - "tomli==1.2.3;python_version < '3.11'", - "typing-extensions==4.4.0;python_version < '3.12'", -] test = [ "pyaml>=17.10.0", "pytest>=6.1", @@ -231,13 +205,6 @@ test = [ "nose-timer~=1.0.0; python_version < '3.10'", "imageio!=2.35.0", # see https://github.com/yt-project/yt/issues/4966 ] -typecheck = [ - "mypy==1.8.0", - "types-PyYAML==6.0.12.12", - "types-chardet==5.0.4.6", - "types-requests==2.31.0.20240125", - "typing-extensions==4.4.0; python_version < '3.12'", -] [project.scripts] yt = "yt.utilities.command_line:run_main" diff --git a/requirements/docs.txt b/requirements/docs.txt new file mode 100644 index 0000000000..e577a5b968 --- /dev/null +++ b/requirements/docs.txt @@ -0,0 +1,12 @@ +alabaster>=0.7.13 +bottle>=0.12.25 +ipykernel>=6.29.4 +jinja2<3.1.0 # see https://github.com/readthedocs/readthedocs.org/issues/9037 +jupyter-client>=8.3.1 +nbsphinx>=0.9.3 +nose~=1.3.7; python_version < '3.10' +pytest>=6.1 +pyx>=0.15 +sphinx>=7.2.5 +sphinx-bootstrap-theme>=0.8.1 +sphinx-rtd-theme>=1.3.0 diff --git a/requirements/minimal_env.txt b/requirements/minimal_env.txt new file mode 100644 index 0000000000..086d0076d2 --- /dev/null +++ b/requirements/minimal_env.txt @@ -0,0 +1,69 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile pyproject.toml --python=3.9 --python-platform=x86_64-unknown-linux-gnu --resolution=lowest-direct --no-build +cmyt==1.1.2 + # via yt (pyproject.toml) +colorspacious==1.1.2 + # via cmyt +cycler==0.12.1 + # via matplotlib +ewah-bool-utils==1.2.0 + # via yt (pyproject.toml) +fonttools==4.53.1 + # via matplotlib +kiwisolver==1.4.7 + # via matplotlib +matplotlib==3.5.0 + # via + # yt (pyproject.toml) + # cmyt +more-itertools==8.4.0 + # via + # yt (pyproject.toml) + # cmyt +mpmath==1.3.0 + # via sympy +numpy==1.19.3 + # via + # yt (pyproject.toml) + # cmyt + # colorspacious + # ewah-bool-utils + # matplotlib + # unyt +packaging==20.9 + # via + # yt (pyproject.toml) + # matplotlib + # setuptools-scm +pillow==8.0.0 + # via + # yt (pyproject.toml) + # matplotlib +pyparsing==3.1.4 + # via + # matplotlib + # packaging +python-dateutil==2.9.0.post0 + # via matplotlib +setuptools==74.1.2 + # via setuptools-scm +setuptools-scm==8.1.0 + # via matplotlib +six==1.16.0 + # via python-dateutil +sympy==1.13.2 + # via unyt +tomli==1.2.3 + # via + # yt (pyproject.toml) + # setuptools-scm +tomli-w==0.4.0 + # via yt (pyproject.toml) +tqdm==3.4.0 + # via yt (pyproject.toml) +typing-extensions==4.4.0 + # via + # yt (pyproject.toml) + # setuptools-scm +unyt==2.9.2 + # via yt (pyproject.toml) diff --git a/requirements/typecheck.txt b/requirements/typecheck.txt new file mode 100644 index 0000000000..8d0e0ceaff --- /dev/null +++ b/requirements/typecheck.txt @@ -0,0 +1,5 @@ +mypy==1.8.0 +types-PyYAML==6.0.12.12 +types-chardet==5.0.4.6 +types-requests==2.31.0.20240125 +typing-extensions==4.4.0; python_version < '3.12' diff --git a/tests/ci_install.sh b/tests/ci_install.sh index b73c90f739..a29d9d5334 100644 --- a/tests/ci_install.sh +++ b/tests/ci_install.sh @@ -32,7 +32,7 @@ fi # but the primary intention is to embed this script in CI jobs if [[ ${dependencies} == "minimal" ]]; then # test with minimal versions of runtime dependencies - python -m pip install -e ".[test,minimal]" + python -m pip install -e ".[test]" -r requirements/minimal_env.txt elif [[ ${dependencies} == "cartopy" ]]; then python -m pip install 'cartopy>=0.22' # scipy is an optional dependency to cartopy From 342340c728ebea72e079f32181eb894cabeb1574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Wed, 11 Sep 2024 17:16:56 +0200 Subject: [PATCH 169/186] DOC: update developer guide on Handling Dependencies --- doc/source/developing/testing.rst | 55 ++++++++++++++----------------- 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/doc/source/developing/testing.rst b/doc/source/developing/testing.rst index 96ddfad842..eab8270fe5 100644 --- a/doc/source/developing/testing.rst +++ b/doc/source/developing/testing.rst @@ -441,24 +441,26 @@ Handling yt Dependencies ------------------------ Our dependencies are specified in ``pyproject.toml``. Hard dependencies are found in -``options.install_requires``, while optional dependencies are specified in -``options.extras_require``. The ``full`` target contains the specs to run our +``project.dependencies``, while optional dependencies are specified in +``project.optional-dependencies``. The ``full`` target contains the specs to run our test suite, which are intended to be as modern as possible (we don't set upper -limits to versions unless we need to). The ``minimal`` target is used to check -that we don't break backward compatibility with old versions of upstream -projects by accident. It is intended to pin strictly our minimal supported -versions. The ``test`` target specifies the tools needed to run the tests, but +limits to versions unless we need to). + +The ``test`` target specifies the tools needed to run the tests, but not needed by yt itself. +Documentation and typechecking requirements are found in ``requirements/``, +and used in ``tests/ci_install.sh``. + **Python version support.** -When a new Python version is released, it takes about -a month or two for yt to support it, since we're dependent on bigger projects -like numpy and matplotlib. We vow to follow numpy's deprecation plan regarding -our supported versions for Python and numpy, defined formally in `NEP 29 -`_. However, we try to -avoid bumping our minimal requirements shortly before a yt release. +We vow to follow numpy's deprecation plan regarding our supported versions for Python +and numpy, defined formally in +`NEP 29 `_, but generally +support larger version intervals than recommended in this document. **Third party dependencies.** +We attempt to make yt compatible with a wide variety of upstream software +versions. However, sometimes a specific version of a project that yt depends on causes some breakage and must be blacklisted in the tests or a more experimental project that yt depends on optionally might change sufficiently @@ -466,29 +468,20 @@ that the yt community decides not to support an old version of that project. **Note.** Some of our optional dependencies are not trivial to install and their support -may vary across platforms. To manage such issue, we currently use requirement -files in additions to ``pyproject.toml``. They are found in -``tests/*requirements.txt`` and used in ``tests/ci_install.sh``. - -We attempt to make yt compatible with a wide variety of upstream software -versions. However, sometimes a specific version of a project that yt depends on -causes some breakage and must be blacklisted in the tests or a more -experimental project that yt depends on optionally might change sufficiently -that the yt community decides not to support an old version of that project. - -To handle cases like this, the versions of upstream software projects installed -on the machines running the yt test suite are pinned to specific version -numbers that must be updated manually. This prevents breaking the yt tests when -a new version of an upstream dependency is released and allows us to manage -updates in upstream projects at our pace. +may vary across platforms. If you would like to add a new dependency for yt (even an optional dependency) or would like to update a version of a yt dependency, you must edit the -``tests/test_requirements.txt`` file, this path is relative to the root of the -repository. This file contains an enumerated list of direct dependencies and -pinned version numbers. For new dependencies, simply append the name of the new +``pyproject.toml`` file. For new dependencies, simply append the name of the new dependency to the end of the file, along with a pin to the latest version number of the package. To update a package's version, simply update the version number in the entry for that package. -Finally, we also run a set of tests with "minimal" dependencies installed. When adding tests that depend on an optional dependency, you can wrap the test with the ``yt.testing.requires_module decorator`` to ensure it does not run during the minimal dependency tests (see yt/frontends/amrvac/tests/test_read_amrvac_namelist.py for a good example). If for some reason you need to update the listing of packages that are installed for the "minimal" dependency tests, you will need to edit ``tests/test_minimal_requirements.txt``. +Finally, we also run a set of tests with "minimal" dependencies installed. +When adding tests that depend on an optional dependency, you can wrap the test +with the ``yt.testing.requires_module decorator`` to ensure it does not run +during the minimal dependency tests (see +``yt/frontends/amrvac/tests/test_read_amrvac_namelist.py`` for a good example). +If for some reason you need to update the listing of packages that are installed +for the "minimal" dependency tests, you will need to update +``requirements/minimal_env.txt``. From f1cdc9c6852c265670d451fa8e8717e9dd853d33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Sep 2024 16:34:27 +0000 Subject: [PATCH 170/186] Bump pypa/gh-action-pypi-publish Bumps the actions group in /.github/workflows with 1 update: [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `pypa/gh-action-pypi-publish` from 1.10.0 to 1.10.1 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.10.0...v1.10.1) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/wheels.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index cd1f65b437..b7484baf22 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -124,7 +124,7 @@ jobs: merge-multiple: true - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.10.0 + uses: pypa/gh-action-pypi-publish@v1.10.1 with: user: __token__ password: ${{ secrets.pypi_token }} From 378de344ac251c85e10205256c80d3f238010af2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Wed, 11 Sep 2024 21:15:34 +0200 Subject: [PATCH 171/186] MNT: restrict dependabot pip updates to requirements/typecheck.txt --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2ed6f7807c..204e5592b6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,7 +7,7 @@ updates: groups: actions: patterns: - - '*' + - 'typecheck.txt' - package-ecosystem: github-actions directory: /.github/workflows From f69487e413d9e44036223d21cbd62b92f65e21cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Wed, 11 Sep 2024 21:18:13 +0200 Subject: [PATCH 172/186] TST: bump typecheck dependencies --- requirements/typecheck.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements/typecheck.txt b/requirements/typecheck.txt index 8d0e0ceaff..3edda47449 100644 --- a/requirements/typecheck.txt +++ b/requirements/typecheck.txt @@ -1,5 +1,5 @@ -mypy==1.8.0 -types-PyYAML==6.0.12.12 +mypy==1.11.2 +types-PyYAML==6.0.12.20240808 types-chardet==5.0.4.6 -types-requests==2.31.0.20240125 +types-requests==2.32.0.20240907 typing-extensions==4.4.0; python_version < '3.12' From e22ab1f0a1c1538c707b288d83348a3a097dc07a Mon Sep 17 00:00:00 2001 From: Shaokun Xie Date: Fri, 13 Sep 2024 16:03:27 +0800 Subject: [PATCH 173/186] BUG: fix orientation of colorbar in multiplanel plot (#4981) --- tests/pytest_mpl_baseline | 2 +- yt/visualization/base_plot_types.py | 24 ++++++++++--- .../tests/test_image_comp_2D_plots.py | 34 +++++++++++++++++++ 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/tests/pytest_mpl_baseline b/tests/pytest_mpl_baseline index 9162b1a3f1..8a30c989a3 160000 --- a/tests/pytest_mpl_baseline +++ b/tests/pytest_mpl_baseline @@ -1 +1 @@ -Subproject commit 9162b1a3f1bdccbe398221fec8ace489e53078d9 +Subproject commit 8a30c989a30b00e681db645cc767cb9047508169 diff --git a/yt/visualization/base_plot_types.py b/yt/visualization/base_plot_types.py index 377e316be9..94792c32d3 100644 --- a/yt/visualization/base_plot_types.py +++ b/yt/visualization/base_plot_types.py @@ -341,7 +341,20 @@ def _set_axes(self) -> None: self.image.axes.set_facecolor(self.colorbar_handler.background_color) self.cax.tick_params(which="both", direction="in") - self.cb = self.figure.colorbar(self.image, self.cax) + + # For creating a multipanel plot by ImageGrid + # we may need the location keyword, which requires Matplotlib >= 3.7.0 + cb_location = getattr(self.cax, "orientation", None) + if matplotlib.__version_info__ >= (3, 7): + self.cb = self.figure.colorbar(self.image, self.cax, location=cb_location) + else: + if cb_location in ["top", "bottom"]: + warnings.warn( + "Cannot properly set the orientation of colorbar. " + "Consider upgrading matplotlib to version 3.7 or newer", + stacklevel=6, + ) + self.cb = self.figure.colorbar(self.image, self.cax) cb_axis: Axis if self.cb.orientation == "vertical": @@ -526,9 +539,12 @@ def _toggle_colorbar(self, choice: bool): def _get_labels(self): labels = super()._get_labels() - cbax = self.cb.ax - labels += cbax.yaxis.get_ticklabels() - labels += [cbax.yaxis.label, cbax.yaxis.get_offset_text()] + if getattr(self.cb, "orientation", "vertical") == "horizontal": + cbaxis = self.cb.ax.xaxis + else: + cbaxis = self.cb.ax.yaxis + labels += cbaxis.get_ticklabels() + labels += [cbaxis.label, cbaxis.get_offset_text()] return labels def hide_axes(self, *, draw_frame=None): diff --git a/yt/visualization/tests/test_image_comp_2D_plots.py b/yt/visualization/tests/test_image_comp_2D_plots.py index 254c8e5338..371256c0eb 100644 --- a/yt/visualization/tests/test_image_comp_2D_plots.py +++ b/yt/visualization/tests/test_image_comp_2D_plots.py @@ -147,6 +147,40 @@ def test_particleprojectionplot_set_colorbar_properties(): return p.plots[field].figure +class TestMultipanelPlot: + @classmethod + def setup_class(cls): + cls.fields = [ + ("gas", "density"), + ("gas", "velocity_x"), + ("gas", "velocity_y"), + ("gas", "velocity_magnitude"), + ] + cls.ds = fake_random_ds(16) + + @pytest.mark.skipif( + mpl.__version_info__ < (3, 7), + reason="colorbar cannot currently be set horizontal in multi-panel plot with matplotlib older than 3.7.0", + ) + @pytest.mark.parametrize("cbar_location", ["top", "bottom", "left", "right"]) + @pytest.mark.mpl_image_compare + def test_multipanelplot_colorbar_orientation_simple(self, cbar_location): + p = SlicePlot(self.ds, "z", self.fields) + return p.export_to_mpl_figure((2, 2), cbar_location=cbar_location) + + @pytest.mark.parametrize("cbar_location", ["top", "bottom"]) + def test_multipanelplot_colorbar_orientation_warning(self, cbar_location): + p = SlicePlot(self.ds, "z", self.fields) + if mpl.__version_info__ < (3, 7): + with pytest.warns( + UserWarning, + match="Cannot properly set the orientation of colorbar.", + ): + p.export_to_mpl_figure((2, 2), cbar_location=cbar_location) + else: + p.export_to_mpl_figure((2, 2), cbar_location=cbar_location) + + class TestProfilePlot: @classmethod def setup_class(cls): From 510c2135be26e8a7af0cad6a2dd84f05ad193d03 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Sun, 15 Sep 2024 17:58:17 -0400 Subject: [PATCH 174/186] fix Machnumber field spelling in amrex frontend for Castro also add abar and Ye as defined fields so they print well --- yt/frontends/amrex/fields.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/yt/frontends/amrex/fields.py b/yt/frontends/amrex/fields.py index fbde2d2bb0..3901cf180c 100644 --- a/yt/frontends/amrex/fields.py +++ b/yt/frontends/amrex/fields.py @@ -328,7 +328,9 @@ class CastroFieldInfo(FieldInfoContainer): ("erg/cm**3", ["kinetic_energy_density"], r"\frac{1}{2}\rho|\mathbf{U}|^2"), ), ("soundspeed", ("cm/s", ["sound_speed"], "Sound Speed")), - ("Machnumber", ("", ["mach_number"], "Mach Number")), + ("MachNumber", ("", ["mach_number"], "Mach Number")), + ("abar", ("", [], r"$\bar{A}$")), + ("Ye", ("", [], r"$Y_e$")), ("entropy", ("erg/(g*K)", ["entropy"], r"s")), ("magvort", ("1/s", ["vorticity_magnitude"], r"|\nabla \times \mathbf{U}|")), ("divu", ("1/s", ["velocity_divergence"], r"\nabla \cdot \mathbf{U}")), From d2acf74f31d8014efb779b061833dfdcdaffecff Mon Sep 17 00:00:00 2001 From: Victor Rufo Pastor Date: Wed, 14 Feb 2024 14:28:47 +0100 Subject: [PATCH 175/186] Fix missing mu in temperature calculation and creating new field: Temperature_over_mu --- yt/frontends/ramses/fields.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/yt/frontends/ramses/fields.py b/yt/frontends/ramses/fields.py index dca50a6282..749964be44 100644 --- a/yt/frontends/ramses/fields.py +++ b/yt/frontends/ramses/fields.py @@ -196,18 +196,28 @@ def star_age(field, data): ) def setup_fluid_fields(self): - def _temperature(field, data): + def _temperature_over_mu(field, data): rv = data["gas", "pressure"] / data["gas", "density"] rv *= mass_hydrogen_cgs / boltzmann_constant_cgs return rv + self.add_field( + ("gas", "temperature_over_mu"), + sampling_type="cell", + function=_temperature_over_mu, + units=self.ds.unit_system["temperature"], + ) + self.create_cooling_fields() + + def _temperature(field, data): + return data["gas", "temperature_over_mu"] * data["gas", "mu"] + self.add_field( ("gas", "temperature"), sampling_type="cell", function=_temperature, units=self.ds.unit_system["temperature"], ) - self.create_cooling_fields() self.species_names = [ known_species_names[fn] @@ -387,10 +397,10 @@ def create_cooling_fields(self): # Function to create the cooling fields def _create_field(name, interp_object, unit): def _func(field, data): - shape = data["gas", "temperature"].shape + shape = data["gas", "temperature_over_mu"].shape d = { "lognH": np.log10(_X * data["gas", "density"] / mh).ravel(), - "logT": np.log10(data["gas", "temperature"]).ravel(), + "logT": np.log10(data["gas", "temperature_over_mu"]).ravel(), } rv = interp_object(d).reshape(shape) if name[-1] != "mu": @@ -446,7 +456,7 @@ def _func(field, data): ["lognH", "logT"], truncate=True, ) - _create_field(("gas", "mu"), interp, tvals["mu"]["unit"]) + _create_field(("gas", "mu"), interp, "dimensionless") # Add the number density field, based on mu def _number_density(field, data): From 73c5befdbd0485268dba4843849830c0f20def56 Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Wed, 14 Feb 2024 16:25:05 +0100 Subject: [PATCH 176/186] Raise warning but do not fail when using old version --- yt/frontends/ramses/fields.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/yt/frontends/ramses/fields.py b/yt/frontends/ramses/fields.py index 749964be44..a10c811038 100644 --- a/yt/frontends/ramses/fields.py +++ b/yt/frontends/ramses/fields.py @@ -1,4 +1,5 @@ import os +import warnings from functools import partial import numpy as np @@ -207,10 +208,23 @@ def _temperature_over_mu(field, data): function=_temperature_over_mu, units=self.ds.unit_system["temperature"], ) - self.create_cooling_fields() - - def _temperature(field, data): - return data["gas", "temperature_over_mu"] * data["gas", "mu"] + found_cooling_fields = self.create_cooling_fields() + + if found_cooling_fields: + + def _temperature(field, data): + return data["gas", "temperature_over_mu"] * data["gas", "mu"] + + else: + + def _temperature(field, data): + warnings.warn( + "Trying to calculate temperature but the cooling tables couldn't be found or read. " + "yt will return T/µ instead of T — this is equivalent to assuming µ=1.0", + category=RuntimeWarning, + stacklevel=1, + ) + return data["gas", "temperature_over_mu"] self.add_field( ("gas", "temperature"), @@ -383,7 +397,8 @@ def _photon_flux(field, data): units=flux_unit, ) - def create_cooling_fields(self): + def create_cooling_fields(self) -> bool: + "Create cooling fields from the cooling files. Return True if successful." num = os.path.basename(self.ds.parameter_filename).split(".")[0].split("_")[1] filename = "%s/cooling_%05i.out" % ( os.path.dirname(self.ds.parameter_filename), @@ -392,7 +407,7 @@ def create_cooling_fields(self): if not os.path.exists(filename): mylog.warning("This output has no cooling fields") - return + return False # Function to create the cooling fields def _create_field(name, interp_object, unit): @@ -435,7 +450,7 @@ def _func(field, data): "This cooling file format is no longer supported. " "Cooling field loading skipped." ) - return + return False if var.size == n1 * n2: tvals[tname] = { "data": var.reshape((n1, n2), order="F"), @@ -514,3 +529,5 @@ def _net_cool(field, data): function=_net_cool, units=cooling_function_units, ) + + return True From ee1f138df5df62460f3d505a21e9d101835832e4 Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Wed, 14 Feb 2024 16:40:02 +0100 Subject: [PATCH 177/186] Test that a warning is properly raised --- yt/frontends/ramses/fields.py | 17 ++++++++----- .../ramses/tests/test_outputs_pytest.py | 25 +++++++++++++++++++ 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/yt/frontends/ramses/fields.py b/yt/frontends/ramses/fields.py index a10c811038..4c871ec08b 100644 --- a/yt/frontends/ramses/fields.py +++ b/yt/frontends/ramses/fields.py @@ -6,6 +6,7 @@ from yt import units from yt._typing import KnownFieldsT +from yt.fields.field_detector import FieldDetector from yt.fields.field_info_container import FieldInfoContainer from yt.frontends.ramses.io import convert_ramses_conformal_time_to_physical_age from yt.utilities.cython_fortran_utils import FortranFile @@ -218,12 +219,16 @@ def _temperature(field, data): else: def _temperature(field, data): - warnings.warn( - "Trying to calculate temperature but the cooling tables couldn't be found or read. " - "yt will return T/µ instead of T — this is equivalent to assuming µ=1.0", - category=RuntimeWarning, - stacklevel=1, - ) + if not isinstance(data, FieldDetector): + warnings.warn( + "Trying to calculate temperature but the cooling tables " + "couldn't be found or read. yt will return T/µ instead of " + "T — this is equivalent to assuming µ=1.0. To suppress this, " + "derive the temperature from temperature_over_mu with " + "some values for mu.", + category=RuntimeWarning, + stacklevel=1, + ) return data["gas", "temperature_over_mu"] self.add_field( diff --git a/yt/frontends/ramses/tests/test_outputs_pytest.py b/yt/frontends/ramses/tests/test_outputs_pytest.py index 40b100fe5d..668c78e2d0 100644 --- a/yt/frontends/ramses/tests/test_outputs_pytest.py +++ b/yt/frontends/ramses/tests/test_outputs_pytest.py @@ -55,3 +55,28 @@ def test_field_config_2(custom_ramses_fields_conf): assert ("ramses", f) in ds.field_list for f in custom_grav: assert ("gravity", f) in ds.field_list + + +@requires_file(output_00080) +@requires_file(ramses_new_format) +def test_warning_T2(): + ds1 = yt.load(output_00080) + ds2 = yt.load(ramses_new_format) + + # Should not raise warnings + ds1.r["gas", "temperature_over_mu"] + ds2.r["gas", "temperature_over_mu"] + + # We cannot read the cooling tables of output_00080 + # so this should raise a warning + with pytest.warns( + RuntimeWarning, + match=( + "Trying to calculate temperature but the cooling tables couldn't be " + r"found or read\. yt will return T/µ instead of T.*" + ), + ): + ds1.r["gas", "temperature"] + + # But this one should not + ds2.r["gas", "temperature"] From 91826500a4f81a0d9b9542b496949c1db99fec6c Mon Sep 17 00:00:00 2001 From: Corentin Cadiou Date: Wed, 18 Sep 2024 20:37:05 +0200 Subject: [PATCH 178/186] Fix linting --- yt/testing.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/yt/testing.py b/yt/testing.py index 442ed4954c..80c238ae1c 100644 --- a/yt/testing.py +++ b/yt/testing.py @@ -6,11 +6,10 @@ import sys import tempfile import unittest -from collections.abc import Mapping from functools import wraps from importlib.util import find_spec from shutil import which -from typing import Callable, Union +from typing import TYPE_CHECKING, Callable, Union from unittest import SkipTest import matplotlib @@ -20,13 +19,17 @@ from unyt.exceptions import UnitOperationError from yt._maintenance.deprecation import issue_deprecation_warning -from yt._typing import AnyFieldKey from yt.config import ytcfg from yt.frontends.stream.data_structures import StreamParticlesDataset from yt.funcs import is_sequence from yt.loaders import load, load_particles from yt.units.yt_array import YTArray, YTQuantity +if TYPE_CHECKING: + from collections.abc import Mapping + + from yt._typing import AnyFieldKey + if sys.version_info < (3, 10): from yt._maintenance.backports import zip From f63e9658b1722a6aea3395295563d8d66e222a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sat, 21 Sep 2024 14:22:08 +0200 Subject: [PATCH 179/186] STY: autofix RUF031 (incorrectly-parenthesized-tuple-in-subscript) --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index bb45c14c1d..8a562a27a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -267,6 +267,7 @@ select = [ "UP", # pyupgrade "I", # isort "NPY", # numpy specific rules + "RUF031"# incorrectly-parenthesized-tuple-in-subscript ] ignore = [ "E501", # line too long From 0b6823b418822c12f04201678c8b6273f130fdb9 Mon Sep 17 00:00:00 2001 From: Shaokun Xie Date: Mon, 23 Sep 2024 23:06:55 +0800 Subject: [PATCH 180/186] DOC: fix typo in a warning message --- yt/visualization/plot_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/visualization/plot_container.py b/yt/visualization/plot_container.py index c8327a2b55..44e150a663 100644 --- a/yt/visualization/plot_container.py +++ b/yt/visualization/plot_container.py @@ -1033,7 +1033,7 @@ def set_zlim( issue_deprecation_warning( "Passing `zmax=None` explicitly is deprecated. " "If you wish to explicitly set zmax to the maximal " - "data value, pass `zmin='max'` instead. " + "data value, pass `zmax='max'` instead. " "Otherwise leave this argument unset.", since="4.1", stacklevel=5, From 16c4283dca6d590001201ad66ca8367924627936 Mon Sep 17 00:00:00 2001 From: chavlin Date: Mon, 23 Sep 2024 12:53:11 -0500 Subject: [PATCH 181/186] ensure YTSlice.coord has units --- yt/data_objects/selection_objects/slices.py | 3 ++- yt/data_objects/tests/test_sph_data_objects.py | 10 ++++++++++ yt/funcs.py | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/yt/data_objects/selection_objects/slices.py b/yt/data_objects/selection_objects/slices.py index 9bb2965604..fd510737f3 100644 --- a/yt/data_objects/selection_objects/slices.py +++ b/yt/data_objects/selection_objects/slices.py @@ -6,6 +6,7 @@ ) from yt.data_objects.static_output import Dataset from yt.funcs import ( + fix_length, is_sequence, iter_fields, validate_3d_array, @@ -78,7 +79,7 @@ def __init__( validate_object(data_source, YTSelectionContainer) YTSelectionContainer2D.__init__(self, axis, ds, field_parameters, data_source) self._set_center(center) - self.coord = coord + self.coord = fix_length(coord, ds) def _generate_container_field(self, field): xax = self.ds.coordinates.x_axis[self.axis] diff --git a/yt/data_objects/tests/test_sph_data_objects.py b/yt/data_objects/tests/test_sph_data_objects.py index 5a1acd4fbc..8cf769f224 100644 --- a/yt/data_objects/tests/test_sph_data_objects.py +++ b/yt/data_objects/tests/test_sph_data_objects.py @@ -64,6 +64,16 @@ def test_slice(): } +def test_slice_to_frb(): + ds = fake_sph_orientation_ds() + frb = ds.slice(0, 0.5).to_frb(ds.domain_width[0], (64, 64)) + ref_vals = frb["gas", "density"] + for center in ((0.5, "code_length"), (0.5, "cm"), ds.quan(0.5, "code_length")): + frb = ds.slice(0, center).to_frb(ds.domain_width[0], (64, 64)) + vals = frb["gas", "density"] + assert_equal(vals, ref_vals) + + def test_region(): ds = fake_sph_orientation_ds() for (left_edge, right_edge), answer in REGION_ANSWERS.items(): diff --git a/yt/funcs.py b/yt/funcs.py index fa0b89abaa..b6d10b2755 100644 --- a/yt/funcs.py +++ b/yt/funcs.py @@ -1106,7 +1106,7 @@ def validate_3d_array(obj): def validate_float(obj): """Validates if the passed argument is a float value. - Raises an exception if `obj` is a single float value + Raises an exception if `obj` is not a single float value or a YTQuantity of size 1. Parameters From fa92ac3995e5c5d7f63bd5459ed55ea47d46de5d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Sep 2024 15:44:04 +0000 Subject: [PATCH 182/186] Bump the actions group in /.github/workflows with 2 updates Bumps the actions group in /.github/workflows with 2 updates: [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) and [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `pypa/cibuildwheel` from 2.20.0 to 2.21.1 - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.20.0...v2.21.1) Updates `pypa/gh-action-pypi-publish` from 1.10.1 to 1.10.2 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.10.1...v1.10.2) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/wheels.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index a304652701..cd63fbc161 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -33,7 +33,7 @@ jobs: uses: actions/checkout@v4 - name: Build wheels for CPython - uses: pypa/cibuildwheel@v2.20.0 + uses: pypa/cibuildwheel@v2.21.1 with: output-dir: dist @@ -125,7 +125,7 @@ jobs: merge-multiple: true - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.10.1 + uses: pypa/gh-action-pypi-publish@v1.10.2 with: user: __token__ password: ${{ secrets.pypi_token }} From 50143ba88bf791bebed761e83a9c38e068d213ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Wed, 25 Sep 2024 17:56:23 +0200 Subject: [PATCH 183/186] TYP: hotfix unsolvable type-checking requirements --- .github/workflows/type-checking.yaml | 3 ++- requirements/typecheck.txt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/type-checking.yaml b/.github/workflows/type-checking.yaml index 3c7f7307d2..83713c2bc0 100644 --- a/.github/workflows/type-checking.yaml +++ b/.github/workflows/type-checking.yaml @@ -7,7 +7,8 @@ on: pull_request: paths: - yt/**/*.py - - setup.cfg + - pyproject.toml + - requirements/typecheck.txt - .github/workflows/type-checking.yaml workflow_dispatch: diff --git a/requirements/typecheck.txt b/requirements/typecheck.txt index 3edda47449..f116cfe1d5 100644 --- a/requirements/typecheck.txt +++ b/requirements/typecheck.txt @@ -2,4 +2,4 @@ mypy==1.11.2 types-PyYAML==6.0.12.20240808 types-chardet==5.0.4.6 types-requests==2.32.0.20240907 -typing-extensions==4.4.0; python_version < '3.12' +typing-extensions==4.6.0; python_version < '3.12' From a760066a3f66f8f62724385319067f10f7dd4486 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Wed, 25 Sep 2024 18:49:30 +0200 Subject: [PATCH 184/186] ignore an obscure typechecking warning --- yt/data_objects/construction_data_containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/data_objects/construction_data_containers.py b/yt/data_objects/construction_data_containers.py index a406018925..38deeb9bbc 100644 --- a/yt/data_objects/construction_data_containers.py +++ b/yt/data_objects/construction_data_containers.py @@ -1426,7 +1426,7 @@ class YTSmoothedCoveringGrid(YTCoveringGrid): filename = None _min_level = None - @wraps(YTCoveringGrid.__init__) + @wraps(YTCoveringGrid.__init__) # type: ignore [misc] def __init__(self, *args, **kwargs): ds = kwargs["ds"] self._base_dx = ( From 59a346b19f02e0d94399d3178aec9e7a0d836c98 Mon Sep 17 00:00:00 2001 From: chavlin Date: Wed, 25 Sep 2024 14:30:35 -0500 Subject: [PATCH 185/186] exclude h5py 3.12.0 from test dependencies --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 8a562a27a7..b8c5f54ee1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -205,6 +205,7 @@ test = [ "nose-exclude; python_version < '3.10'", "nose-timer~=1.0.0; python_version < '3.10'", "imageio!=2.35.0", # see https://github.com/yt-project/yt/issues/4966 + "h5py!=3.12.0", # see https://github.com/h5py/h5py/issues/2505 ] [project.scripts] From cf54d4869cb8e6e57e1301ad870bbeb8a13a8ee4 Mon Sep 17 00:00:00 2001 From: chavlin Date: Wed, 25 Sep 2024 14:54:43 -0500 Subject: [PATCH 186/186] move h5py exclusion up to HDF5 set --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b8c5f54ee1..08d31233d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ answer-testing = "yt.utilities.answer_testing.framework:AnswerTesting" [project.optional-dependencies] # some generic, reusable constraints on optional-deps -HDF5 = ["h5py>=3.1.0"] +HDF5 = ["h5py>=3.1.0,!=3.12.0; platform_system=='Windows'"] # see https://github.com/h5py/h5py/issues/2505 netCDF4 = ["netCDF4!=1.6.1,>=1.5.3"] # see https://github.com/Unidata/netcdf4-python/issues/1192 Fortran = ["f90nml>=1.1"] @@ -205,7 +205,6 @@ test = [ "nose-exclude; python_version < '3.10'", "nose-timer~=1.0.0; python_version < '3.10'", "imageio!=2.35.0", # see https://github.com/yt-project/yt/issues/4966 - "h5py!=3.12.0", # see https://github.com/h5py/h5py/issues/2505 ] [project.scripts]