Merge pull request #167 from LSSTDESC/oroob

Fix crash when certain large geometries would cause coverage map to go out-of-bounds.

.github/workflows/python-package-pr.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
 
     steps:
     - uses: actions/checkout@v3

.github/workflows/python-package-publish.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.12"
           cache: "pip"
           cache-dependency-path: "setup.cfg"
 
@@ -42,7 +42,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.12"
           cache: "pip"
           cache-dependency-path: "setup.cfg"
 

.github/workflows/python-package-push.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.12"]
 
     steps:
     - uses: actions/checkout@v3

healsparse/healSparseMap.py

@@ -553,15 +553,15 @@ def update_values_pix(self, pixels, values, nest=True, operation='replace'):
                     if not self.is_integer_map:
                         raise ValueError("Cannot set non-integer map with an integer")
                     is_single_value = True
-                    _values = np.array([values], dtype=self.dtype)
+                    _values = np.asarray([values], dtype=self.dtype)
                 elif isinstance(values, numbers.Real):
                     if self.is_integer_map:
                         raise ValueError("Cannot set non-floating point map with a floating point.")
                     is_single_value = True
-                    _values = np.array([values], dtype=self.dtype)
+                    _values = np.asarray([values], dtype=self.dtype)
                 elif isinstance(values, (bool, np.bool_)):
                     is_single_value = True
-                    _values = np.array([values], dtype=bool)
+                    _values = np.asarray([values], dtype=bool)
 
         if isinstance(values, np.ndarray) and len(values) == 1:
             is_single_value = True
@@ -695,6 +695,10 @@ def _update_values_pixel_ranges(self, pixel_ranges, value, operation, no_append)
         # Compute the coverage pixels.
         cov_pix_ranges = np.right_shift(pixel_ranges, self._cov_map.bit_shift)
         # After the bit shift these pixel ranges are inclusive, not exclusive.
+        # But we also need to protect against an overrun at the high end.
+        if cov_pix_ranges[-1, 1] == len(self.coverage_mask):
+            cov_pix_ranges[-1, 1] = len(self.coverage_mask) - 1
+
         cov_pix_to_set = hpg.pixel_ranges_to_pixels(cov_pix_ranges, inclusive=True)
         cov_pix_to_set = np.unique(cov_pix_to_set)
 
@@ -881,7 +885,7 @@ def get_values_pix(self, pixels, nest=True, valid_mask=False, nside=None):
             if self._is_wide_mask:
                 return np.zeros((0, self._wide_mask_width), dtype=self.dtype)
             else:
-                return np.array([], dtype=self.dtype)
+                return np.asarray([], dtype=self.dtype)
 
         if not nest:
             _pix = hpg.ring_to_nest(self._nside_sparse, pixels)
@@ -1343,7 +1347,7 @@ def valid_pixels(self):
         elif self._is_bit_packed:
             # This is dangerous because it expands into a full array first; this
             # can blow up memory.
-            valid_pixel_inds, = np.where(np.array(self._sparse_map) != self._sentinel)
+            valid_pixel_inds, = np.where(np.asarray(self._sparse_map) != self._sentinel)
         else:
             valid_pixel_inds, = np.where(self._sparse_map != self._sentinel)
 
@@ -1442,7 +1446,7 @@ def valid_pixels_single_covpix(self, cov_pix):
         """
         # Check if this is in the coverage mask.
         if not self.coverage_mask[cov_pix]:
-            return np.array([], dtype=np.int64)
+            return np.asarray([], dtype=np.int64)
 
         # This is the start of the coverage pixel slice.
         start = (self._cov_map[cov_pix] +
@@ -1454,7 +1458,7 @@ def valid_pixels_single_covpix(self, cov_pix):
         elif self._is_wide_mask:
             valid_pixel_inds, = np.where(np.any(self._sparse_map[s, :] != self._sentinel, axis=1))
         elif self._is_bit_packed:
-            valid_pixel_inds, = np.where(np.array(self._sparse_map[s]) != self._sentinel)
+            valid_pixel_inds, = np.where(np.asarray(self._sparse_map[s]) != self._sentinel)
         else:
             valid_pixel_inds, = np.where(self._sparse_map[s] != self._sentinel)
 
@@ -1825,7 +1829,7 @@ def __getitem__(self, key):
         elif isinstance(key, numbers.Integral):
             # Get a single pixel
             # Return a single (non-array) value
-            return self.get_values_pix(np.array([key]))[0]
+            return self.get_values_pix(np.asarray([key]))[0]
         elif isinstance(key, slice):
             # Get a slice of pixels
             start = key.start if key.start is not None else 0
@@ -1855,7 +1859,7 @@ def __setitem__(self, key, value):
         """
         if isinstance(key, numbers.Integral):
             # Set a single pixel
-            return self.update_values_pix(np.array([key]), value)
+            return self.update_values_pix(np.asarray([key]), value)
         elif isinstance(key, slice):
             # Set a slice of pixels
             start = key.start if key.start is not None else 0
@@ -2521,7 +2525,7 @@ def _apply_boolean_map_operation(self, other, name, in_place=False):
                     rhs = _PackedBoolArray.from_boolean_array(other._sparse_map[start_other: end_other])
                 elif not self._is_bit_packed and other._is_bit_packed:
                     # Expand the RHS to a regular boolean array.
-                    rhs = np.array(other._sparse_map[start_other: end_other])
+                    rhs = np.asarray(other._sparse_map[start_other: end_other])
 
                 if name == "and":
                     lhs &= rhs

healsparse/io_map_fits.py

@@ -390,7 +390,7 @@ def _read_healsparse_fits_file_and_degrade(filename, pixels, nside_out, reductio
         if cov_map_weight.nside_coverage != cov_map.nside_coverage:
             raise ValueError("The weightfile %s must have same coverage nside." % (weightfile))
         cov_pix_weight, = np.where(cov_map_weight.coverage_mask)
-        if not np.all(np.in1d(_pixels, cov_pix_weight)):
+        if not np.all(np.isin(_pixels, cov_pix_weight)):
             raise ValueError("The weightfile %s must have coverage in all the "
                              "pixels to read." % (weightfile))
         use_weightfile = True

tests/test_applymask.py

@@ -83,7 +83,7 @@ def test_apply_mask_int(self):
         # Pixels that are in the original but are not in the masked pixels should be 1
         testing.assert_array_equal(int_map.get_values_pix(valid_pixels[still_good0]), 1)
 
-    def test_apply_mask_float(self):
+    def test_apply_mask_float32(self):
         """
         Test apply_mask on a float map
         """
@@ -119,8 +119,9 @@ def test_apply_mask_float(self):
         masked_map = float_map.apply_mask(mask_map, in_place=False)
         masked_pixels = mask_map.valid_pixels
 
-        # Masked pixels should be zero
-        testing.assert_almost_equal(masked_map.get_values_pix(masked_pixels), hpg.UNSEEN)
+        # Masked pixels should be UNSEEN
+        delta = masked_map.get_values_pix(masked_pixels) - hpg.UNSEEN
+        testing.assert_array_almost_equal(delta, 0.0)
 
         # Pixels that are in the original but are not in the masked pixels should be 1
         still_good0, = np.where((float_map.get_values_pix(valid_pixels) > hpg.UNSEEN) &
@@ -134,8 +135,9 @@ def test_apply_mask_float(self):
         masked_map = float_map.apply_mask(mask_map, mask_bits=4, in_place=False)
         masked_pixels = mask_map.valid_pixels
 
-        # Masked pixels should be zero
-        testing.assert_almost_equal(masked_map.get_values_pix(masked_pixels), hpg.UNSEEN)
+        # Masked pixels should be UNSEEN
+        delta = masked_map.get_values_pix(masked_pixels) - hpg.UNSEEN
+        testing.assert_array_almost_equal(delta, 0.0)
 
         # Pixels that are in the original but are not in the masked pixels should be 1
         still_good, = np.where((float_map.get_values_pix(valid_pixels) > 0) &
@@ -157,8 +159,9 @@ def test_apply_mask_float(self):
         float_map.apply_mask(mask_map, in_place=True)
         masked_pixels = mask_map.valid_pixels
 
-        # Masked pixels should be zero
-        testing.assert_almost_equal(float_map.get_values_pix(masked_pixels), hpg.UNSEEN)
+        # Masked pixels should be UNSEEN
+        delta = float_map.get_values_pix(masked_pixels) - hpg.UNSEEN
+        testing.assert_array_almost_equal(delta, 0.0)
 
         # Pixels that are in the original but are not in the masked pixels should be 1
         testing.assert_almost_equal(float_map.get_values_pix(valid_pixels[still_good0]), 1.0)

tests/test_astype.py

@@ -62,8 +62,8 @@ def test_int_to_float(self):
         testing.assert_array_equal(sparse_map.valid_pixels, sparse_map_float.valid_pixels)
         testing.assert_array_almost_equal(sparse_map[sparse_map.valid_pixels],
                                           sparse_map_float[sparse_map.valid_pixels])
-        testing.assert_array_equal(sparse_map_float._sparse_map[0: nfine_per_cov],
-                                   hpg.UNSEEN)
+        delta = sparse_map_float._sparse_map[0: nfine_per_cov] - hpg.UNSEEN
+        testing.assert_array_almost_equal(delta, 0.0)
 
         # Convert to a float map with 0 sentinel
         sparse_map_float2 = sparse_map.astype(np.float32, sentinel=0.0)
@@ -132,8 +132,8 @@ def test_float_to_float(self):
         testing.assert_array_equal(sparse_map.valid_pixels, sparse_map_float.valid_pixels)
         testing.assert_array_almost_equal(sparse_map[sparse_map.valid_pixels],
                                           sparse_map_float[sparse_map.valid_pixels])
-        testing.assert_array_equal(sparse_map_float._sparse_map[0: nfine_per_cov],
-                                   hpg.UNSEEN)
+        delta = sparse_map_float._sparse_map[0: nfine_per_cov] - hpg.UNSEEN
+        testing.assert_array_almost_equal(delta, 0.0)
 
         # Convert to a different float map with 0 sentinel
         sparse_map_float2 = sparse_map.astype(np.float32, sentinel=0.0)

tests/test_buildmaps.py

@@ -111,8 +111,9 @@ def test_build_maps_recarray(self):
         sparse_map = healsparse.HealSparseMap.make_empty(nside_coverage, nside_map, dtype, primary='col1')
 
         # Look up all the values, make sure they're all UNSEEN
-        testing.assert_almost_equal(sparse_map.get_values_pos(ra, dec, lonlat=True)['col1'], hpg.UNSEEN)
-        testing.assert_almost_equal(sparse_map.get_values_pos(ra, dec, lonlat=True)['col2'], hpg.UNSEEN)
+        delta = sparse_map.get_values_pos(ra, dec, lonlat=True)['col1'] - hpg.UNSEEN
+        testing.assert_array_almost_equal(delta, 0.0)
+        testing.assert_array_almost_equal(sparse_map.get_values_pos(ra, dec, lonlat=True)['col2'], hpg.UNSEEN)
 
         pixel = np.arange(4000, 20000)
         values = np.zeros_like(pixel, dtype=dtype)

tests/test_cat_files.py

@@ -31,7 +31,7 @@ def test_cat_maps(self):
                 primary = None
                 wide_mask_maxbits = None
                 sentinel = hpg.UNSEEN
-                data = np.array([100.0], dtype=dtype)
+                data = np.asarray([100.0], dtype=dtype)
             elif t == 'recarray':
                 dtype = [('a', 'f4'),
                          ('b', 'i4')]
@@ -137,7 +137,7 @@ def test_cat_maps_overlap(self):
                 primary = None
                 wide_mask_maxbits = None
                 sentinel = -9999
-                data = np.array([100.0], dtype=dtype)
+                data = np.asarray([100.0], dtype=dtype)
             elif t == 'recarray':
                 dtype = [('a', 'f4'),
                          ('b', 'i4')]
@@ -187,11 +187,11 @@ def test_cat_maps_overlap(self):
                                                                     sentinel=sentinel)
             if combined_map.is_integer_map:
                 combined_map[map1.valid_pixels] = map1[map1.valid_pixels]
-                overlap = np.in1d(map2.valid_pixels, combined_map.valid_pixels)
+                overlap = np.isin(map2.valid_pixels, combined_map.valid_pixels)
                 combined_map[map2.valid_pixels[overlap]] = map2[map2.valid_pixels[overlap]] | \
                     combined_map[map2.valid_pixels[overlap]]
                 combined_map[map2.valid_pixels[~overlap]] = map2[map2.valid_pixels[~overlap]]
-                overlap = np.in1d(map3.valid_pixels, combined_map.valid_pixels)
+                overlap = np.isin(map3.valid_pixels, combined_map.valid_pixels)
                 combined_map[map3.valid_pixels[overlap]] = map3[map3.valid_pixels[overlap]] | \
                     combined_map[map3.valid_pixels[overlap]]
                 combined_map[map3.valid_pixels[~overlap]] = map3[map3.valid_pixels[~overlap]]

tests/test_degrade.py

@@ -220,7 +220,7 @@ def test_degrade_widemask_or(self):
         sparse_map_or.set_bits_pix(pixel2_all, [4, 12])
 
         # Get the pixel number of the bad pixels
-        pixel2_bad = np.array([0])
+        pixel2_bad = np.asarray([0])
         sparse_map_or.clear_bits_pix(pixel2_bad, [4])  # set low value in the first pixel
 
         # Degrade with or

tests/test_emptypixels.py

@@ -14,10 +14,10 @@ def test_emptypixels_get_values(self):
         m = healsparse.HealSparseMap.make_empty(32, 512, np.float32)
         m[1000] = 1.0
 
-        empty_arr = np.array([], dtype=m.dtype)
+        empty_arr = np.asarray([], dtype=m.dtype)
 
         testing.assert_array_equal(m.get_values_pix([]), empty_arr)
-        testing.assert_array_equal(m.get_values_pix(np.array([], dtype=np.int64)), empty_arr)
+        testing.assert_array_equal(m.get_values_pix(np.asarray([], dtype=np.int64)), empty_arr)
 
     def test_emptypixels_get_values_widemask(self):
         """
@@ -33,10 +33,10 @@ def test_emptypixels_get_by_index(self):
         m = healsparse.HealSparseMap.make_empty(32, 512, np.float64)
         m[1000] = 1.0
 
-        empty_arr = np.array([], dtype=m.dtype)
+        empty_arr = np.asarray([], dtype=m.dtype)
 
         testing.assert_array_equal(m[[]], empty_arr)
-        testing.assert_array_equal(m[np.array([], dtype=np.int64)], empty_arr)
+        testing.assert_array_equal(m[np.asarray([], dtype=np.int64)], empty_arr)
 
     def test_emptypixels_get_by_index_widemask(self):
         """
@@ -52,7 +52,7 @@ def test_emptypixels_get_by_slice(self):
         m = healsparse.HealSparseMap.make_empty(32, 512, np.float64)
         m[1000] = 1.0
 
-        empty_arr = np.array([], dtype=m.dtype)
+        empty_arr = np.asarray([], dtype=m.dtype)
 
         testing.assert_array_equal(m[0: 0], empty_arr)
 
@@ -70,23 +70,23 @@ def test_emptypixels_update_values(self):
         m = healsparse.HealSparseMap.make_empty(32, 512, np.int64)
         m[1000] = 1
 
-        m.update_values_pix([], np.array([], dtype=np.int64))
+        m.update_values_pix([], np.asarray([], dtype=np.int64))
         self.assertEqual(m[1000], 1)
         self.assertEqual(len(m.valid_pixels), 1)
 
-        m.update_values_pix(np.array([], dtype=np.int32), np.array([], dtype=np.int64))
+        m.update_values_pix(np.asarray([], dtype=np.int32), np.asarray([], dtype=np.int64))
         self.assertEqual(m[1000], 1)
         self.assertEqual(len(m.valid_pixels), 1)
 
-        m.update_values_pix([], np.array([], dtype=np.int64))
+        m.update_values_pix([], np.asarray([], dtype=np.int64))
         m.update_values_pix([], 5)
 
         self.assertWarns(UserWarning, m.update_values_pix, [], np.zeros(5, dtype=m.dtype))
 
         m.update_values_pos(
-            np.array([], dtype=np.float64),
-            np.array([], dtype=np.float64),
-            np.array([], dtype=np.int64),
+            np.asarray([], dtype=np.float64),
+            np.asarray([], dtype=np.float64),
+            np.asarray([], dtype=np.int64),
         )
 
     def test_emptypixels_set_by_index(self):
@@ -96,15 +96,15 @@ def test_emptypixels_set_by_index(self):
         m = healsparse.HealSparseMap.make_empty(32, 512, np.int32)
         m[1000] = 1
 
-        m[[]] = np.array([], dtype=np.int32)
+        m[[]] = np.asarray([], dtype=np.int32)
         self.assertEqual(m[1000], 1)
         self.assertEqual(len(m.valid_pixels), 1)
 
-        m[np.array([], dtype=np.int32)] = np.array([], dtype=np.int32)
+        m[np.asarray([], dtype=np.int32)] = np.asarray([], dtype=np.int32)
         self.assertEqual(m[1000], 1)
         self.assertEqual(len(m.valid_pixels), 1)
 
-        m[[]] = np.array([], dtype=np.int32)
+        m[[]] = np.asarray([], dtype=np.int32)
         m[[]] = 0
 
         self.assertWarns(UserWarning, m.__setitem__, [], np.zeros(5, dtype=np.int32))