From 476ebae590c75784021df9800bd05b7a467912c7 Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Tue, 3 Dec 2024 20:15:33 -0800 Subject: [PATCH] Fix indexing for arrays with non-zero base (#1824) * Fix indexing N-based system arrays * Support negative base arrays * Fix for Mono --- Src/IronPython/Runtime/Operations/ArrayOps.cs | 90 ++++++++++-- Tests/test_array.py | 138 ++++++++++++++++-- 2 files changed, 202 insertions(+), 26 deletions(-) diff --git a/Src/IronPython/Runtime/Operations/ArrayOps.cs b/Src/IronPython/Runtime/Operations/ArrayOps.cs index 87c4bcb62..60d2993b2 100644 --- a/Src/IronPython/Runtime/Operations/ArrayOps.cs +++ b/Src/IronPython/Runtime/Operations/ArrayOps.cs @@ -171,14 +171,14 @@ public static Array Multiply(Array data, int count) { if (data == null) throw PythonOps.TypeError("expected Array, got None"); if (data.Rank != 1) throw PythonOps.TypeError("bad dimensions for array, got {0} expected {1}", 1, data.Rank); - return data.GetValue(PythonOps.FixIndex(index, data.Length) + data.GetLowerBound(0)); + return data.GetValue(FixIndex(data, index, 0)); } [SpecialName] public static object GetItem(Array data, Slice slice) { if (data == null) throw PythonOps.TypeError("expected Array, got None"); - return GetSlice(data, data.Length, slice); + return GetSlice(data, slice); } [SpecialName] @@ -201,7 +201,6 @@ public static object GetItem(Array data, Slice slice) { int[] iindices = TupleToIndices(data, indices); if (data.Rank != indices.Length) throw PythonOps.TypeError("bad dimensions for array, got {0} expected {1}", indices.Length, data.Rank); - for (int i = 0; i < iindices.Length; i++) iindices[i] += data.GetLowerBound(i); return data.GetValue(iindices); } @@ -210,7 +209,7 @@ public static void SetItem(Array data, int index, object value) { if (data == null) throw PythonOps.TypeError("expected Array, got None"); if (data.Rank != 1) throw PythonOps.TypeError("bad dimensions for array, got {0} expected {1}", 1, data.Rank); - data.SetValue(Converter.Convert(value, data.GetType().GetElementType()), PythonOps.FixIndex(index, data.Length) + data.GetLowerBound(0)); + data.SetValue(Converter.Convert(value, data.GetType().GetElementType()), FixIndex(data, index, 0)); } [SpecialName] @@ -231,8 +230,6 @@ public static void SetItem(Array a, params object[] indexAndValue) { if (a.Rank != args.Length) throw PythonOps.TypeError("bad dimensions for array, got {0} expected {1}", args.Length, a.Rank); - for (int i = 0; i < indices.Length; i++) indices[i] += a.GetLowerBound(i); - Type elm = t.GetElementType()!; a.SetValue(Converter.Convert(indexAndValue[indexAndValue.Length - 1], elm), indices); } @@ -243,9 +240,15 @@ public static void SetItem(Array a, Slice index, object? value) { Type elm = a.GetType().GetElementType()!; + int lb = a.GetLowerBound(0); + if (lb != 0) { + FixSlice(index, a, out int start, out int stop, out int step); + index = new Slice(start - lb, stop - lb, step); + } + index.DoSliceAssign( delegate (int idx, object? val) { - a.SetValue(Converter.Convert(val, elm), idx + a.GetLowerBound(0)); + a.SetValue(Converter.Convert(val, elm), idx + lb); }, a.Length, value); @@ -375,10 +378,10 @@ public static string __repr__(CodeContext/*!*/ context, [NotNone] Array/*!*/ sel return GetSlice(data, start, stop, step); } - internal static Array GetSlice(Array data, int size, Slice slice) { + private static Array GetSlice(Array data, Slice slice) { if (data.Rank != 1) throw PythonOps.NotImplementedError("slice on multi-dimensional array"); - slice.Indices(size, out int start, out int stop, out int step); + FixSlice(slice, data, out int start, out int stop, out int step); if ((step > 0 && start >= stop) || (step < 0 && start <= stop)) { if (data.GetType().GetElementType() == typeof(object)) @@ -387,17 +390,17 @@ internal static Array GetSlice(Array data, int size, Slice slice) { return Array.CreateInstance(data.GetType().GetElementType()!, 0); } - if (step == 1) { + if (step == 1 && (!ClrModule.IsMono || data.GetLowerBound(0) == 0)) { int n = stop - start; Array ret = Array.CreateInstance(data.GetType().GetElementType()!, n); - Array.Copy(data, start + data.GetLowerBound(0), ret, 0, n); + Array.Copy(data, start, ret, 0, n); // doesn't work OK on Mono with non-0-based arrays return ret; } else { int n = PythonOps.GetSliceCount(start, stop, step); Array ret = Array.CreateInstance(data.GetType().GetElementType()!, n); int ri = 0; for (int i = 0, index = start; i < n; i++, index += step) { - ret.SetValue(data.GetValue(index + data.GetLowerBound(0)), ri++); + ret.SetValue(data.GetValue(index), ri++); } return ret; } @@ -427,11 +430,72 @@ private static int[] TupleToIndices(Array a, IList tuple) { int[] indices = new int[tuple.Count]; for (int i = 0; i < indices.Length; i++) { int iindex = Converter.ConvertToInt32(tuple[i]); - indices[i] = i < a.Rank ? PythonOps.FixIndex(iindex, a.GetLength(i)) : int.MinValue; + indices[i] = i < a.Rank ? FixIndex(a, iindex, i) : int.MinValue; } return indices; } + private static int FixIndex(Array a, int v, int axis) { + int idx = v; + int lb = a.GetLowerBound(axis); + int ub = a.GetUpperBound(axis); + if (idx < 0 && lb >= 0) { + idx += ub + 1; + } + if (idx < lb || idx > ub) { + throw PythonOps.IndexError("index out of range: {0}", v); + } + return idx; + } + + private static void FixSlice(Slice slice, Array a, out int ostart, out int ostop, out int ostep) { + Debug.Assert(a.Rank == 1); + + if (slice.step == null) { + ostep = 1; + } else { + ostep = Converter.ConvertToIndex(slice.step); + if (ostep == 0) { + throw PythonOps.ValueError("step cannot be zero"); + } + } + + int lb = a.GetLowerBound(0); + int ub = a.GetUpperBound(0); + + if (slice.start == null) { + ostart = ostep > 0 ? lb : ub; + } else { + ostart = Converter.ConvertToIndex(slice.start); + if (ostart < lb) { + if (ostart < 0 && lb >= 0) { + ostart += ub + 1; + } + if (ostart < lb) { + ostart = ostep > 0 ? lb : lb - 1; + } + } else if (ostart > ub) { + ostart = ostep > 0 ? ub + 1 : ub; + } + } + + if (slice.stop == null) { + ostop = ostep > 0 ? ub + 1 : lb - 1; + } else { + ostop = Converter.ConvertToIndex(slice.stop); + if (ostop < lb) { + if (ostop < 0 && lb >= 0) { + ostop += ub + 1; + } + if (ostop < 0) { + ostop = ostep > 0 ? lb : lb - 1; + } + } else if (ostop > ub) { + ostop = ostep > 0 ? ub + 1 : ub; + } + } + } + #endregion } } diff --git a/Tests/test_array.py b/Tests/test_array.py index 6287ff46a..97d0bc902 100644 --- a/Tests/test_array.py +++ b/Tests/test_array.py @@ -6,6 +6,26 @@ ## Test array support by IronPython (System.Array) ## +""" +Indexing of CLI arrays in IronPython: + + +| Base | Index >= 0 | Index < 0 | +|------|--------------------------------------|-------------------| +| > 0 | absolue | relative from end | +| 0 | absolute == relative from beginning | relative from end | +| < 0 | absolute | absolute | + +Comparison to indexing in C# and CPython: + +* Index >= 0, any base is C# compliant. +* Base 0, any index is CPython compliant. +* Base 0, index < 0 is not supported by C# but can be achieved by `System.Index` with 1-dim arrays only; then IronPython indexing is C# compliant. +* Base > 0, index < 0 is not supported by C#; IronPython follows CPython convention as more practical. +* Base < 0, index < 0 is C# compliant. +* Base != 0 is not supported by CPython for any builtin structures. +""" + from iptest import IronPythonTestCase, is_cli, run_test, skipUnlessIronPython if is_cli: @@ -146,6 +166,13 @@ def test_slice(self): def f(): array1[::2] = [x * 2 for x in range(11)] self.assertRaises(ValueError, f) + # slices on non-1-dim arrays are not supported + array2 = System.Array.CreateInstance(int, 20, 20) + self.assertRaises(NotImplementedError, lambda: array2[:]) # TODO: TypeError? + self.assertRaises(TypeError, lambda: array2[:, :]) # TODO: NotImplementedError? This would work in Numpy and Sympy + self.assertRaises(TypeError, lambda: array2[:, :, :]) # OK + + def test_creation(self): t = System.Array ti = type(System.Array.CreateInstance(int, 1)) @@ -173,35 +200,55 @@ def test_constructor(self): def test_nonzero_lowerbound(self): a = System.Array.CreateInstance(int, (5,), (5,)) - for i in range(5): a[i] = i + for i in range(5, 5 + a.Length): a[i] = i - self.assertEqual(a[:2], System.Array[int]((0,1))) - self.assertEqual(a[2:], System.Array[int]((2,3,4))) - self.assertEqual(a[2:4], System.Array[int]((2,3))) - self.assertEqual(a[-1], 4) + self.assertEqual(a[:7], System.Array[int]((5,6))) + self.assertEqual(a[7:], System.Array[int]((7,8,9))) + self.assertEqual(a[7:9], System.Array[int]((7,8))) + self.assertEqual(a[-1:-3:-1], System.Array[int]((9,8))) + self.assertEqual(a[-1], 9) - self.assertEqual(repr(a), 'Array[int]((0, 1, 2, 3, 4), base: 5)') + self.assertEqual(repr(a), 'Array[int]((5, 6, 7, 8, 9), base: 5)') a = System.Array.CreateInstance(int, (5,), (15,)) b = System.Array.CreateInstance(int, (5,), (20,)) self.assertEqual(a.Length, b.Length) for i in range(a.Length): - self.assertEqual(a[i], b[i]) + self.assertEqual(a[i + 15], b[i + 20]) + + a0 = System.Array.CreateInstance(int, 5) # regular, 0-based + for i in range(5): a0[i] = i - ## 5-dimension + a[17:19] = a0[2:4] + self.assertEqual(a[17:19], System.Array[int]((2, 3))) + + self.assertEqual(a0[3:1:-1], System.Array[int]((3, 2))) + self.assertEqual(a[18:16:-1], System.Array[int]((3, 2))) + + self.assertEqual(a0[-3:-1], System.Array[int]((2, 3))) + self.assertEqual(a[-3:-1], System.Array[int]((2, 3))) + + a[18:16:-1] = a0[2:4] + self.assertEqual(a[17:19], System.Array[int]((3, 2))) + + ## 5-dimension, 2-length/dim, progressive lowerbound a = System.Array.CreateInstance(int, (2,2,2,2,2), (1,2,3,4,5)) - self.assertEqual(a[0,0,0,0,0], 0) + self.assertEqual(a[1,2,3,4,5], 0) + + ## 5-dimension, 2-length/dim, progressive lowerbound + a = System.Array.CreateInstance(int, (2,2,2,2,2), (1,2,3,4,5)) + self.assertEqual(a[1,2,3,4,5], 0) for i in range(5): - index = [0,0,0,0,0] - index[i] = 1 + index = [1,2,3,4,5] + index[i] += 1 a[index[0], index[1], index[2], index[3], index[4]] = i self.assertEqual(a[index[0], index[1], index[2], index[3], index[4]], i) for i in range(5): - index = [0,0,0,0,0] - index[i] = 0 + index = [2,3,4,5,6] + index[i] -= 1 a[index[0], index[1], index[2], index[3], index[4]] = i self.assertEqual(a[index[0], index[1], index[2], index[3], index[4]], i) @@ -218,6 +265,71 @@ def sliceArrayAssign(arr, index, val): self.assertRaises(NotImplementedError, sliceArrayAssign, a, -200, 1) self.assertRaises(NotImplementedError, sliceArrayAssign, a, 1, 1) + def test_base1(self): + # For positive base arrays, indices are indexing elements directly (in absolute terms) + # rather than relative form the base. + # Negative indices are indexing relative form the end. + + # 1-based 2x2 matrix + arr = System.Array.CreateInstance(str, (2,2), (1,1)) + + self.assertEqual(arr.GetLowerBound(0), 1) + self.assertEqual(arr.GetLowerBound(1), 1) + self.assertEqual(arr.GetUpperBound(0), 2) + self.assertEqual(arr.GetUpperBound(1), 2) + + arr.SetValue("a_1,1", System.Array[System.Int32]((1,1))) + arr.SetValue("a_1,2", System.Array[System.Int32]((1,2))) + arr.SetValue("a_2,1", System.Array[System.Int32]((2,1))) + arr.SetValue("a_2,2", System.Array[System.Int32]((2,2))) + + self.assertEqual(arr[1, 1], "a_1,1") + self.assertEqual(arr[1, 2], "a_1,2") + self.assertEqual(arr[2, 1], "a_2,1") + self.assertEqual(arr[2, 2], "a_2,2") + + arr[1, 1] = "b_1,1" + arr[1, 2] = "b_1,2" + arr[2, 1] = "b_2,1" + arr[2, 2] = "b_2,2" + + self.assertEqual(arr.GetValue(System.Array[System.Int32]((1,1))), "b_1,1") + self.assertEqual(arr.GetValue(System.Array[System.Int32]((1,2))), "b_1,2") + self.assertEqual(arr.GetValue(System.Array[System.Int32]((2,1))), "b_2,1") + self.assertEqual(arr.GetValue(System.Array[System.Int32]((2,2))), "b_2,2") + + def test_base_negative(self): + # For negative base arrays, negative indices are indexing elements directly (like non negative indices) + # rather than indexing relative from the end. + + # 2-dim array [-1, 0, 1] x [-1, 0, 1] + arr = System.Array.CreateInstance(str, (3,3), (-1,-1)) + for i in range(-1, 2): + for j in range(-1, 2): + arr[i, j] = "a_%d,%d" % (i, j) + + for i in range(-1, 2): + for j in range(-1, 2): + self.assertEqual(arr[i, j], "a_%d,%d" % (i, j)) + + # test that VauleError is raised when the index is out of range + self.assertRaises(IndexError, lambda: arr[-2, 0]) + self.assertRaises(IndexError, lambda: arr[2, 0]) + self.assertRaises(IndexError, lambda: arr[0, -2]) + self.assertRaises(IndexError, lambda: arr[0, 2]) + + # test slice indexing + # 1-dim array [-1, 0, 1] + arr1 = System.Array.CreateInstance(int, (3,), (-1,)) + for i in range(-1, 2): + arr1[i] = i + self.assertEqual(arr1[-1:1], System.Array[int]((-1, 0))) + self.assertEqual(arr1[-2:1], System.Array[int]((-1, 0))) + self.assertEqual(arr1[0:], System.Array[int]((0, 1))) + self.assertEqual(arr1[:1], System.Array[int]((-1, 0))) + self.assertEqual(arr1[:], System.Array[int]((-1, 0, 1))) + self.assertEqual(arr1[:-2], System.Array[int](0)) + def test_array_type(self): def type_helper(array_type, instance):