diff --git a/Src/IronPython.Modules/IterTools.cs b/Src/IronPython.Modules/IterTools.cs index 5de12db3d..e4c0db29c 100644 --- a/Src/IronPython.Modules/IterTools.cs +++ b/Src/IronPython.Modules/IterTools.cs @@ -168,9 +168,23 @@ public PythonTuple __reduce__() { } public void __setstate__(PythonTuple state) { - // TODO: error handling? - ie = state[0] as IEnumerator; - inner = (state.Count > 1) ? state[1] as IEnumerator : null; + IEnumerator iter; + IEnumerator innerIter; + switch (state.Count) { + case 0: throw PythonOps.TypeError("function takes at least 1 argument (0 given)"); + case 1: + iter = state[0] as IEnumerator ?? throw PythonOps.TypeError("Arguments must be iterators."); + innerIter = null; + break; + case 2: + iter = state[0] as IEnumerator ?? throw PythonOps.TypeError("Arguments must be iterators."); + innerIter = state[1] as IEnumerator ?? throw PythonOps.TypeError("Arguments must be iterators."); + break; + default: + throw PythonOps.TypeError("function takes at most 2 argument ({0} given)", state.Count); + } + ie = iter; + inner = innerIter; InnerEnumerator = LazyYielder(); } @@ -515,11 +529,14 @@ public void __setstate__(object state) { private IEnumerator Yielder(IEnumerator iter) { object curKey = _starterKey; if (MoveNextHelper(iter)) { + curKey = GetKey(iter.Current); + yield return PythonTuple.MakeTuple(curKey, Grouper(iter, curKey)); + while (!_fFinished) { while (PythonContext.Equal(GetKey(iter.Current), curKey)) { - if (!MoveNextHelper(iter)) { - _fFinished = true; - yield break; + if (!MoveNextHelper(iter)) { + _fFinished = true; + yield break; } } curKey = GetKey(iter.Current); diff --git a/Src/IronPython.Modules/_codecs.cs b/Src/IronPython.Modules/_codecs.cs index 277b74160..1bab21f30 100644 --- a/Src/IronPython.Modules/_codecs.cs +++ b/Src/IronPython.Modules/_codecs.cs @@ -227,6 +227,20 @@ public static PythonTuple mbcs_encode(CodeContext/*!*/ context, [NotNone] string #endregion + #region OEM Functions + + [SupportedOSPlatform("windows"), PythonHidden(PlatformsAttribute.PlatformFamily.Unix)] + public static PythonTuple oem_decode(CodeContext/*!*/ context, [NotNone] IBufferProtocol input, string? errors = null, bool final = false) { + using IPythonBuffer buffer = input.GetBuffer(); + return DoDecode(context, "oem", StringOps.CodecsInfo.OemEncoding, buffer, errors).ToPythonTuple(); + } + + [SupportedOSPlatform("windows"), PythonHidden(PlatformsAttribute.PlatformFamily.Unix)] + public static PythonTuple oem_encode(CodeContext/*!*/ context, [NotNone] string input, string? errors = null) + => DoEncode(context, "oem", StringOps.CodecsInfo.OemEncoding, input, errors).ToPythonTuple(); + + #endregion + #region Code Page Functions [SupportedOSPlatform("windows"), PythonHidden(PlatformsAttribute.PlatformFamily.Unix)] diff --git a/Src/IronPython.Modules/_collections.cs b/Src/IronPython.Modules/_collections.cs index f0f838ba4..77067c319 100644 --- a/Src/IronPython.Modules/_collections.cs +++ b/Src/IronPython.Modules/_collections.cs @@ -551,20 +551,13 @@ public void __delitem__(CodeContext/*!*/ context, object index) { } } - public PythonTuple __reduce__() { - lock (_lockObj) { - object[] items = new object[_itemCnt]; - int curItem = 0; - WalkDeque(delegate(int curIndex) { - items[curItem++] = _data[curIndex]; - return true; - }); - - return PythonTuple.MakeTuple( - DynamicHelpers.GetPythonType(this), - PythonTuple.MakeTuple(PythonList.FromArrayNoCopy(items)) - ); - } + public PythonTuple __reduce__(CodeContext context) { + return PythonTuple.MakeTuple( + DynamicHelpers.GetPythonType(this), + _maxLen == -1 ? PythonTuple.EMPTY : PythonTuple.MakeTuple(PythonTuple.EMPTY, maxlen), + GetType() == typeof(deque) ? null : PythonOps.GetBoundAttr(context, this, "__dict__"), + PythonOps.GetEnumeratorObject(context, this) + ); } public int __len__() { diff --git a/Src/IronPython.Modules/_csv.cs b/Src/IronPython.Modules/_csv.cs index 9623552b7..6d6c86ba6 100644 --- a/Src/IronPython.Modules/_csv.cs +++ b/Src/IronPython.Modules/_csv.cs @@ -259,6 +259,7 @@ public class Dialect { private Dialect() { } + [PythonHidden] public static Dialect Create(CodeContext/*!*/ context, [ParamDictionary] IDictionary kwArgs, params object[] args) { @@ -410,6 +411,7 @@ private static string SetString(string name, object src, bool found, string @def } return result; } + #endregion public Dialect(CodeContext/*!*/ context, @@ -503,6 +505,13 @@ public Dialect(CodeContext/*!*/ context, throw PythonOps.TypeError("lineterminator must be set"); } + // CPython defines these overloads on Dialect since 3.10 + [Documentation("raises an exception to avoid pickling")] + public object __reduce__(params object[] args) => throw PythonOps.TypeError("cannot pickle 'Dialect' instances"); + + [Documentation("raises an exception to avoid pickling")] + public object __reduce_ex__(params object[] args) => throw PythonOps.TypeError("cannot pickle 'Dialect' instances"); + public string escapechar { get { return _escapechar; } } diff --git a/Src/IronPython.Modules/_ctypes/CFuncPtrType.cs b/Src/IronPython.Modules/_ctypes/CFuncPtrType.cs index 4e0d81143..bb97ddc34 100644 --- a/Src/IronPython.Modules/_ctypes/CFuncPtrType.cs +++ b/Src/IronPython.Modules/_ctypes/CFuncPtrType.cs @@ -69,6 +69,12 @@ internal static PythonType MakeSystemType(Type underlyingSystemType) { return PythonType.SetPythonType(underlyingSystemType, new CFuncPtrType(underlyingSystemType)); } + public object from_buffer(object obj) + => throw PythonOps.TypeError("abstract class"); + + public object from_buffer_copy(object obj) + => throw PythonOps.TypeError("abstract class"); + /// /// Converts an object into a function call parameter. /// diff --git a/Src/IronPython.Modules/_ctypes/NativeArgument.cs b/Src/IronPython.Modules/_ctypes/NativeArgument.cs index 880d097d2..062819b77 100644 --- a/Src/IronPython.Modules/_ctypes/NativeArgument.cs +++ b/Src/IronPython.Modules/_ctypes/NativeArgument.cs @@ -4,12 +4,9 @@ #if FEATURE_CTYPES -using System; - -using Microsoft.Scripting.Runtime; - using IronPython.Runtime; using IronPython.Runtime.Types; +using IronPython.Runtime.Operations; namespace IronPython.Modules { /// @@ -35,11 +32,8 @@ public CData _obj { #region ICodeFormattable Members - public string __repr__(CodeContext context) { - return String.Format("", - _type, - IdDispenser.GetId(__obj));// TODO: should be a real address - } + public string __repr__(CodeContext context) + => $""; #endregion } diff --git a/Src/IronPython.Modules/_ctypes/PointerType.cs b/Src/IronPython.Modules/_ctypes/PointerType.cs index 197863398..dba2709ae 100644 --- a/Src/IronPython.Modules/_ctypes/PointerType.cs +++ b/Src/IronPython.Modules/_ctypes/PointerType.cs @@ -47,6 +47,12 @@ private PointerType(Type underlyingSystemType) : base(underlyingSystemType) { } + public object from_buffer(object obj) + => throw PythonOps.TypeError("abstract class"); + + public object from_buffer_copy(object obj) + => throw PythonOps.TypeError("abstract class"); + public object from_param([NotNone] CData obj) { return new NativeArgument((CData)PythonCalls.Call(this, obj), "P"); } diff --git a/Src/IronPython.Modules/_ctypes/_ctypes.cs b/Src/IronPython.Modules/_ctypes/_ctypes.cs index a1f5807e7..47f4283ad 100644 --- a/Src/IronPython.Modules/_ctypes/_ctypes.cs +++ b/Src/IronPython.Modules/_ctypes/_ctypes.cs @@ -101,27 +101,21 @@ private static IntPtr Cast(IntPtr data, IntPtr obj, IntPtr type) { GCHandle typeHandle = GCHandle.FromIntPtr(type); try { CData cdata = objHandle.Target as CData; - PythonType pt = (PythonType)typeHandle.Target; + PythonType pt = typeHandle.Target as PythonType; - CData res = (CData)pt.CreateInstance(pt.Context.SharedContext); - if (IsPointer(pt)) { - res.MemHolder = new MemoryHolder(IntPtr.Size); - if (IsPointer(DynamicHelpers.GetPythonType(cdata))) { - res.MemHolder.WriteIntPtr(0, cdata.MemHolder.ReadIntPtr(0)); - } else { - res.MemHolder.WriteIntPtr(0, data); - } + if (!IsPointer(pt)) throw PythonOps.TypeError("cast() argument 2 must be a pointer type, not {0}", PythonOps.GetPythonTypeName(typeHandle.Target)); - if (cdata != null) { - res.MemHolder.Objects = cdata.MemHolder.Objects; - res.MemHolder.AddObject(IdDispenser.GetId(cdata), cdata); - } + CData res = (CData)pt.CreateInstance(pt.Context.SharedContext); + res.MemHolder = new MemoryHolder(IntPtr.Size); + if (IsPointer(DynamicHelpers.GetPythonType(cdata))) { + res.MemHolder.WriteIntPtr(0, cdata.MemHolder.ReadIntPtr(0)); } else { - if (cdata != null) { - res.MemHolder = new MemoryHolder(data, ((INativeType)pt).Size, cdata.MemHolder); - } else { - res.MemHolder = new MemoryHolder(data, ((INativeType)pt).Size); - } + res.MemHolder.WriteIntPtr(0, data); + } + + if (cdata != null) { + res.MemHolder.Objects = cdata.MemHolder.Objects; + res.MemHolder.AddObject(IdDispenser.GetId(cdata), cdata); } return GCHandle.ToIntPtr(GCHandle.Alloc(res)); diff --git a/Src/IronPython.Modules/_functools.cs b/Src/IronPython.Modules/_functools.cs index 6bd2d9dd9..fe2ba1fc0 100644 --- a/Src/IronPython.Modules/_functools.cs +++ b/Src/IronPython.Modules/_functools.cs @@ -193,21 +193,33 @@ public void __setstate__(CodeContext context, [NotNone] PythonTuple state) { } public string __repr__(CodeContext context) { - var builder = new StringBuilder(); - builder.Append("functools.partial("); - builder.Append(PythonOps.Repr(context, func)); - foreach (var x in _args) { - builder.Append(", "); - builder.Append(PythonOps.Repr(context, x)); + var infinite = PythonOps.GetAndCheckInfinite(this); + if (infinite == null) { + return "..."; } - foreach (var p in _keywordArgs) { - builder.Append(", "); - builder.Append(p.Key); - builder.Append('='); - builder.Append(PythonOps.Repr(context, p.Value)); + + int infiniteIndex = infinite.Count; + infinite.Add(this); + try { + var builder = new StringBuilder(); + builder.Append("functools.partial("); + builder.Append(PythonOps.Repr(context, func)); + foreach (var x in _args) { + builder.Append(", "); + builder.Append(PythonOps.Repr(context, x)); + } + foreach (var p in _keywordArgs) { + builder.Append(", "); + builder.Append(p.Key); + builder.Append('='); + builder.Append(PythonOps.Repr(context, p.Value)); + } + builder.Append(')'); + return builder.ToString(); + } finally { + System.Diagnostics.Debug.Assert(infiniteIndex == infinite.Count - 1); + infinite.RemoveAt(infiniteIndex); } - builder.Append(')'); - return builder.ToString(); } #endregion diff --git a/Src/IronPython.Modules/_heapq.cs b/Src/IronPython.Modules/_heapq.cs index a9c1f3b68..989874216 100644 --- a/Src/IronPython.Modules/_heapq.cs +++ b/Src/IronPython.Modules/_heapq.cs @@ -76,6 +76,7 @@ public static object heapreplace(CodeContext/*!*/ context, PythonList list, obje } } + // TODO: removed in Python 3.5 [Documentation("Find the n largest elements in a dataset.\n\n" + "Equivalent to: sorted(iterable, reverse=True)[:n]\n" )] @@ -108,6 +109,7 @@ public static PythonList nlargest(CodeContext/*!*/ context, int n, object iterab return ret; } + // TODO: removed in Python 3.5 [Documentation("Find the n smallest elements in a dataset.\n\n" + "Equivalent to: sorted(iterable)[:n]\n" )] diff --git a/Src/IronPython.Modules/_operator.cs b/Src/IronPython.Modules/_operator.cs index 7dd04872a..c10432010 100644 --- a/Src/IronPython.Modules/_operator.cs +++ b/Src/IronPython.Modules/_operator.cs @@ -23,6 +23,7 @@ namespace IronPython.Modules { public static class PythonOperator { public const string __doc__ = "Provides programmatic access to various operators (addition, accessing members, etc...)"; + [PythonType] public sealed class attrgetter : ICodeFormattable { private readonly object[] _names; @@ -40,31 +41,32 @@ public attrgetter([NotNone] params object[] attrs) { public PythonTuple __reduce__() => PythonTuple.MakeTuple(DynamicHelpers.GetPythonType(this), PythonTuple.MakeTuple(_names)); [SpecialName] - public object Call(CodeContext context, object param) { + public object? Call(CodeContext context, object? param) { if (_names.Length == 1) { return GetOneAttr(context, param, _names[0]); } - object[] res = new object[_names.Length]; + object?[] res = new object[_names.Length]; for (int i = 0; i < _names.Length; i++) { res[i] = GetOneAttr(context, param, _names[i]); } return PythonTuple.MakeTuple(res); } - private static object GetOneAttr(CodeContext context, object param, object val) { + private static object? GetOneAttr(CodeContext context, object? param, object val) { if (val is not string s) { throw PythonOps.TypeError("attribute name must be string"); } int dotPos = s.IndexOf('.'); if (dotPos >= 0) { - object nextParam = GetOneAttr(context, param, s.Substring(0, dotPos)); + object? nextParam = GetOneAttr(context, param, s.Substring(0, dotPos)); return GetOneAttr(context, nextParam, s.Substring(dotPos + 1)); } return PythonOps.GetBoundAttr(context, param, s); } } + [PythonType] public sealed class itemgetter : ICodeFormattable { private readonly object?[] _items; @@ -82,7 +84,7 @@ public itemgetter([NotNone] params object?[] items) { public PythonTuple __reduce__() => PythonTuple.MakeTuple(DynamicHelpers.GetPythonType(this), PythonTuple.MakeTuple(_items)); [SpecialName] - public object Call(CodeContext/*!*/ context, object param) { + public object? Call(CodeContext/*!*/ context, object? param) { if (_items.Length == 1) { return PythonOps.GetIndex(context, param, _items[0]); } diff --git a/Src/IronPython.Modules/_socket.cs b/Src/IronPython.Modules/_socket.cs index 9587d6864..52bedab8c 100644 --- a/Src/IronPython.Modules/_socket.cs +++ b/Src/IronPython.Modules/_socket.cs @@ -136,14 +136,27 @@ public void __init__(CodeContext/*!*/ context, int family = DefaultAddressFamily throw MakeException(context, new SocketException((int)SocketError.ProtocolNotSupported)); } - Socket? socket; + Socket? socket = null; if (fileno is socket sock) { socket = sock._socket; _hostName = sock._hostName; // we now own the lifetime of the socket GC.SuppressFinalize(sock); - } else if (fileno != null && (socket = HandleToSocket((long)fileno)) != null) { - // nothing to do here + } else if (fileno != null) { + if (!PythonOps.TryToIndex(fileno, out object? handleObj)) { + throw PythonOps.TypeErrorForUnIndexableObject(fileno); + } + long handle = Converter.ConvertToInt64(handleObj); + // Windows reserves only INVALID_SOCKET (~0) as an invalid handle + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? handle == -1 : handle < 0) { + throw PythonOps.ValueError("negative file descriptor"); + } + socket = HandleToSocket(handle); + if (socket is null) { + throw PythonOps.OSError(RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? PythonErrorNumber.WSAENOTSOCK : PythonErrorNumber.EBADF, + "Bad file descriptor"); + } } else { try { socket = new Socket(addressFamily, socketType, protocolType); @@ -413,7 +426,7 @@ public Bytes recv(int bufsize, int flags = 0) { try { bytesRead = _socket.Receive(buffer, bufsize, (SocketFlags)flags); } catch (Exception e) { - throw MakeRecvException(e, SocketError.NotConnected); + throw MakeException(_context, e); } var bytes = new byte[bytesRead]; @@ -449,7 +462,7 @@ public int recv_into([NotNone] IBufferProtocol buffer, int nbytes = 0, int flags try { bytesRead = _socket.Receive(byteBuffer, nbytes, (SocketFlags)flags); } catch (Exception e) { - throw MakeRecvException(e, SocketError.NotConnected); + throw MakeException(_context, e); } byteBuffer.AsSpan(0, bytesRead).CopyTo(span); @@ -486,7 +499,7 @@ public PythonTuple recvfrom(int bufsize, int flags = 0) { try { bytesRead = _socket.ReceiveFrom(buffer, bufsize, (SocketFlags)flags, ref remoteEP); } catch (Exception e) { - throw MakeRecvException(e, SocketError.InvalidArgument); + throw MakeException(_context, e); } var bytes = new byte[bytesRead]; @@ -519,7 +532,7 @@ public PythonTuple recvfrom_into([NotNone] IBufferProtocol buffer, int nbytes = try { bytesRead = _socket.ReceiveFrom(byteBuffer, nbytes, (SocketFlags)flags, ref remoteEP); } catch (Exception e) { - throw MakeRecvException(e, SocketError.InvalidArgument); + throw MakeException(_context, e); } byteBuffer.AsSpan(0, bytesRead).CopyTo(span); @@ -551,18 +564,6 @@ private static int byteBufferSize(string funcName, int nbytes, int bufLength, in } } - private Exception MakeRecvException(Exception e, SocketError errorCode = SocketError.InvalidArgument) { - if (e is ObjectDisposedException) return MakeException(_context, e); - - // on the socket recv throw a special socket error code when SendTimeout is zero - if (_socket.SendTimeout == 0) { - var s = new SocketException((int)errorCode); - return PythonExceptions.CreateThrowable(error, s.ErrorCode, s.Message); - } else { - return MakeException(_context, e); - } - } - [Documentation("send(string[, flags]) -> bytes_sent\n\n" + "Send data to the remote socket. The socket must be connected to a remote\n" + "socket (by calling either connect() or accept(). Returns the number of bytes\n" @@ -1787,7 +1788,7 @@ internal static Exception MakeException(CodeContext/*!*/ context, Exception exce } } else if (exception is ObjectDisposedException) { return PythonExceptions.CreateThrowable(error, PythonErrorNumber.EBADF, "Socket is closed"); - } else if (exception is InvalidOperationException) { + } else if (exception is InvalidOperationException or ArgumentException) { return MakeException(context, new SocketException((int)SocketError.InvalidArgument)); } else { return exception; diff --git a/Src/IronPython.Modules/_struct.cs b/Src/IronPython.Modules/_struct.cs index ba89527d7..d7b547704 100644 --- a/Src/IronPython.Modules/_struct.cs +++ b/Src/IronPython.Modules/_struct.cs @@ -192,9 +192,22 @@ public void __init__(CodeContext/*!*/ context, object fmt) { break; case FormatType.Float: for (int j = 0; j < curFormat.Count; j++) { - WriteFloat(res, _isLittleEndian, (float)GetDoubleValue(context, curObj++, values)); + var d = GetDoubleValue(context, curObj++, values); + var val = (float)d; + if (float.IsInfinity(val) && !double.IsInfinity(d)) throw PythonOps.OverflowError("float too large to pack with f format"); + WriteFloat(res, _isLittleEndian, val); } break; +#if NET6_0_OR_GREATER + case FormatType.Half: + for (int j = 0; j < curFormat.Count; j++) { + var d = GetDoubleValue(context, curObj++, values); + var val = (Half)d; + if (Half.IsInfinity(val) && !double.IsInfinity(d)) throw PythonOps.OverflowError("float too large to pack with e format"); + WriteHalf(res, _isLittleEndian, val); + } + break; +#endif case FormatType.CString: WriteString(res, curFormat.Count, GetStringValue(context, curObj++, values)); break; @@ -273,7 +286,7 @@ public void pack_into(CodeContext/*!*/ context, [NotNone] IBufferProtocol/*!*/ b break; case FormatType.SignedChar: for (int j = 0; j < curFormat.Count; j++) { - res[res_idx++] = (int)(sbyte)CreateCharValue(context, ref curIndex, data); + res[res_idx++] = (int)unchecked((sbyte)CreateCharValue(context, ref curIndex, data)); } break; case FormatType.UnsignedChar: @@ -336,7 +349,7 @@ public void pack_into(CodeContext/*!*/ context, [NotNone] IBufferProtocol/*!*/ b break; case FormatType.SizeT: for (int j = 0; j < curFormat.Count; j++) { - res[res_idx++] = CreateUIntValue(context, ref curIndex, _isLittleEndian, data); + res[res_idx++] = BigIntegerOps.__int__(CreateUIntValue(context, ref curIndex, _isLittleEndian, data)); } break; case FormatType.LongLong: @@ -349,9 +362,16 @@ public void pack_into(CodeContext/*!*/ context, [NotNone] IBufferProtocol/*!*/ b res[res_idx++] = BigIntegerOps.__int__(CreateULongValue(context, ref curIndex, _isLittleEndian, data)); } break; +#if NET6_0_OR_GREATER + case FormatType.Half: + for (int j = 0; j < curFormat.Count; j++) { + res[res_idx++] = (double)CreateHalfValue(context, ref curIndex, _isLittleEndian, data); + } + break; +#endif case FormatType.Float: for (int j = 0; j < curFormat.Count; j++) { - res[res_idx++] = CreateFloatValue(context, ref curIndex, _isLittleEndian, data); + res[res_idx++] = (double)CreateFloatValue(context, ref curIndex, _isLittleEndian, data); } break; case FormatType.Double: @@ -472,6 +492,12 @@ private static Struct CompileAndCache(CodeContext/*!*/ context, string/*!*/ fmt) res.Add(new Format(FormatType.UnsignedLongLong, count)); count = 1; break; +#if NET6_0_OR_GREATER + case 'e': // half + res.Add(new Format(FormatType.Half, count)); + count = 1; + break; +#endif case 'f': // float res.Add(new Format(FormatType.Float, count)); count = 1; @@ -697,6 +723,9 @@ private enum FormatType { Short, UnsignedShort, +#if NET6_0_OR_GREATER + Half, +#endif Int, UnsignedInt, @@ -730,6 +759,9 @@ private static int GetNativeSize(FormatType c) { return 1; case FormatType.Short: case FormatType.UnsignedShort: +#if NET6_0_OR_GREATER + case FormatType.Half: +#endif return 2; case FormatType.Int: case FormatType.UnsignedInt: @@ -911,6 +943,18 @@ private static void WriteSignedNetPointer(this MemoryStream res, bool fLittleEnd res.WritePointer(fLittleEndian, unchecked((ulong)val.ToInt64())); } +#if NET6_0_OR_GREATER + private static void WriteHalf(this MemoryStream res, bool fLittleEndian, Half val) { + byte[] bytes = BitConverter.GetBytes(val); + if (BitConverter.IsLittleEndian == fLittleEndian) { + res.Write(bytes, 0, bytes.Length); + } else { + res.WriteByte(bytes[1]); + res.WriteByte(bytes[0]); + } + } +#endif + private static void WriteFloat(this MemoryStream res, bool fLittleEndian, float val) { byte[] bytes = BitConverter.GetBytes(val); if (BitConverter.IsLittleEndian == fLittleEndian) { @@ -1134,7 +1178,7 @@ internal static ulong GetULongLongValue(CodeContext/*!*/ context, int index, obj internal static double GetDoubleValue(CodeContext/*!*/ context, int index, object[] args) { object val = GetValue(context, index, args); if (Converter.TryConvertToDouble(val, out double res)) return res; - throw Error(context, "expected double value"); + throw Error(context, "required argument is not a float"); } internal static IList GetStringValue(CodeContext/*!*/ context, int index, object[] args) { @@ -1194,6 +1238,28 @@ internal static ushort CreateUShortValue(CodeContext/*!*/ context, ref int index } } +#if NET6_0_OR_GREATER + internal static Half CreateHalfValue(CodeContext/*!*/ context, ref int index, bool fLittleEndian, IList data) { + byte[] bytes = new byte[2]; + if (fLittleEndian == BitConverter.IsLittleEndian) { + bytes[0] = data[index++]; + bytes[1] = data[index++]; + } else { + bytes[1] = data[index++]; + bytes[0] = data[index++]; + } + Half res = BitConverter.ToHalf(bytes, 0); + + if (context.LanguageContext.FloatFormat == FloatFormat.Unknown) { + if (Half.IsNaN(res) || Half.IsInfinity(res)) { + throw PythonOps.ValueError("can't unpack IEEE 754 special value on non-IEEE platform"); + } + } + + return res; + } +#endif + internal static float CreateFloatValue(CodeContext/*!*/ context, ref int index, bool fLittleEndian, IList data) { byte[] bytes = new byte[4]; if (fLittleEndian == BitConverter.IsLittleEndian) { diff --git a/Src/IronPython.Modules/array.cs b/Src/IronPython.Modules/array.cs index 578ec8d2f..2bee03a5f 100644 --- a/Src/IronPython.Modules/array.cs +++ b/Src/IronPython.Modules/array.cs @@ -196,11 +196,7 @@ public array InPlaceMultiply(int value) { if (value <= 0) { _data.Clear(); } else { - var myData = __copy__(); - - for (int i = 0; i < (value - 1); i++) { - ExtendArray(myData); - } + _data.InPlaceMultiply(value); } return this; } diff --git a/Src/IronPython.Modules/bz2/dotnetzip/BZip2/BZip2InputStream.cs b/Src/IronPython.Modules/bz2/dotnetzip/BZip2/BZip2InputStream.cs index 113d139aa..5c731f00b 100644 --- a/Src/IronPython.Modules/bz2/dotnetzip/BZip2/BZip2InputStream.cs +++ b/Src/IronPython.Modules/bz2/dotnetzip/BZip2/BZip2InputStream.cs @@ -1105,7 +1105,7 @@ private void SetupBlock() /* Check: unzftab entries in range. */ for (i = 0; i <= 255; i++) { - if (s.unzftab[i] < 0 || s.unzftab[i] > this.last) + if (s.unzftab[i] < 0 || s.unzftab[i] > this.last+1) throw new Exception("BZ_DATA_ERROR"); } diff --git a/Src/IronPython.Modules/grp.cs b/Src/IronPython.Modules/grp.cs index d46de20f5..dc0344358 100644 --- a/Src/IronPython.Modules/grp.cs +++ b/Src/IronPython.Modules/grp.cs @@ -6,18 +6,19 @@ using System; using System.Collections.Generic; +using System.Numerics; using System.Runtime.InteropServices; using Microsoft.Scripting.Runtime; using IronPython.Runtime; using IronPython.Runtime.Operations; - -using System.Numerics; +using IronPython.Runtime.Exceptions; +using IronPython.Runtime.Types; [assembly: PythonModule("grp", typeof(IronPython.Modules.PythonGrp), PlatformsAttribute.PlatformFamily.Unix)] namespace IronPython.Modules { - + public static class PythonGrp { public const string __doc__ = @"Access to the Unix group database. @@ -79,13 +80,10 @@ private static struct_group Make(IntPtr pwd) { return new struct_group(g.gr_name, g.gr_passwd, g.gr_gid, PythonList.FromEnumerable(MarshalStringArray(g.gr_mem))); } - private static IEnumerable MarshalStringArray(IntPtr arrayPtr) - { - if (arrayPtr != IntPtr.Zero) - { + private static IEnumerable MarshalStringArray(IntPtr arrayPtr) { + if (arrayPtr != IntPtr.Zero) { IntPtr ptr = Marshal.ReadIntPtr(arrayPtr); - while (ptr != IntPtr.Zero) - { + while (ptr != IntPtr.Zero) { string key = Marshal.PtrToStringAnsi(ptr); yield return key; arrayPtr = new IntPtr(arrayPtr.ToInt64() + IntPtr.Size); @@ -96,16 +94,27 @@ private static IEnumerable MarshalStringArray(IntPtr arrayPtr) public static struct_group getgrgid(int gid) { var grp = _getgrgid(gid); - if(grp == IntPtr.Zero) { + if (grp == IntPtr.Zero) { throw PythonOps.KeyError($"getgrgid(): gid not found: {gid}"); } return Make(grp); } + public static struct_group getgrgid(CodeContext context, object gid) { + var g = BigIntegerOps.__new__(context, TypeCache.BigInteger, gid); + PythonOps.Warn(context, PythonExceptions.DeprecationWarning, $"group id must be int, not {PythonOps.GetPythonTypeName(gid)}"); + return g switch { + int i => getgrgid(i), + BigInteger bi => getgrgid((int)bi), + _ => throw new InvalidOperationException(), + }; + } + public static struct_group getgrnam(string name) { + if (name is not null && name.IndexOf((char)0) != -1) throw PythonOps.ValueError("embedded null byte"); var grp = _getgrnam(name); - if(grp == IntPtr.Zero) { + if (grp == IntPtr.Zero) { throw PythonOps.KeyError($"getgrnam()): name not found: {name}"); } @@ -116,31 +125,32 @@ public static PythonList getgrall() { var res = new PythonList(); setgrent(); IntPtr val = getgrent(); - while(val != IntPtr.Zero) { + while (val != IntPtr.Zero) { res.Add(Make(val)); val = getgrent(); } - + return res; } #region P/Invoke Declarations - [DllImport("libc", EntryPoint="getgrgid", CallingConvention=CallingConvention.Cdecl)] + [DllImport("libc", EntryPoint = "getgrgid", CallingConvention = CallingConvention.Cdecl)] private static extern IntPtr _getgrgid(int uid); - [DllImport("libc", EntryPoint="getgrnam", CallingConvention=CallingConvention.Cdecl)] + [DllImport("libc", EntryPoint = "getgrnam", CallingConvention = CallingConvention.Cdecl)] private static extern IntPtr _getgrnam([MarshalAs(UnmanagedType.LPStr)] string name); - [DllImport("libc", CallingConvention=CallingConvention.Cdecl)] + [DllImport("libc", CallingConvention = CallingConvention.Cdecl)] private static extern void setgrent(); - [DllImport("libc", CallingConvention=CallingConvention.Cdecl)] + [DllImport("libc", CallingConvention = CallingConvention.Cdecl)] private static extern IntPtr getgrent(); #endregion } } + #endif diff --git a/Src/IronPython.Modules/math.cs b/Src/IronPython.Modules/math.cs index f104210b2..cde32f792 100644 --- a/Src/IronPython.Modules/math.cs +++ b/Src/IronPython.Modules/math.cs @@ -335,16 +335,22 @@ public static double expm1(double v0) { return Check(v0, Math.Tanh(v0 / 2.0) * (Math.Exp(v0) + 1.0)); } - public static double asinh(double v0) { - if (v0 == 0.0 || double.IsInfinity(v0)) { - return v0; + public static double asinh(double x) { + if (x == 0.0 || double.IsInfinity(x)) { + return x; } - // rewrote ln(v0 + sqrt(v0**2 + 1)) for precision - if (Math.Abs(v0) > 1.0) { - return Math.Sign(v0) * (Math.Log(Math.Abs(v0)) + Math.Log(1.0 + MathUtils.Hypot(1.0, 1.0 / v0))); + +#if NETCOREAPP + return Math.Asinh(x); +#else + // rewrote ln(x + sqrt(x**2 + 1)) for precision + if (Math.Abs(x) > 1.0) { + return Math.Sign(x) * (Math.Log(Math.Abs(x)) + Math.Log(1.0 + MathUtils.Hypot(1.0, 1.0 / x))); } else { - return Math.Log(v0 + MathUtils.Hypot(1.0, v0)); + var x2 = x * x; + return log1p(x + x2 / (Math.Sqrt(x2 + 1) + 1)); } +#endif } public static double asinh(object value) { @@ -358,15 +364,25 @@ public static double asinh(object value) { } } - public static double acosh(double v0) { - if (v0 < 1.0) { + public static double acosh(double x) { + if (x < 1.0) { throw PythonOps.ValueError("math domain error"); - } else if (double.IsPositiveInfinity(v0)) { + } else if (double.IsPositiveInfinity(x)) { return double.PositiveInfinity; } - // rewrote ln(v0 + sqrt(v0**2 - 1)) for precision - double c = Math.Sqrt(v0 + 1.0); - return Math.Log(c) + Math.Log(v0 / c + Math.Sqrt(v0 - 1.0)); + +#if NETCOREAPP + return Math.Acosh(x); +#else + if (x < 2) { + var c = x - 1; + return log1p(c + Math.Sqrt(c * c + 2 * c)); + } else { + // rewrote ln(x + sqrt(x**2 - 1)) for precision + double c = Math.Sqrt(x + 1.0); + return Math.Log(c) + Math.Log(x / c + Math.Sqrt(x - 1.0)); + } +#endif } public static double acosh(object value) { @@ -380,15 +396,20 @@ public static double acosh(object value) { } } - public static double atanh(double v0) { - if (v0 >= 1.0 || v0 <= -1.0) { + public static double atanh(double x) { + if (x >= 1.0 || x <= -1.0) { throw PythonOps.ValueError("math domain error"); - } else if (v0 == 0.0) { + } else if (x == 0.0) { // preserve +/-0.0 - return v0; + return x; } - return Math.Log((1.0 + v0) / (1.0 - v0)) * 0.5; +#if NETCOREAPP + return Math.Atanh(x); +#else + return (log1p(x) - log1p(-x)) * 0.5; +#endif + } public static double atanh(BigInteger value) { diff --git a/Src/IronPython.Modules/nt.cs b/Src/IronPython.Modules/nt.cs index 21e340e4e..5811c9cf7 100644 --- a/Src/IronPython.Modules/nt.cs +++ b/Src/IronPython.Modules/nt.cs @@ -30,9 +30,6 @@ using IronPython.Runtime.Operations; using IronPython.Runtime.Types; -using NotNullWhenAttribute = System.Diagnostics.CodeAnalysis.NotNullWhenAttribute; -using NotNullAttribute = System.Diagnostics.CodeAnalysis.NotNullAttribute; - #if FEATURE_PIPES using System.IO.Pipes; #endif @@ -101,7 +98,7 @@ public static PythonTuple _getdiskusage([NotNone] string path) { private static extern int GetFinalPathNameByHandle([In] SafeFileHandle hFile, [Out] StringBuilder lpszFilePath, [In] int cchFilePath, [In] int dwFlags); [SupportedOSPlatform("windows"), PythonHidden(PlatformsAttribute.PlatformFamily.Unix)] - public static string _getfinalpathname([NotNone] string path) { + public static string _getfinalpathname(CodeContext/*!*/ context, [NotNone] string path) { var hFile = CreateFile(path, 0, 0, IntPtr.Zero, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, IntPtr.Zero); if (hFile.IsInvalid) { throw GetLastWin32Error(path); @@ -115,6 +112,20 @@ public static string _getfinalpathname([NotNone] string path) { return sb.ToString(); } + [SupportedOSPlatform("windows"), PythonHidden(PlatformsAttribute.PlatformFamily.Unix)] + public static Bytes _getfinalpathname(CodeContext/*!*/ context, [NotNone] Bytes path) + => _getfinalpathname(context, path.ToFsString(context)).ToFsBytes(context); + + [SupportedOSPlatform("windows"), PythonHidden(PlatformsAttribute.PlatformFamily.Unix)] + public static object _getfinalpathname(CodeContext/*!*/ context, object? path) { + return ToFsPath(context, path, nameof(path)) switch { + string s => _getfinalpathname(context, s), + Extensible es => _getfinalpathname(context, es.Value), + Bytes b => _getfinalpathname(context, b), + _ => throw new InvalidOperationException(), + }; + } + public static string _getfullpathname(CodeContext/*!*/ context, [NotNone] string/*!*/ path) { PlatformAdaptationLayer pal = context.LanguageContext.DomainManager.Platform; @@ -192,8 +203,14 @@ static bool IsWindows() { public static Bytes _getfullpathname(CodeContext/*!*/ context, [NotNone] Bytes path) => _getfullpathname(context, path.ToFsString(context)).ToFsBytes(context); - public static Bytes _getfullpathname(CodeContext/*!*/ context, object? path) - => _getfullpathname(context, ConvertToFsString(context, path, nameof(path))).ToFsBytes(context); + public static object _getfullpathname(CodeContext/*!*/ context, object? path) { + return ToFsPath(context, path, nameof(path)) switch { + string s => _getfullpathname(context, s), + Extensible es => _getfullpathname(context, es.Value), + Bytes b => _getfullpathname(context, b), + _ => throw new InvalidOperationException(), + }; + } #if FEATURE_PROCESS public static void abort() { @@ -347,6 +364,8 @@ public static void closerange(CodeContext/*!*/ context, int fd_low, int fd_high) } } + public static object? cpu_count() => null; // TODO: implement me! + public static int dup(CodeContext/*!*/ context, int fd) { PythonFileManager fileManager = context.LanguageContext.FileManager; @@ -444,7 +463,7 @@ public static void _exit(CodeContext/*!*/ context, int status) { } public static object fspath(CodeContext context, [AllowNull] object path) - => PythonOps.FsPath(path); + => PythonOps.FsPath(context, path); [LightThrowing] public static object fstat(CodeContext/*!*/ context, int fd) { @@ -588,8 +607,8 @@ public static PythonList listdir(CodeContext context, [NotNone] Bytes path) { return ret; } - public static PythonList listdir(CodeContext context, object? path) - => listdir(context, ConvertToFsString(context, path, nameof(path))); + public static PythonList listdir(CodeContext context, [NotNone] object path) + => listdir(context, ConvertToFsString(context, path, nameof(path), orType: "None")); public static BigInteger lseek(CodeContext context, int fd, long offset, int whence) { var streams = context.LanguageContext.FileManager.GetStreams(fd); @@ -613,7 +632,6 @@ public static object lstat([NotNone] string path, [ParamDictionary, NotNone] IDi public static object lstat(CodeContext context, [NotNone] Bytes path, [ParamDictionary, NotNone] IDictionary kwargs) => lstat(path.ToFsString(context), kwargs); - [LightThrowing, Documentation("")] public static object lstat(CodeContext context, object? path, [ParamDictionary, NotNone] IDictionary kwargs) => lstat(ConvertToFsString(context, path, nameof(path)), kwargs); @@ -644,7 +662,7 @@ internal DirEntry(CodeContext context, FileSystemInfo info, bool asBytes) { public bool is_file(bool follow_symlinks = true) => !is_dir(); - public bool is_symlink() => throw new NotImplementedException(); + public bool is_symlink() => info.Attributes.HasFlag(FileAttributes.ReparsePoint) ? throw new NotImplementedException() : false; [LightThrowing] public object? stat(bool follow_symlinks = true) => PythonNT.stat(info.FullName, new Dictionary()); @@ -687,8 +705,8 @@ internal ScandirIterator(CodeContext context, IEnumerable list, public static ScandirIterator scandir(CodeContext context, string? path = null) => new ScandirIterator(context, ScandirHelper(context, path), asBytes: false); - public static ScandirIterator scandir(CodeContext context, [NotNone] IBufferProtocol path) - => new ScandirIterator(context, ScandirHelper(context, ConvertToFsString(context, path, nameof(path))), asBytes: true); + public static ScandirIterator scandir(CodeContext context, [NotNone] object path) + => new ScandirIterator(context, ScandirHelper(context, ConvertToFsString(context, path, nameof(path), orType: "None")), asBytes: true); private static IEnumerable ScandirHelper(CodeContext context, string? path) { if (path == null) { @@ -831,7 +849,7 @@ public static void mkdir([NotNone] string path, [ParamDictionary, NotNone] IDict } } - if (Directory.Exists(path)) throw DirectoryExists(); + if (Directory.Exists(path)) throw DirectoryExistsError(path); // we ignore mode try { @@ -1532,11 +1550,15 @@ public static object stat(CodeContext context, [NotNone] Bytes path, [ParamDicti => stat(path.ToFsString(context), dict); [LightThrowing, Documentation("")] - public static object stat(CodeContext context, [NotNone] IBufferProtocol path, [ParamDictionary, NotNone] IDictionary dict) { - // TODO: accept object? path to get nicer error message? - // TODO: Python 3.6: os.PathLike - PythonOps.Warn(context, PythonExceptions.DeprecationWarning, $"{nameof(stat)}: {nameof(path)} should be string, bytes or integer, not {PythonOps.GetPythonTypeName(path)}"); // deprecated in 3.6 - return stat(path.ToFsBytes(context).ToFsString(context), dict); + public static object stat(CodeContext context, object? path, [ParamDictionary, NotNone] IDictionary dict) { + if (PythonOps.TryToIndex(path, out BigInteger bi)) { + if (bi.AsInt32(out int i)) { + return stat(context, i); + } + throw PythonOps.OverflowError("fd is greater than maximum"); + } + + return stat(ConvertToFsString(context, path, nameof(path), orType: "integer"), dict); } [LightThrowing, Documentation("")] @@ -1632,11 +1654,16 @@ public static void truncate(CodeContext context, [NotNone] Bytes path, BigIntege => truncate(context, path.ToFsString(context), length); [Documentation("")] - public static void truncate(CodeContext context, [NotNone] IBufferProtocol path, BigInteger length) { - // TODO: accept object? path to get nicer error message? - // TODO: Python 3.6: os.PathLike - PythonOps.Warn(context, PythonExceptions.DeprecationWarning, $"{nameof(truncate)}: {nameof(path)} should be string, bytes or integer, not {PythonOps.GetPythonTypeName(path)}"); // deprecated in 3.6 - truncate(context, path.ToFsBytes(context).ToFsString(context), length); + public static void truncate(CodeContext context, object? path, BigInteger length) { + if (PythonOps.TryToIndex(path, out BigInteger bi)) { + if (bi.AsInt32(out int i)) { + truncate(context, i, length); + return; + } + throw PythonOps.OverflowError("fd is greater than maximum"); + } + + truncate(context, ConvertToFsString(context, path, nameof(path), orType: "integer"), length); } public static void truncate(CodeContext context, int fd, BigInteger length) @@ -2313,8 +2340,13 @@ private static bool TryGetShellCommand(string command, [NotNullWhen(true)] out s #endif - private static Exception DirectoryExists() { - return PythonOps.OSError(PythonExceptions._OSError.ERROR_ALREADY_EXISTS, "directory already exists", null, PythonExceptions._OSError.ERROR_ALREADY_EXISTS); + private static Exception DirectoryExistsError(string? filename) { +#if FEATURE_NATIVE + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { + return GetWin32Error(PythonExceptions._OSError.ERROR_ALREADY_EXISTS, filename); + } +#endif + return GetOsError(PythonErrorNumber.EEXIST, filename); } #if FEATURE_NATIVE @@ -2365,21 +2397,26 @@ private static Encoding _getFileSystemEncoding(CodeContext context) { private static Bytes ToFsBytes(this string s, CodeContext context) => Bytes.Make(_getFileSystemEncoding(context).GetBytes(s)); - private static Bytes ToFsBytes(this IBufferProtocol bp, CodeContext context) { - // TODO: Python 3.6: "path should be string, bytes or os.PathLike" - PythonOps.Warn(context, PythonExceptions.DeprecationWarning, "path should be string or bytes, not {0}", PythonOps.GetPythonTypeName(bp)); - return new Bytes(bp); // accepts FULL_RO buffers in CPython + private static object ToFsPath(CodeContext context, object? o, string argname, [CallerMemberName] string? methodname = null, string? orType = null) { + if (o is not Bytes && o is IBufferProtocol bp) { + if (orType is null) + PythonOps.Warn(context, PythonExceptions.DeprecationWarning, "{0}: {1} should be string, bytes or os.PathLike, not {2}", methodname, argname, PythonOps.GetPythonTypeName(o)); + else + PythonOps.Warn(context, PythonExceptions.DeprecationWarning, "{0}: {1} should be string, bytes, os.PathLike or {3}, not {2}", methodname, argname, PythonOps.GetPythonTypeName(o), orType); + o = new Bytes(bp); // accepts FULL_RO buffers in CPython + } + + if (PythonOps.TryToFsPath(context, o, out var res)) + return res; + + if (orType is null) + throw PythonOps.TypeError("{0}: {1} should be string, bytes or os.PathLike, not {2}", methodname, argname, PythonOps.GetPythonTypeName(o)); + else + throw PythonOps.TypeError("{0}: {1} should be string, bytes, os.PathLike or {3}, not {2}", methodname, argname, PythonOps.GetPythonTypeName(o), orType); } - private static string ConvertToFsString(CodeContext context, [NotNull] object? o, string argname, [CallerMemberName] string? methodname = null) - => o switch { - string s => s, - ExtensibleString es => es.Value, - Bytes b => b.ToFsString(context), - IBufferProtocol bp => bp.ToFsBytes(context).ToFsString(context), - // TODO: Python 3.6: os.PathLike - _ => throw PythonOps.TypeError("{0}: {1} should be string or bytes, not '{2}'", methodname, argname, PythonOps.GetPythonTypeName(o)) - }; + private static string ConvertToFsString(CodeContext context, object? o, string argname, [CallerMemberName] string? methodname = null, string? orType = null) + => PythonOps.DecodeFsPath(context, ToFsPath(context, o, argname, methodname, orType)); private static void CheckOptionalArgsCount(int numRegParms, int numOptPosParms, int numKwParms, int numOptPosArgs, int numKwArgs, [CallerMemberName] string? methodname = null) { if (numOptPosArgs > numOptPosParms) diff --git a/Src/IronPython.Modules/time.cs b/Src/IronPython.Modules/time.cs index f3aa4c69a..806efd26c 100644 --- a/Src/IronPython.Modules/time.cs +++ b/Src/IronPython.Modules/time.cs @@ -340,10 +340,12 @@ private static List PythonFormatToCLIFormat(string format) { List newFormat = new List(); for (int i = 0; i < format.Length; i++) { - if (format[i] == '%') { + var ch = format[i]; + if (ch == '%') { if (i + 1 == format.Length) throw PythonOps.ValueError("Invalid format string"); switch (format[++i]) { + case '\0': throw PythonOps.ValueError("embedded null character"); case 'a': newFormat.Add(new FormatInfo("ddd")); break; case 'A': newFormat.Add(new FormatInfo("dddd")); break; case 'b': newFormat.Add(new FormatInfo("MMM")); break; @@ -379,10 +381,11 @@ private static List PythonFormatToCLIFormat(string format) { throw PythonOps.ValueError("Invalid format string"); } } else { + if (ch == '\0') throw PythonOps.ValueError("embedded null character"); if (newFormat.Count == 0 || newFormat[newFormat.Count - 1].Type != FormatInfoType.UserText) - newFormat.Add(new FormatInfo(FormatInfoType.UserText, format[i].ToString())); + newFormat.Add(new FormatInfo(FormatInfoType.UserText, ch.ToString())); else - newFormat[newFormat.Count - 1].Text += format[i]; + newFormat[newFormat.Count - 1].Text += ch; } } diff --git a/Src/IronPython.Modules/winreg.cs b/Src/IronPython.Modules/winreg.cs index 5b80836d2..fa885bae0 100644 --- a/Src/IronPython.Modules/winreg.cs +++ b/Src/IronPython.Modules/winreg.cs @@ -4,9 +4,8 @@ #if FEATURE_REGISTRY -using Microsoft.Win32; -using Microsoft.Win32.SafeHandles; using System; +using System.Buffers.Binary; using System.Collections.Concurrent; using System.ComponentModel; using System.Diagnostics; @@ -22,6 +21,9 @@ using IronPython.Runtime.Exceptions; using IronPython.Runtime.Types; +using Microsoft.Win32; +using Microsoft.Win32.SafeHandles; + [assembly: PythonModule("winreg", typeof(IronPython.Modules.PythonWinReg), PlatformsAttribute.PlatformFamily.Windows)] namespace IronPython.Modules { [SupportedOSPlatform("windows")] @@ -70,6 +72,7 @@ public static class PythonWinReg { public const int REG_FULL_RESOURCE_DESCRIPTOR = 0X9; public const int REG_RESOURCE_REQUIREMENTS_LIST = 0XA; public const int REG_QWORD = 0xB; + public const int REG_QWORD_LITTLE_ENDIAN = 0XB; public const int REG_NOTIFY_CHANGE_NAME = 0X1; public const int REG_NOTIFY_CHANGE_ATTRIBUTES = 0X2; @@ -346,19 +349,22 @@ private static void QueryValueExImpl(SafeRegistryHandle handle, string valueName break; case REG_EXPAND_SZ: case REG_SZ: - if (length >= 2 && data[length - 1] == 0 && data[length - 2] == 0) { - value = ExtractString(data, 0, (int)length - 2); - } else { - value = ExtractString(data, 0, (int)length); - } + var span = MemoryMarshal.Cast(data.AsSpan()); + var len = span.IndexOf((char)0); + if (len != -1) span = span.Slice(0, len); + value = span.ToString(); break; case REG_DWORD: - if (BitConverter.IsLittleEndian) { - value = (uint)((data[3] << 24) | (data[2] << 16) | (data[1] << 8) | data[0]); - } else { - value = (uint)((data[0] << 24) | (data[1] << 16) | (data[2] << 8) | data[3]); - } + var dval = BitConverter.ToUInt32(data, 0); + if (!BitConverter.IsLittleEndian) dval = BinaryPrimitives.ReverseEndianness(dval); + value = dval > int.MaxValue ? (BigInteger)dval : unchecked((int)dval); break; + case REG_QWORD: + var qval = BitConverter.ToUInt64(data, 0); + if (!BitConverter.IsLittleEndian) qval = BinaryPrimitives.ReverseEndianness(qval); + value = (BigInteger)qval; + break; + default: value = null; break; diff --git a/Src/IronPython.Modules/winsound.cs b/Src/IronPython.Modules/winsound.cs index 56cb0cfc8..045b99124 100644 --- a/Src/IronPython.Modules/winsound.cs +++ b/Src/IronPython.Modules/winsound.cs @@ -120,20 +120,20 @@ public static void PlaySound(CodeContext/*!*/ context, [NotNone] IBufferProtocol This parameter must be in the range 37 through 32,767. The duration argument specifies the number of milliseconds. ")] - public static void Beep(CodeContext/*!*/ context, int freq, int dur) { - if (freq < 37 || freq > 32767) { + public static void Beep(CodeContext/*!*/ context, int frequency, int duration) { + if (frequency < 37 || frequency > 32767) { throw PythonOps.ValueError("frequency must be in 37 thru 32767"); } - bool ok = Beep(freq, dur); + bool ok = Beep(frequency, duration); if (!ok) { throw PythonOps.RuntimeError("Failed to beep"); } } [Documentation("MessageBeep(x) - call Windows MessageBeep(x). x defaults to MB_OK.")] - public static void MessageBeep(CodeContext/*!*/ context, int x=MB_OK) { - MessageBeep(x); + public static void MessageBeep(CodeContext/*!*/ context, int type=MB_OK) { + MessageBeep(type); } #endregion diff --git a/Src/IronPython/Lib/iptest/ipunittest.py b/Src/IronPython/Lib/iptest/ipunittest.py index ca990a35a..91c4cddb9 100644 --- a/Src/IronPython/Lib/iptest/ipunittest.py +++ b/Src/IronPython/Lib/iptest/ipunittest.py @@ -206,24 +206,25 @@ def _flatten_suite(suite): def generate_suite(tests, failing_tests, skip_tests=[]): all_tests = _flatten_suite(tests) + test_indices = {t: i for i, t in enumerate(all_tests)} unknown_tests = [] for t in skip_tests: try: - all_tests.remove(t) + all_tests[test_indices.pop(t)] = None except ValueError: unknown_tests.append(t) for t in failing_tests: try: - all_tests[all_tests.index(t)] = unittest.expectedFailure(t) - except: + all_tests[test_indices.pop(t)] = unittest.expectedFailure(t) + except ValueError: unknown_tests.append(t) if unknown_tests: raise ValueError("Unknown tests:\n - {}".format('\n - '.join(str(t) for t in unknown_tests))) - return unittest.TestSuite(all_tests) + return unittest.TestSuite(t for t in all_tests if t is not None) # class testpath: # # find the ironpython root directory diff --git a/Src/IronPython/Modules/Builtin.cs b/Src/IronPython/Modules/Builtin.cs index 6d21a52f1..3131022e7 100644 --- a/Src/IronPython/Modules/Builtin.cs +++ b/Src/IronPython/Modules/Builtin.cs @@ -135,7 +135,7 @@ public static string bin(object? number) { public static PythonType bytes => TypeCache.Bytes; - public static PythonType bytearray => DynamicHelpers.GetPythonTypeFromType(typeof(ByteArray)); + public static PythonType bytearray => TypeCache.ByteArray; [Documentation("callable(object) -> bool\n\nReturn whether the object is callable (i.e., some kind of function).")] public static bool callable(CodeContext/*!*/ context, object? o) { diff --git a/Src/IronPython/Modules/_fileio.cs b/Src/IronPython/Modules/_fileio.cs index 51b1b58e8..9eb796fda 100644 --- a/Src/IronPython/Modules/_fileio.cs +++ b/Src/IronPython/Modules/_fileio.cs @@ -82,6 +82,9 @@ public FileIO(CodeContext/*!*/ context, int fd, string mode = "r", bool closefd public FileIO(CodeContext/*!*/ context, string name, string mode = "r", bool closefd = true, object opener = null) : base(context) { + if (name.Contains('\0')) { + throw PythonOps.ValueError("embedded null character"); + } if (!closefd) { throw PythonOps.ValueError("Cannot use closefd=False with file name"); } @@ -543,5 +546,10 @@ private void EnsureWritable() { #endregion } + +#if !NETCOREAPP + private static bool Contains(this string str, char value) + => str.IndexOf(value) != -1; +#endif } } diff --git a/Src/IronPython/Modules/_io.cs b/Src/IronPython/Modules/_io.cs index 94cb80a30..9c896f8e6 100644 --- a/Src/IronPython/Modules/_io.cs +++ b/Src/IronPython/Modules/_io.cs @@ -3144,9 +3144,8 @@ private static bool TryGetBigInt(object? i, out BigInteger res) { private static int GetInt(object? i, string? msg = null) { if (i == Missing.Value) return 0; - int res; - if (TryGetInt(i, out res)) { - return res; + if (Converter.TryConvertToIndex(i, out int index, throwOverflowError: true)) { + return index; } if (msg == null) { @@ -3156,12 +3155,12 @@ private static int GetInt(object? i, string? msg = null) { throw PythonOps.TypeError(msg); } - private static int GetInt(object? i, int defaultValue, string? msg = null) { - if (i == null) { + private static int GetInt(object? i, int defaultValue) { + if (i is null) { return defaultValue; } - return GetInt(i, msg); + return GetInt(i); } private static bool TryGetInt(object? i, out int value) { diff --git a/Src/IronPython/Modules/_io/BytesIO.cs b/Src/IronPython/Modules/_io/BytesIO.cs index c0f386221..637c8b76f 100644 --- a/Src/IronPython/Modules/_io/BytesIO.cs +++ b/Src/IronPython/Modules/_io/BytesIO.cs @@ -192,13 +192,15 @@ private Bytes readline(int size = -1) { } public Bytes readline(object size) { - if (size == null) { + _checkClosed(); + if (size is null) { return readline(-1); } + if (Converter.TryConvertToIndex(size, out int index, throwOverflowError: true)) { + return readline(index); + } - _checkClosed(); - - throw PythonOps.TypeError("integer argument expected, got '{0}'", PythonOps.GetPythonTypeName(size)); + throw PythonOps.TypeError("argument should be integer or None, not '{0}'", PythonOps.GetPythonTypeName(size)); } [Documentation("readlines([size]) -> list of bytes objects, each a line from the file.\n\n" diff --git a/Src/IronPython/Runtime/ArrayData.cs b/Src/IronPython/Runtime/ArrayData.cs index c8f2b60ca..4b12de25e 100644 --- a/Src/IronPython/Runtime/ArrayData.cs +++ b/Src/IronPython/Runtime/ArrayData.cs @@ -27,6 +27,7 @@ internal interface ArrayData : IList { void InsertRange(int index, int count, ArrayData value); void RemoveSlice(Slice slice); ArrayData Multiply(int count); + void InPlaceMultiply(int count); new bool Remove(object? item); void Reverse(); Span AsByteSpan(); diff --git a/Src/IronPython/Runtime/ByteArray.cs b/Src/IronPython/Runtime/ByteArray.cs index ef3f6a2a0..38e54bb6a 100644 --- a/Src/IronPython/Runtime/ByteArray.cs +++ b/Src/IronPython/Runtime/ByteArray.cs @@ -9,16 +9,16 @@ using System.Collections.Generic; using System.Numerics; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; using System.Text; -using Microsoft.Scripting.Runtime; -using Microsoft.Scripting.Utils; - using IronPython.Runtime.Exceptions; using IronPython.Runtime.Operations; using IronPython.Runtime.Types; +using Microsoft.Scripting; +using Microsoft.Scripting.Runtime; +using Microsoft.Scripting.Utils; + namespace IronPython.Runtime { /// /// bytearray(string, encoding[, errors]) -> bytearray @@ -56,6 +56,12 @@ internal ByteArray(IEnumerable bytes) { _bytes = new ArrayData(bytes); } + [StaticExtensionMethod] + public static object __new__(CodeContext context, [NotNone] PythonType cls, [ParamDictionary, NotNone] IDictionary dict, [NotNone] params object[] args) { + if (cls == TypeCache.ByteArray) return new ByteArray(); + return cls.CreateInstance(context); + } + public void __init__() { lock (this) { _bytes.Clear(); @@ -74,7 +80,7 @@ public void __init__(int source) { } public void __init__([NotNone] IBufferProtocol source) { - if (Converter.TryConvertToIndex(source, out int size, throwNonInt: false)) { + if (Converter.TryConvertToIndex(source, out int size, throwTypeError: false)) { __init__(size); } else { lock (this) { @@ -86,7 +92,7 @@ public void __init__([NotNone] IBufferProtocol source) { } public void __init__(CodeContext context, object? source) { - if (Converter.TryConvertToIndex(source, out int size, throwNonInt: false)) { + if (Converter.TryConvertToIndex(source, out int size, throwTypeError: false)) { __init__(size); } else if (source is IEnumerable en) { lock (this) { @@ -466,8 +472,13 @@ public int find(BigInteger @byte, object? start, object? end) { } } - public static ByteArray fromhex([NotNone] string @string) { - return new ByteArray(IListOfByteOps.FromHex(@string)); + [ClassMethod] + public static object fromhex(CodeContext context, [NotNone] PythonType cls, [NotNone] string @string) { + var hex = IListOfByteOps.FromHex(@string); + if (cls == TypeCache.ByteArray) { + return new ByteArray(hex); + } + return PythonTypeOps.CallParams(context, cls, new Bytes(hex)); } public string hex() => Bytes.ToHex(_bytes.AsByteSpan()); // new in CPython 3.5 diff --git a/Src/IronPython/Runtime/Bytes.cs b/Src/IronPython/Runtime/Bytes.cs index 1f4c3d1d0..e6d5ac5d3 100644 --- a/Src/IronPython/Runtime/Bytes.cs +++ b/Src/IronPython/Runtime/Bytes.cs @@ -61,7 +61,7 @@ public static object __new__(CodeContext context, [NotNone] PythonType cls, [Not return source; } else if (TryInvokeBytesOperator(context, source, out Bytes? res)) { return res; - } else if (Converter.TryConvertToIndex(source, out int size, throwNonInt: false)) { + } else if (Converter.TryConvertToIndex(source, out int size, throwTypeError: false)) { if (size < 0) throw PythonOps.ValueError("negative count"); return new Bytes(new byte[size]); } else { @@ -79,7 +79,7 @@ public static object __new__(CodeContext context, [NotNone] PythonType cls, obje return @object; } else if (TryInvokeBytesOperator(context, @object, out Bytes? res)) { return res; - } else if (Converter.TryConvertToIndex(@object, out int size, throwNonInt: false)) { + } else if (Converter.TryConvertToIndex(@object, out int size, throwTypeError: false)) { if (size < 0) throw PythonOps.ValueError("negative count"); return new Bytes(new byte[size]); } else { @@ -367,8 +367,13 @@ public int find(BigInteger @byte, object? start, object? end) { } [ClassMethod] - public static object fromhex(CodeContext context, [NotNone] PythonType cls, [NotNone] string @string) - => __new__(context, cls, IListOfByteOps.FromHex(@string)); + public static object fromhex(CodeContext context, [NotNone] PythonType cls, [NotNone] string @string) { + var hex = IListOfByteOps.FromHex(@string); + if (cls == TypeCache.Bytes) { + return new Bytes(hex); + } + return PythonTypeOps.CallParams(context, cls, new Bytes(hex)); + } public string hex() => ToHex(_bytes.AsSpan()); // new in CPython 3.5 @@ -1190,6 +1195,13 @@ public bool __eq__(CodeContext context, [NotNone] string value) { public bool __eq__(CodeContext context, [NotNone] Extensible value) => __eq__(context, value.Value); + public bool __eq__(CodeContext context, [NotNone] int value) { + if (context.LanguageContext.PythonOptions.BytesWarning != Microsoft.Scripting.Severity.Ignore) { + PythonOps.Warn(context, PythonExceptions.BytesWarning, "Comparison between bytes and int"); + } + return false; + } + [return: MaybeNotImplemented] public NotImplementedType __eq__(CodeContext context, object? value) => NotImplementedType.Value; @@ -1199,6 +1211,8 @@ public bool __eq__(CodeContext context, [NotNone] string value) { public bool __ne__(CodeContext context, [NotNone] Extensible value) => !__eq__(context, value); + public bool __ne__(CodeContext context, [NotNone] int value) => !__eq__(context, value); + [return: MaybeNotImplemented] public NotImplementedType __ne__(CodeContext context, object? value) => NotImplementedType.Value; diff --git a/Src/IronPython/Runtime/Converter.cs b/Src/IronPython/Runtime/Converter.cs index e8e860457..9dc49999e 100644 --- a/Src/IronPython/Runtime/Converter.cs +++ b/Src/IronPython/Runtime/Converter.cs @@ -335,28 +335,34 @@ public static IEnumerable ConvertToIEnumerable(object o) { /// If throwOverflowError is true then BigInteger's outside the normal range of integers will /// result in an OverflowError. /// - /// When throwNonInt is true, a TypeError will be thrown if __index__ returned a non-int. + /// When throwTypeError is true, a TypeError will be thrown if __index__ throw a TypeError or + /// returned a non-int. /// - internal static bool TryConvertToIndex(object? value, out int index, bool throwOverflowError = true, bool throwNonInt = true) { + internal static bool TryConvertToIndex(object? value, out int index, bool throwOverflowError = true, bool throwTypeError = true) { if (TryGetInt(value, out index, throwOverflowError, value)) { return true; } - if (PythonTypeOps.TryInvokeUnaryOperator(DefaultContext.Default, value, "__index__", out object indexObj)) { - if (TryGetInt(indexObj, out index, throwOverflowError, value)) { - return true; - } + if (throwTypeError) { + if (PythonTypeOps.TryInvokeUnaryOperator(DefaultContext.Default, value, "__index__", out object indexObj)) { + if (TryGetInt(indexObj, out index, throwOverflowError, value)) { + return true; + } - if (throwNonInt) { throw PythonOps.TypeError("__index__ returned non-int (type {0})", PythonOps.GetPythonTypeName(indexObj)); } + } else { + try { + return PythonTypeOps.TryInvokeUnaryOperator(DefaultContext.Default, value, "__index__", out object indexObj) && + TryGetInt(indexObj, out index, throwOverflowError, value); + } catch (Exceptions.TypeErrorException) { } } return false; } public static int ConvertToIndex(object? value, bool throwOverflowError = false) { - if (TryConvertToIndex(value, out int index, throwOverflowError: throwOverflowError, throwNonInt: true)) + if (TryConvertToIndex(value, out int index, throwOverflowError: throwOverflowError)) return index; throw PythonOps.TypeError("expected integer value, got {0}", PythonOps.GetPythonTypeName(value)); diff --git a/Src/IronPython/Runtime/DictionaryOps.cs b/Src/IronPython/Runtime/DictionaryOps.cs index cf5f3cd0d..dec342ce0 100644 --- a/Src/IronPython/Runtime/DictionaryOps.cs +++ b/Src/IronPython/Runtime/DictionaryOps.cs @@ -51,7 +51,12 @@ public static class DictionaryOps { buf.Append(PythonOps.Repr(context, kv.Key)); buf.Append(": "); - buf.Append(PythonOps.Repr(context, kv.Value)); + try { + PythonOps.FunctionPushFrame(context.LanguageContext); + buf.Append(PythonOps.Repr(context, kv.Value)); + } finally { + PythonOps.FunctionPopFrame(); + } } buf.Append("}"); return buf.ToString(); @@ -94,12 +99,12 @@ public static object pop(PythonDictionary self, object key, object defaultValue) } } - public static PythonTuple popitem(IDictionary self) { - IEnumerator> ie = self.GetEnumerator(); + public static PythonTuple popitem(PythonDictionary self) { + using IEnumerator> ie = self.GetEnumerator(); if (ie.MoveNext()) { object key = ie.Current.Key; object val = ie.Current.Value; - self.Remove(key); + self.RemoveDirect(key); return PythonTuple.MakeTuple(key, val); } throw PythonOps.KeyError("dictionary is empty"); diff --git a/Src/IronPython/Runtime/Exceptions/PythonExceptions.cs b/Src/IronPython/Runtime/Exceptions/PythonExceptions.cs index 8ff54a605..55e97d29b 100644 --- a/Src/IronPython/Runtime/Exceptions/PythonExceptions.cs +++ b/Src/IronPython/Runtime/Exceptions/PythonExceptions.cs @@ -881,7 +881,7 @@ internal static void RemoveTraceBack(this Exception e) { string sourceLine = PythonContext.GetSourceLine(e); string fileName = e.GetSymbolDocumentName(); - object column = (e.Column == 0 || e.Data[PythonContext._syntaxErrorNoCaret] != null) ? null : (object)e.Column; + int column = e.Data[PythonContext._syntaxErrorNoCaret] is not null ? 0 : e.Column; se.args = PythonTuple.MakeTuple(e.Message, PythonTuple.MakeTuple(fileName, e.Line, column, sourceLine)); diff --git a/Src/IronPython/Runtime/Operations/PythonOps.cs b/Src/IronPython/Runtime/Operations/PythonOps.cs index 2d4554f9b..f95fd6acf 100644 --- a/Src/IronPython/Runtime/Operations/PythonOps.cs +++ b/Src/IronPython/Runtime/Operations/PythonOps.cs @@ -325,25 +325,40 @@ public static string FormatString(CodeContext/*!*/ context, string str, object d return StringFormatter.Format(context, str, data); } - internal static object FsPath(object? path) { - if (path is string) return path; - if (path is Extensible) return path; - if (path is Bytes) return path; - - if (PythonTypeOps.TryInvokeUnaryOperator(DefaultContext.Default, path, "__fspath__", out object res)) { - return res switch { - string => res, - Extensible => res, - Bytes => res, - _ => throw PythonOps.TypeError("expected {0}.__fspath__() to return str or bytes, not {0}", PythonOps.GetPythonTypeName(path), PythonOps.GetPythonTypeName(res)) - }; - } + internal static object FsPath(CodeContext context, object? path) { + if (TryToFsPath(context, path, out var res)) + return res; throw PythonOps.TypeError("expected str, bytes or os.PathLike object, not {0}", PythonOps.GetPythonTypeName(path)); } - internal static string FsPathDecoded(CodeContext context, object? path) { - return PythonOps.FsPath(path) switch { + internal static bool TryToFsPath(CodeContext context, object? path, [NotNullWhen(true)] out object? res) { + res = path; + if (res is string || res is Extensible || res is Bytes) return true; + + if (PythonTypeOps.TryInvokeUnaryOperator(DefaultContext.Default, path, "__fspath__", out res)) { + if (res is string || res is Extensible || res is Bytes) return true; + throw PythonOps.TypeError("expected {0}.__fspath__() to return str or bytes, not {1}", PythonOps.GetPythonTypeName(path), PythonOps.GetPythonTypeName(res)); + } + + return false; + } + + internal static string FsPathDecoded(CodeContext context, object? path) + => DecodeFsPath(context, FsPath(context, path)); + + internal static bool TryToFsPathDecoded(CodeContext context, object? path, [NotNullWhen(true)] out string? res) { + if (PythonOps.TryToFsPath(context, path, out object? obj)) { + res = DecodeFsPath(context, obj); + return true; + } + + res = null; + return false; + } + + internal static string DecodeFsPath(CodeContext context, object obj) { + return obj switch { string s => s, Extensible es => es, Bytes b => b.decode(context, SysModule.getfilesystemencoding(context), SysModule.getfilesystemencodeerrors()), @@ -772,7 +787,7 @@ public static object PowerMod(CodeContext/*!*/ context, object? x, object? y, ob } } - throw PythonOps.TypeErrorForBinaryOp("power with modulus", x, y); + throw PythonOps.TypeError("unsupported operand type(s) for pow(): '{0}', '{1}', '{2}'", GetPythonTypeName(x), GetPythonTypeName(y), GetPythonTypeName(y)); } public static long Id(object? o) { @@ -912,10 +927,16 @@ public static int Length(object? o) { } internal static bool TryInvokeLengthHint(CodeContext context, object? sequence, out int hint) { - if (PythonTypeOps.TryInvokeUnaryOperator(context, sequence, "__len__", out object len_obj) || - PythonTypeOps.TryInvokeUnaryOperator(context, sequence, "__length_hint__", out len_obj)) { + if (PythonTypeOps.TryInvokeUnaryOperator(context, sequence, "__len__", out object len_obj)) { + if (!(len_obj is NotImplementedType)) { + hint = Converter.ConvertToInt32(len_obj); + if (hint < 0) throw ValueError("__len__() should return >= 0"); + return true; + } + } else if (PythonTypeOps.TryInvokeUnaryOperator(context, sequence, "__length_hint__", out len_obj)) { if (!(len_obj is NotImplementedType)) { hint = Converter.ConvertToInt32(len_obj); + if (hint < 0) throw ValueError("__length_hint__() should return >= 0"); return true; } } @@ -4138,7 +4159,7 @@ internal static Exception TypeErrorForBadInstance(string template, object? insta } public static Exception TypeErrorForBinaryOp(string opSymbol, object? x, object? y) { - throw PythonOps.TypeError("unsupported operand type(s) for {0}: '{1}' and '{2}'", + throw PythonOps.TypeError("'{0}' not supported between instances of '{1}' and '{2}'", opSymbol, GetPythonTypeName(x), GetPythonTypeName(y)); } diff --git a/Src/IronPython/Runtime/Operations/StringOps.cs b/Src/IronPython/Runtime/Operations/StringOps.cs index 9a3e98a7c..36a2e3723 100644 --- a/Src/IronPython/Runtime/Operations/StringOps.cs +++ b/Src/IronPython/Runtime/Operations/StringOps.cs @@ -1219,6 +1219,7 @@ private static void AppendValueForTranslate(this StringBuilder ret, object? mapp return; case int mappedInt: if (mappedInt > 0xFFFF) { + // TODO: change to a ValueError in Python 3.5 throw PythonOps.TypeError("character mapping must be in range(0x10000)"); } ret.Append((char)mappedInt); @@ -1991,17 +1992,28 @@ private static string UserDecode(CodeContext context, PythonTuple codecInfo, obj internal static partial class CodecsInfo { internal static readonly Encoding MbcsEncoding; + internal static readonly Encoding OemEncoding; internal static readonly Encoding RawUnicodeEscapeEncoding = new UnicodeEscapeEncoding(raw: true); internal static readonly Encoding UnicodeEscapeEncoding = new UnicodeEscapeEncoding(raw: false); internal static readonly IDictionary> Codecs; + [DllImport(Interop.Libraries.Kernel32, ExactSpelling = true, CharSet = CharSet.Auto)] + private static extern int GetOEMCP(); + static CodecsInfo() { #if !NETFRAMEWORK // This ensures that Encoding.GetEncoding(0) will return the default Windows ANSI code page Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); #endif - // Use Encoding.GetEncoding(0) instead of Encoding.Default (which returns UTF-8 with .NET Core) - MbcsEncoding = Encoding.GetEncoding(0); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { + // Use Encoding.GetEncoding(0) instead of Encoding.Default (which returns UTF-8 with .NET Core) + MbcsEncoding = Encoding.GetEncoding(0); + OemEncoding = Encoding.GetEncoding(GetOEMCP()); + } else { + MbcsEncoding = null!; + OemEncoding = null!; + } Codecs = MakeCodecsDict(); } @@ -2030,6 +2042,7 @@ static CodecsInfo() { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { d["mbcs"] = makeEncodingProxy(() => MbcsEncoding); + d["oem"] = makeEncodingProxy(() => OemEncoding); } // TODO: revisit the exceptions to rules below once _codecs_cn, _codecs_hk, _codecs_jp, and _codecs_kr are implemented @@ -2379,7 +2392,14 @@ private class BackslashReplaceFallbackBuffer : EncoderFallbackBuffer { private int _index; public override bool Fallback(char charUnknownHigh, char charUnknownLow, int index) { - return false; + var val = char.ConvertToUtf32(charUnknownHigh, charUnknownLow); + _buffer.Add('\\'); + _buffer.Add('U'); + AddCharacter(val >> 24); + AddCharacter((val >> 16) & 0xFF); + AddCharacter((val >> 8) & 0xFF); + AddCharacter(val & 0xFF); + return true; } public override bool Fallback(char charUnknown, int index) { diff --git a/Src/IronPython/Runtime/PythonDictionary.cs b/Src/IronPython/Runtime/PythonDictionary.cs index 8a6033482..5fd1cff15 100644 --- a/Src/IronPython/Runtime/PythonDictionary.cs +++ b/Src/IronPython/Runtime/PythonDictionary.cs @@ -87,12 +87,12 @@ internal static PythonDictionary MakeSymbolDictionary(int count) { return new PythonDictionary(new StringDictionaryStorage(count)); } - public void __init__(CodeContext/*!*/ context, object o\u00F8, [ParamDictionary]IDictionary kwArgs) { + public void __init__(CodeContext/*!*/ context, object o\u00F8, [ParamDictionary] IDictionary kwArgs) { update(context, o\u00F8); update(context, kwArgs); } - public void __init__(CodeContext/*!*/ context, [ParamDictionary]IDictionary kwArgs) { + public void __init__(CodeContext/*!*/ context, [ParamDictionary] IDictionary kwArgs) { update(context, kwArgs); } @@ -356,7 +356,7 @@ public object setdefault(object key, object defaultValue) { public void update() { } - public void update(CodeContext/*!*/ context, [ParamDictionary]IDictionary other\u00F8) { + public void update(CodeContext/*!*/ context, [ParamDictionary] IDictionary other\u00F8) { DictionaryOps.update(context, this, other\u00F8); } @@ -364,7 +364,7 @@ public void update(CodeContext/*!*/ context, object other\u00F8) { DictionaryOps.update(context, this, other\u00F8); } - public void update(CodeContext/*!*/ context, object other\u00F8, [ParamDictionary]IDictionary otherArgs\u00F8) { + public void update(CodeContext/*!*/ context, object other\u00F8, [ParamDictionary] IDictionary otherArgs\u00F8) { DictionaryOps.update(context, this, other\u00F8); DictionaryOps.update(context, this, otherArgs\u00F8); } @@ -1491,7 +1491,7 @@ void ICollection.CopyTo(object[] array, int arrayIndex) { int ICollection.Count => _dict.Count; - bool ICollection.IsReadOnly => true; + bool ICollection.IsReadOnly => true; bool ICollection.Remove(object item) => throw new NotSupportedException("Collection is read-only"); diff --git a/Src/IronPython/Runtime/PythonList.cs b/Src/IronPython/Runtime/PythonList.cs index a2a2b5e33..286d55010 100644 --- a/Src/IronPython/Runtime/PythonList.cs +++ b/Src/IronPython/Runtime/PythonList.cs @@ -772,7 +772,10 @@ public void extend([NotNone] PythonTuple/*!*/ seq) { public void extend(CodeContext context, object? seq) { if (PythonOps.TryInvokeLengthHint(context, seq, out int len)) { - EnsureSize(len); + // CPython proceeds without resizing if the length overflows + if (int.MaxValue - len >= Count) { + EnsureSize(Count + len); + } } ExtendNoLengthCheck(context, seq); diff --git a/Src/IronPython/Runtime/PythonRange.cs b/Src/IronPython/Runtime/PythonRange.cs index 750791250..d5cde273d 100644 --- a/Src/IronPython/Runtime/PythonRange.cs +++ b/Src/IronPython/Runtime/PythonRange.cs @@ -83,9 +83,9 @@ public PythonTuple __reduce__() { ); } - public int __len__() { - return (int)_length; - } + public int __len__() => (int)_length; + + public bool __bool__() => _length != 0; public object this[int index] => this[(BigInteger)index]; diff --git a/Src/IronPython/Runtime/Types/TypeCache.Generated.cs b/Src/IronPython/Runtime/Types/TypeCache.Generated.cs index 31185b5e0..a1c36d561 100644 --- a/Src/IronPython/Runtime/Types/TypeCache.Generated.cs +++ b/Src/IronPython/Runtime/Types/TypeCache.Generated.cs @@ -29,6 +29,7 @@ public static class TypeCache { private static PythonType pythontype; private static PythonType str; private static PythonType bytes; + private static PythonType bytearray; private static PythonType pythontuple; private static PythonType weakreference; private static PythonType pythonlist; @@ -131,6 +132,13 @@ public static PythonType Bytes { } } + public static PythonType ByteArray { + get { + if (bytearray == null) bytearray = DynamicHelpers.GetPythonTypeFromType(typeof(ByteArray)); + return bytearray; + } + } + public static PythonType PythonTuple { get { if (pythontuple == null) pythontuple = DynamicHelpers.GetPythonTypeFromType(typeof(PythonTuple)); diff --git a/Src/Scripts/generate_typecache.py b/Src/Scripts/generate_typecache.py index 7a9a3ae47..32eca2a36 100644 --- a/Src/Scripts/generate_typecache.py +++ b/Src/Scripts/generate_typecache.py @@ -45,6 +45,7 @@ def __init__(self, type, name=None, typeType='PythonType', entryName=None): TypeData('PythonType'), TypeData('String', 'str'), TypeData('Bytes'), + TypeData('ByteArray'), TypeData('PythonTuple'), TypeData('WeakReference'), TypeData('PythonList'), diff --git a/Tests/test_codeccallbacks_stdlib.py b/Tests/test_codeccallbacks_stdlib.py index 78e31407a..920b8c568 100644 --- a/Tests/test_codeccallbacks_stdlib.py +++ b/Tests/test_codeccallbacks_stdlib.py @@ -15,7 +15,6 @@ def load_tests(loader, standard_tests, pattern): if is_ironpython: failing_tests = [ - test.test_codeccallbacks.CodecCallbackTest('test_backslashescape'), # UTF-16 vs. UTF-32 test.test_codeccallbacks.CodecCallbackTest('test_badandgoodbackslashreplaceexceptions'), # UTF-16 vs. UTF-32 test.test_codeccallbacks.CodecCallbackTest('test_badandgoodreplaceexceptions'), # UTF-16 vs. UTF-32 test.test_codeccallbacks.CodecCallbackTest('test_badandgoodxmlcharrefreplaceexceptions'), # UTF-16 vs. UTF-32 diff --git a/Tests/test_codecs_stdlib.py b/Tests/test_codecs_stdlib.py index ed66ccdd1..061dc11b9 100644 --- a/Tests/test_codecs_stdlib.py +++ b/Tests/test_codecs_stdlib.py @@ -35,8 +35,6 @@ def load_tests(loader, standard_tests, pattern): test.test_codecs.SurrogateEscapeTest('test_charmap'), # .NET iso-8859-3 decodes b'\xa5' to 'uf7f5' rather than undefined test.test_codecs.TransformCodecTest('test_custom_hex_error_is_wrapped'), # "^decoding with 'hex_codec' codec failed" does not match "Odd-length string" test.test_codecs.TransformCodecTest('test_custom_zlib_error_is_wrapped'), # "^decoding with 'zlib_codec' codec failed" does not match "Error -3 while decompressing data: incorrect header check" - test.test_codecs.TransformCodecTest('test_read'), # TypeError: expected str, got bytes - test.test_codecs.TransformCodecTest('test_readline'), # Exception: BZ_DATA_ERROR test.test_codecs.UTF7Test('test_errors'), # AssertionError: UnicodeDecodeError not raised by utf_7_decode test.test_codecs.UTF7Test('test_lone_surrogates'), # UnicodeEncodeError: 'utf_8' codec can't encode character '\ud801' in position 503: Unable to translate Unicode character \uD801 at index 503 to specified code page. ] diff --git a/Tests/test_ssl_stdlib.py b/Tests/test_ssl_stdlib.py index 0f4e928db..4f99fc8e4 100644 --- a/Tests/test_ssl_stdlib.py +++ b/Tests/test_ssl_stdlib.py @@ -66,7 +66,6 @@ def load_tests(loader, standard_tests, pattern): test.test_ssl.NetworkedTests('test_connect_cadata'), # # https://github.com/IronLanguages/ironpython3/issues/1523 test.test_ssl.NetworkedTests('test_connect_ex'), # https://github.com/IronLanguages/ironpython3/issues/1523 test.test_ssl.NetworkedTests('test_get_server_certificate'), # https://github.com/IronLanguages/ironpython3/issues/1523 - test.test_ssl.ThreadedTests('test_rude_shutdown'), # ValueError: Value does not fall within the expected range. ] skip_tests = [ diff --git a/Tests/test_struct_stdlib.py b/Tests/test_struct_stdlib.py index bd23acb06..56d99128d 100644 --- a/Tests/test_struct_stdlib.py +++ b/Tests/test_struct_stdlib.py @@ -15,7 +15,6 @@ def load_tests(loader, standard_tests, pattern): if is_ironpython: failing_tests = [ - test.test_struct.StructTest('test_705836'), # AssertionError: OverflowError not raised by pack test.test_struct.StructTest('test_bool'), # struct.error: expected bool value got IronPython.NewTypes.System.Object_1$1 test.test_struct.StructTest('test_calcsize'), # AssertionError: 4 not greater than or equal to 8 test.test_struct.StructTest('test_count_overflow'), # AssertionError: error not raised by calcsize