Backport some changes from the 3.6 branch

Src/IronPython.Modules/IterTools.cs

@@ -168,9 +168,23 @@ public PythonTuple __reduce__() {
             }
 
             public void __setstate__(PythonTuple state) {
-                // TODO: error handling?
-                ie = state[0] as IEnumerator;
-                inner = (state.Count > 1) ? state[1] as IEnumerator : null;
+                IEnumerator iter;
+                IEnumerator innerIter;
+                switch (state.Count) {
+                    case 0: throw PythonOps.TypeError("function takes at least 1 argument (0 given)");
+                    case 1:
+                        iter = state[0] as IEnumerator ?? throw PythonOps.TypeError("Arguments must be iterators.");
+                        innerIter = null;
+                        break;
+                    case 2:
+                        iter = state[0] as IEnumerator ?? throw PythonOps.TypeError("Arguments must be iterators.");
+                        innerIter = state[1] as IEnumerator ?? throw PythonOps.TypeError("Arguments must be iterators.");
+                        break;
+                    default:
+                        throw PythonOps.TypeError("function takes at most 2 argument ({0} given)", state.Count);
+                }
+                ie = iter;
+                inner = innerIter;
                 InnerEnumerator = LazyYielder();
             }
 
@@ -515,11 +529,14 @@ public void __setstate__(object state) {
             private IEnumerator<object> Yielder(IEnumerator iter) {
                 object curKey = _starterKey;
                 if (MoveNextHelper(iter)) {
+                    curKey = GetKey(iter.Current);
+                    yield return PythonTuple.MakeTuple(curKey, Grouper(iter, curKey));
+
                     while (!_fFinished) {
                         while (PythonContext.Equal(GetKey(iter.Current), curKey)) {
-                            if (!MoveNextHelper(iter)) { 
-                                _fFinished = true; 
-                                yield break; 
+                            if (!MoveNextHelper(iter)) {
+                                _fFinished = true;
+                                yield break;
                             }
                         }
                         curKey = GetKey(iter.Current);

Src/IronPython.Modules/_codecs.cs

@@ -227,6 +227,20 @@ public static PythonTuple mbcs_encode(CodeContext/*!*/ context, [NotNone] string
 
         #endregion
 
+        #region OEM Functions
+
+        [SupportedOSPlatform("windows"), PythonHidden(PlatformsAttribute.PlatformFamily.Unix)]
+        public static PythonTuple oem_decode(CodeContext/*!*/ context, [NotNone] IBufferProtocol input, string? errors = null, bool final = false) {
+            using IPythonBuffer buffer = input.GetBuffer();
+            return DoDecode(context, "oem", StringOps.CodecsInfo.OemEncoding, buffer, errors).ToPythonTuple();
+        }
+
+        [SupportedOSPlatform("windows"), PythonHidden(PlatformsAttribute.PlatformFamily.Unix)]
+        public static PythonTuple oem_encode(CodeContext/*!*/ context, [NotNone] string input, string? errors = null)
+            => DoEncode(context, "oem", StringOps.CodecsInfo.OemEncoding, input, errors).ToPythonTuple();
+
+        #endregion
+
         #region Code Page Functions
 
         [SupportedOSPlatform("windows"), PythonHidden(PlatformsAttribute.PlatformFamily.Unix)]

Src/IronPython.Modules/_collections.cs

@@ -551,20 +551,13 @@ public void __delitem__(CodeContext/*!*/ context, object index) {
                 }
             }
 
-            public PythonTuple __reduce__() {
-                lock (_lockObj) {
-                    object[] items = new object[_itemCnt];
-                    int curItem = 0;
-                    WalkDeque(delegate(int curIndex) {
-                        items[curItem++] = _data[curIndex];
-                        return true;
-                    });
-
-                    return PythonTuple.MakeTuple(
-                        DynamicHelpers.GetPythonType(this),
-                        PythonTuple.MakeTuple(PythonList.FromArrayNoCopy(items))
-                    );
-                }
+            public PythonTuple __reduce__(CodeContext context) {
+                return PythonTuple.MakeTuple(
+                    DynamicHelpers.GetPythonType(this),
+                    _maxLen == -1 ? PythonTuple.EMPTY : PythonTuple.MakeTuple(PythonTuple.EMPTY, maxlen),
+                    GetType() == typeof(deque) ? null : PythonOps.GetBoundAttr(context, this, "__dict__"),
+                    PythonOps.GetEnumeratorObject(context, this)
+                );
             }
 
             public int __len__() {

Src/IronPython.Modules/_csv.cs

@@ -259,6 +259,7 @@ public class Dialect {
             private Dialect() {
             }
 
+            [PythonHidden]
             public static Dialect Create(CodeContext/*!*/ context,
                 [ParamDictionary] IDictionary<object, object> kwArgs,
                 params object[] args) {
@@ -410,6 +411,7 @@ private static string SetString(string name, object src, bool found, string @def
                 }
                 return result;
             }
+
             #endregion
 
             public Dialect(CodeContext/*!*/ context,
@@ -503,6 +505,13 @@ public Dialect(CodeContext/*!*/ context,
                     throw PythonOps.TypeError("lineterminator must be set");
             }
 
+            // CPython defines these overloads on Dialect since 3.10
+            [Documentation("raises an exception to avoid pickling")]
+            public object __reduce__(params object[] args) => throw PythonOps.TypeError("cannot pickle 'Dialect' instances");
+
+            [Documentation("raises an exception to avoid pickling")]
+            public object __reduce_ex__(params object[] args) => throw PythonOps.TypeError("cannot pickle 'Dialect' instances");
+
             public string escapechar {
                 get { return _escapechar; }
             }

Src/IronPython.Modules/_ctypes/CFuncPtrType.cs

@@ -69,6 +69,12 @@ internal static PythonType MakeSystemType(Type underlyingSystemType) {
                 return PythonType.SetPythonType(underlyingSystemType, new CFuncPtrType(underlyingSystemType));
             }
 
+            public object from_buffer(object obj)
+                => throw PythonOps.TypeError("abstract class");
+
+            public object from_buffer_copy(object obj)
+                => throw PythonOps.TypeError("abstract class");
+
             /// <summary>
             /// Converts an object into a function call parameter.
             /// </summary>

Src/IronPython.Modules/_ctypes/NativeArgument.cs

@@ -4,12 +4,9 @@
 
 #if FEATURE_CTYPES
 
-using System;
-
-using Microsoft.Scripting.Runtime;
-
 using IronPython.Runtime;
 using IronPython.Runtime.Types;
+using IronPython.Runtime.Operations;
 
 namespace IronPython.Modules {
     /// <summary>
@@ -35,11 +32,8 @@ public CData _obj {
 
             #region ICodeFormattable Members
 
-            public string __repr__(CodeContext context) {
-                return String.Format("<cparam '{0}' ({1})>",
-                    _type,
-                    IdDispenser.GetId(__obj));// TODO: should be a real address
-            }
+            public string __repr__(CodeContext context)
+                => $"<cparam '{_type}' ({PythonOps.Repr(context, __obj)})>";
 
             #endregion
         }

Src/IronPython.Modules/_ctypes/PointerType.cs

@@ -47,6 +47,12 @@ private PointerType(Type underlyingSystemType)
                 : base(underlyingSystemType) {
             }
 
+            public object from_buffer(object obj)
+                => throw PythonOps.TypeError("abstract class");
+
+            public object from_buffer_copy(object obj)
+                => throw PythonOps.TypeError("abstract class");
+
             public object from_param([NotNone] CData obj) {
                 return new NativeArgument((CData)PythonCalls.Call(this, obj), "P");
             }

Src/IronPython.Modules/_ctypes/_ctypes.cs

@@ -101,27 +101,21 @@ private static IntPtr Cast(IntPtr data, IntPtr obj, IntPtr type) {
             GCHandle typeHandle = GCHandle.FromIntPtr(type);
             try {
                 CData cdata = objHandle.Target as CData;
-                PythonType pt = (PythonType)typeHandle.Target;
+                PythonType pt = typeHandle.Target as PythonType;
 
-                CData res = (CData)pt.CreateInstance(pt.Context.SharedContext);
-                if (IsPointer(pt)) {
-                    res.MemHolder = new MemoryHolder(IntPtr.Size);
-                    if (IsPointer(DynamicHelpers.GetPythonType(cdata))) {
-                        res.MemHolder.WriteIntPtr(0, cdata.MemHolder.ReadIntPtr(0));
-                    } else {
-                        res.MemHolder.WriteIntPtr(0, data);
-                    }
+                if (!IsPointer(pt)) throw PythonOps.TypeError("cast() argument 2 must be a pointer type, not {0}", PythonOps.GetPythonTypeName(typeHandle.Target));
 
-                    if (cdata != null) {
-                        res.MemHolder.Objects = cdata.MemHolder.Objects;
-                        res.MemHolder.AddObject(IdDispenser.GetId(cdata), cdata);
-                    }
+                CData res = (CData)pt.CreateInstance(pt.Context.SharedContext);
+                res.MemHolder = new MemoryHolder(IntPtr.Size);
+                if (IsPointer(DynamicHelpers.GetPythonType(cdata))) {
+                    res.MemHolder.WriteIntPtr(0, cdata.MemHolder.ReadIntPtr(0));
                 } else {
-                    if (cdata != null) {
-                        res.MemHolder = new MemoryHolder(data, ((INativeType)pt).Size, cdata.MemHolder);
-                    } else {
-                        res.MemHolder = new MemoryHolder(data, ((INativeType)pt).Size);
-                    }
+                    res.MemHolder.WriteIntPtr(0, data);
+                }
+
+                if (cdata != null) {
+                    res.MemHolder.Objects = cdata.MemHolder.Objects;
+                    res.MemHolder.AddObject(IdDispenser.GetId(cdata), cdata);
                 }
 
                 return GCHandle.ToIntPtr(GCHandle.Alloc(res));

Src/IronPython.Modules/_functools.cs

@@ -193,21 +193,33 @@ public void __setstate__(CodeContext context, [NotNone] PythonTuple state) {
             }
 
             public string __repr__(CodeContext context) {
-                var builder = new StringBuilder();
-                builder.Append("functools.partial(");
-                builder.Append(PythonOps.Repr(context, func));
-                foreach (var x in _args) {
-                    builder.Append(", ");
-                    builder.Append(PythonOps.Repr(context, x));
+                var infinite = PythonOps.GetAndCheckInfinite(this);
+                if (infinite == null) {
+                    return "...";
                 }
-                foreach (var p in _keywordArgs) {
-                    builder.Append(", ");
-                    builder.Append(p.Key);
-                    builder.Append('=');
-                    builder.Append(PythonOps.Repr(context, p.Value));
+
+                int infiniteIndex = infinite.Count;
+                infinite.Add(this);
+                try {
+                    var builder = new StringBuilder();
+                    builder.Append("functools.partial(");
+                    builder.Append(PythonOps.Repr(context, func));
+                    foreach (var x in _args) {
+                        builder.Append(", ");
+                        builder.Append(PythonOps.Repr(context, x));
+                    }
+                    foreach (var p in _keywordArgs) {
+                        builder.Append(", ");
+                        builder.Append(p.Key);
+                        builder.Append('=');
+                        builder.Append(PythonOps.Repr(context, p.Value));
+                    }
+                    builder.Append(')');
+                    return builder.ToString();
+                } finally {
+                    System.Diagnostics.Debug.Assert(infiniteIndex == infinite.Count - 1);
+                    infinite.RemoveAt(infiniteIndex);
                 }
-                builder.Append(')');
-                return builder.ToString();
             }
 
             #endregion

Src/IronPython.Modules/_heapq.cs

@@ -76,6 +76,7 @@ public static object heapreplace(CodeContext/*!*/ context, PythonList list, obje
             }
         }
 
+        // TODO: removed in Python 3.5
         [Documentation("Find the n largest elements in a dataset.\n\n"
             + "Equivalent to:  sorted(iterable, reverse=True)[:n]\n"
             )]
@@ -108,6 +109,7 @@ public static PythonList nlargest(CodeContext/*!*/ context, int n, object iterab
             return ret;
         }
 
+        // TODO: removed in Python 3.5
         [Documentation("Find the n smallest elements in a dataset.\n\n"
             + "Equivalent to:  sorted(iterable)[:n]\n"
             )]

Src/IronPython.Modules/_operator.cs

@@ -23,6 +23,7 @@ namespace IronPython.Modules {
     public static class PythonOperator {
         public const string __doc__ = "Provides programmatic access to various operators (addition, accessing members, etc...)";
 
+        [PythonType]
         public sealed class attrgetter : ICodeFormattable {
             private readonly object[] _names;
 
@@ -40,31 +41,32 @@ public attrgetter([NotNone] params object[] attrs) {
             public PythonTuple __reduce__() => PythonTuple.MakeTuple(DynamicHelpers.GetPythonType(this), PythonTuple.MakeTuple(_names));
 
             [SpecialName]
-            public object Call(CodeContext context, object param) {
+            public object? Call(CodeContext context, object? param) {
                 if (_names.Length == 1) {
                     return GetOneAttr(context, param, _names[0]);
                 }
 
-                object[] res = new object[_names.Length];
+                object?[] res = new object[_names.Length];
                 for (int i = 0; i < _names.Length; i++) {
                     res[i] = GetOneAttr(context, param, _names[i]);
                 }
                 return PythonTuple.MakeTuple(res);
             }
 
-            private static object GetOneAttr(CodeContext context, object param, object val) {
+            private static object? GetOneAttr(CodeContext context, object? param, object val) {
                 if (val is not string s) {
                     throw PythonOps.TypeError("attribute name must be string");
                 }
                 int dotPos = s.IndexOf('.');
                 if (dotPos >= 0) {
-                    object nextParam = GetOneAttr(context, param, s.Substring(0, dotPos));
+                    object? nextParam = GetOneAttr(context, param, s.Substring(0, dotPos));
                     return GetOneAttr(context, nextParam, s.Substring(dotPos + 1));
                 }
                 return PythonOps.GetBoundAttr(context, param, s);
             }
         }
 
+        [PythonType]
         public sealed class itemgetter : ICodeFormattable {
             private readonly object?[] _items;
 
@@ -82,7 +84,7 @@ public itemgetter([NotNone] params object?[] items) {
             public PythonTuple __reduce__() => PythonTuple.MakeTuple(DynamicHelpers.GetPythonType(this), PythonTuple.MakeTuple(_items));
 
             [SpecialName]
-            public object Call(CodeContext/*!*/ context, object param) {
+            public object? Call(CodeContext/*!*/ context, object? param) {
                 if (_items.Length == 1) {
                     return PythonOps.GetIndex(context, param, _items[0]);
                 }