From e954ac7205d7a6e356c1736eb372d2b50dbd9f69 Mon Sep 17 00:00:00 2001 From: Grant Ramsay Date: Mon, 27 Nov 2023 17:01:44 +1300 Subject: [PATCH 01/37] gh-63284: Add support for TLS-PSK (pre-shared key) to the ssl module (#103181) Add support for TLS-PSK (pre-shared key) to the ssl module. --------- Co-authored-by: Oleg Iarygin Co-authored-by: Gregory P. Smith --- Doc/library/ssl.rst | 88 +++++++ .../pycore_global_objects_fini_generated.h | 2 + Include/internal/pycore_global_strings.h | 2 + .../internal/pycore_runtime_init_generated.h | 2 + .../internal/pycore_unicodeobject_generated.h | 6 + Lib/test/test_ssl.py | 99 ++++++++ Misc/ACKS | 1 + ...3-11-27-12-41-23.gh-issue-63284.q2Qi9q.rst | 1 + Modules/_ssl.c | 224 ++++++++++++++++++ Modules/clinic/_ssl.c.h | 137 ++++++++++- 10 files changed, 561 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-11-27-12-41-23.gh-issue-63284.q2Qi9q.rst diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 21b38ae62fe02f..206294528e0016 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -2006,6 +2006,94 @@ to speed up repeated connections from the same clients. >>> ssl.create_default_context().verify_mode # doctest: +SKIP +.. method:: SSLContext.set_psk_client_callback(callback) + + Enables TLS-PSK (pre-shared key) authentication on a client-side connection. + + In general, certificate based authentication should be preferred over this method. + + The parameter ``callback`` is a callable object with the signature: + ``def callback(hint: str | None) -> tuple[str | None, bytes]``. + The ``hint`` parameter is an optional identity hint sent by the server. + The return value is a tuple in the form (client-identity, psk). + Client-identity is an optional string which may be used by the server to + select a corresponding PSK for the client. The string must be less than or + equal to ``256`` octets when UTF-8 encoded. PSK is a + :term:`bytes-like object` representing the pre-shared key. Return a zero + length PSK to reject the connection. + + Setting ``callback`` to :const:`None` removes any existing callback. + + .. note:: + When using TLS 1.3: + + - the ``hint`` parameter is always :const:`None`. + - client-identity must be a non-empty string. + + Example usage:: + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + context.maximum_version = ssl.TLSVersion.TLSv1_2 + context.set_ciphers('PSK') + + # A simple lambda: + psk = bytes.fromhex('c0ffee') + context.set_psk_client_callback(lambda hint: (None, psk)) + + # A table using the hint from the server: + psk_table = { 'ServerId_1': bytes.fromhex('c0ffee'), + 'ServerId_2': bytes.fromhex('facade') + } + def callback(hint): + return 'ClientId_1', psk_table.get(hint, b'') + context.set_psk_client_callback(callback) + + .. versionadded:: 3.13 + +.. method:: SSLContext.set_psk_server_callback(callback, identity_hint=None) + + Enables TLS-PSK (pre-shared key) authentication on a server-side connection. + + In general, certificate based authentication should be preferred over this method. + + The parameter ``callback`` is a callable object with the signature: + ``def callback(identity: str | None) -> bytes``. + The ``identity`` parameter is an optional identity sent by the client which can + be used to select a corresponding PSK. + The return value is a :term:`bytes-like object` representing the pre-shared key. + Return a zero length PSK to reject the connection. + + Setting ``callback`` to :const:`None` removes any existing callback. + + The parameter ``identity_hint`` is an optional identity hint string sent to + the client. The string must be less than or equal to ``256`` octets when + UTF-8 encoded. + + .. note:: + When using TLS 1.3 the ``identity_hint`` parameter is not sent to the client. + + Example usage:: + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.maximum_version = ssl.TLSVersion.TLSv1_2 + context.set_ciphers('PSK') + + # A simple lambda: + psk = bytes.fromhex('c0ffee') + context.set_psk_server_callback(lambda identity: psk) + + # A table using the identity of the client: + psk_table = { 'ClientId_1': bytes.fromhex('c0ffee'), + 'ClientId_2': bytes.fromhex('facade') + } + def callback(identity): + return psk_table.get(identity, b'') + context.set_psk_server_callback(callback, 'ServerId_1') + + .. versionadded:: 3.13 + .. index:: single: certificates .. index:: single: X509 certificate diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 0808076f44de31..89ec8cbbbcd649 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -826,6 +826,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(call)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(call_exception_handler)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(call_soon)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cancel)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(capath)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(category)); @@ -971,6 +972,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(id)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ident)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(identity_hint)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ignore)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(imag)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(importlib)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 8d22a9ba261010..62c3ee3ae2a0bd 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -315,6 +315,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(call) STRUCT_FOR_ID(call_exception_handler) STRUCT_FOR_ID(call_soon) + STRUCT_FOR_ID(callback) STRUCT_FOR_ID(cancel) STRUCT_FOR_ID(capath) STRUCT_FOR_ID(category) @@ -460,6 +461,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(hook) STRUCT_FOR_ID(id) STRUCT_FOR_ID(ident) + STRUCT_FOR_ID(identity_hint) STRUCT_FOR_ID(ignore) STRUCT_FOR_ID(imag) STRUCT_FOR_ID(importlib) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index d41a7478db663f..1defa39f816e78 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -824,6 +824,7 @@ extern "C" { INIT_ID(call), \ INIT_ID(call_exception_handler), \ INIT_ID(call_soon), \ + INIT_ID(callback), \ INIT_ID(cancel), \ INIT_ID(capath), \ INIT_ID(category), \ @@ -969,6 +970,7 @@ extern "C" { INIT_ID(hook), \ INIT_ID(id), \ INIT_ID(ident), \ + INIT_ID(identity_hint), \ INIT_ID(ignore), \ INIT_ID(imag), \ INIT_ID(importlib), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 0c02e902b308e3..be9baa3eebecfc 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -786,6 +786,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(call_soon); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(callback); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(cancel); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1221,6 +1224,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(ident); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(identity_hint); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(ignore); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index d8ae7b75e18150..9ade595ef8ae7e 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -4236,6 +4236,105 @@ def test_session_handling(self): self.assertEqual(str(e.exception), 'Session refers to a different SSLContext.') + @requires_tls_version('TLSv1_2') + def test_psk(self): + psk = bytes.fromhex('deadbeef') + + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + client_context.set_ciphers('PSK') + client_context.set_psk_client_callback(lambda hint: (None, psk)) + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.maximum_version = ssl.TLSVersion.TLSv1_2 + server_context.set_ciphers('PSK') + server_context.set_psk_server_callback(lambda identity: psk) + + # correct PSK should connect + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + + # incorrect PSK should fail + incorrect_psk = bytes.fromhex('cafebabe') + client_context.set_psk_client_callback(lambda hint: (None, incorrect_psk)) + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + with self.assertRaises(ssl.SSLError): + s.connect((HOST, server.port)) + + # identity_hint and client_identity should be sent to the other side + identity_hint = 'identity-hint' + client_identity = 'client-identity' + + def client_callback(hint): + self.assertEqual(hint, identity_hint) + return client_identity, psk + + def server_callback(identity): + self.assertEqual(identity, client_identity) + return psk + + client_context.set_psk_client_callback(client_callback) + server_context.set_psk_server_callback(server_callback, identity_hint) + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + + # adding client callback to server or vice versa raises an exception + with self.assertRaisesRegex(ssl.SSLError, 'Cannot add PSK server callback'): + client_context.set_psk_server_callback(server_callback, identity_hint) + with self.assertRaisesRegex(ssl.SSLError, 'Cannot add PSK client callback'): + server_context.set_psk_client_callback(client_callback) + + # test with UTF-8 identities + identity_hint = '身份暗示' # Translation: "Identity hint" + client_identity = '客户身份' # Translation: "Customer identity" + + client_context.set_psk_client_callback(client_callback) + server_context.set_psk_server_callback(server_callback, identity_hint) + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + + @requires_tls_version('TLSv1_3') + def test_psk_tls1_3(self): + psk = bytes.fromhex('deadbeef') + identity_hint = 'identity-hint' + client_identity = 'client-identity' + + def client_callback(hint): + # identity_hint is not sent to the client in TLS 1.3 + self.assertIsNone(hint) + return client_identity, psk + + def server_callback(identity): + self.assertEqual(identity, client_identity) + return psk + + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + client_context.set_ciphers('PSK') + client_context.set_psk_client_callback(client_callback) + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.minimum_version = ssl.TLSVersion.TLSv1_3 + server_context.set_ciphers('PSK') + server_context.set_psk_server_callback(server_callback, identity_hint) + + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + @unittest.skipUnless(has_tls_version('TLSv1_3'), "Test needs TLS 1.3") class TestPostHandshakeAuth(unittest.TestCase): diff --git a/Misc/ACKS b/Misc/ACKS index 6d3a4e3fdb8fe7..5fe3a177a26292 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1482,6 +1482,7 @@ Ajith Ramachandran Dhushyanth Ramasamy Ashwin Ramaswami Jeff Ramnani +Grant Ramsay Bayard Randel Varpu Rantala Brodie Rao diff --git a/Misc/NEWS.d/next/Library/2023-11-27-12-41-23.gh-issue-63284.q2Qi9q.rst b/Misc/NEWS.d/next/Library/2023-11-27-12-41-23.gh-issue-63284.q2Qi9q.rst new file mode 100644 index 00000000000000..abb57dccd5a91a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-27-12-41-23.gh-issue-63284.q2Qi9q.rst @@ -0,0 +1 @@ +Added support for TLS-PSK (pre-shared key) mode to the :mod:`ssl` module. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 7bc30cb3529d47..707e7ad9543acb 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -301,6 +301,8 @@ typedef struct { BIO *keylog_bio; /* Cached module state, also used in SSLSocket and SSLSession code. */ _sslmodulestate *state; + PyObject *psk_client_callback; + PyObject *psk_server_callback; } PySSLContext; typedef struct { @@ -3123,6 +3125,8 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) self->alpn_protocols = NULL; self->set_sni_cb = NULL; self->state = get_ssl_state(module); + self->psk_client_callback = NULL; + self->psk_server_callback = NULL; /* Don't check host name by default */ if (proto_version == PY_SSL_VERSION_TLS_CLIENT) { @@ -3235,6 +3239,8 @@ context_clear(PySSLContext *self) Py_CLEAR(self->set_sni_cb); Py_CLEAR(self->msg_cb); Py_CLEAR(self->keylog_filename); + Py_CLEAR(self->psk_client_callback); + Py_CLEAR(self->psk_server_callback); if (self->keylog_bio != NULL) { PySSL_BEGIN_ALLOW_THREADS BIO_free_all(self->keylog_bio); @@ -4662,6 +4668,222 @@ _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) return NULL; } +static unsigned int psk_client_callback(SSL *s, + const char *hint, + char *identity, + unsigned int max_identity_len, + unsigned char *psk, + unsigned int max_psk_len) +{ + PyGILState_STATE gstate = PyGILState_Ensure(); + PyObject *callback = NULL; + + PySSLSocket *ssl = SSL_get_app_data(s); + if (ssl == NULL || ssl->ctx == NULL) { + goto error; + } + callback = ssl->ctx->psk_client_callback; + if (callback == NULL) { + goto error; + } + + PyObject *hint_str = (hint != NULL && hint[0] != '\0') ? + PyUnicode_DecodeUTF8(hint, strlen(hint), "strict") : + Py_NewRef(Py_None); + if (hint_str == NULL) { + /* The remote side has sent an invalid UTF-8 string + * (breaking the standard), drop the connection without + * raising a decode exception. */ + PyErr_Clear(); + goto error; + } + PyObject *result = PyObject_CallFunctionObjArgs(callback, hint_str, NULL); + Py_DECREF(hint_str); + + if (result == NULL) { + goto error; + } + + const char *psk_; + const char *identity_; + Py_ssize_t psk_len_; + Py_ssize_t identity_len_ = 0; + if (!PyArg_ParseTuple(result, "z#y#", &identity_, &identity_len_, &psk_, &psk_len_)) { + Py_DECREF(result); + goto error; + } + + if (identity_len_ + 1 > max_identity_len || psk_len_ > max_psk_len) { + Py_DECREF(result); + goto error; + } + memcpy(psk, psk_, psk_len_); + if (identity_ != NULL) { + memcpy(identity, identity_, identity_len_); + } + identity[identity_len_] = 0; + + Py_DECREF(result); + + PyGILState_Release(gstate); + return (unsigned int)psk_len_; + +error: + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(callback); + } + PyGILState_Release(gstate); + return 0; +} + +/*[clinic input] +_ssl._SSLContext.set_psk_client_callback + callback: object + +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLContext_set_psk_client_callback_impl(PySSLContext *self, + PyObject *callback) +/*[clinic end generated code: output=0aba86f6ed75119e input=7627bae0e5ee7635]*/ +{ + if (self->protocol == PY_SSL_VERSION_TLS_SERVER) { + _setSSLError(get_state_ctx(self), + "Cannot add PSK client callback to a " + "PROTOCOL_TLS_SERVER context", 0, __FILE__, __LINE__); + return NULL; + } + + SSL_psk_client_cb_func ssl_callback; + if (callback == Py_None) { + callback = NULL; + // Delete the existing callback + ssl_callback = NULL; + } else { + if (!PyCallable_Check(callback)) { + PyErr_SetString(PyExc_TypeError, "callback must be callable"); + return NULL; + } + ssl_callback = psk_client_callback; + } + + Py_XDECREF(self->psk_client_callback); + Py_XINCREF(callback); + + self->psk_client_callback = callback; + SSL_CTX_set_psk_client_callback(self->ctx, ssl_callback); + + Py_RETURN_NONE; +} + +static unsigned int psk_server_callback(SSL *s, + const char *identity, + unsigned char *psk, + unsigned int max_psk_len) +{ + PyGILState_STATE gstate = PyGILState_Ensure(); + PyObject *callback = NULL; + + PySSLSocket *ssl = SSL_get_app_data(s); + if (ssl == NULL || ssl->ctx == NULL) { + goto error; + } + callback = ssl->ctx->psk_server_callback; + if (callback == NULL) { + goto error; + } + + PyObject *identity_str = (identity != NULL && identity[0] != '\0') ? + PyUnicode_DecodeUTF8(identity, strlen(identity), "strict") : + Py_NewRef(Py_None); + if (identity_str == NULL) { + /* The remote side has sent an invalid UTF-8 string + * (breaking the standard), drop the connection without + * raising a decode exception. */ + PyErr_Clear(); + goto error; + } + PyObject *result = PyObject_CallFunctionObjArgs(callback, identity_str, NULL); + Py_DECREF(identity_str); + + if (result == NULL) { + goto error; + } + + char *psk_; + Py_ssize_t psk_len_; + if (PyBytes_AsStringAndSize(result, &psk_, &psk_len_) < 0) { + Py_DECREF(result); + goto error; + } + + if (psk_len_ > max_psk_len) { + Py_DECREF(result); + goto error; + } + memcpy(psk, psk_, psk_len_); + + Py_DECREF(result); + + PyGILState_Release(gstate); + return (unsigned int)psk_len_; + +error: + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(callback); + } + PyGILState_Release(gstate); + return 0; +} + +/*[clinic input] +_ssl._SSLContext.set_psk_server_callback + callback: object + identity_hint: str(accept={str, NoneType}) = None + +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLContext_set_psk_server_callback_impl(PySSLContext *self, + PyObject *callback, + const char *identity_hint) +/*[clinic end generated code: output=1f4d6a4e09a92b03 input=65d4b6022aa85ea3]*/ +{ + if (self->protocol == PY_SSL_VERSION_TLS_CLIENT) { + _setSSLError(get_state_ctx(self), + "Cannot add PSK server callback to a " + "PROTOCOL_TLS_CLIENT context", 0, __FILE__, __LINE__); + return NULL; + } + + SSL_psk_server_cb_func ssl_callback; + if (callback == Py_None) { + callback = NULL; + // Delete the existing callback and hint + ssl_callback = NULL; + identity_hint = NULL; + } else { + if (!PyCallable_Check(callback)) { + PyErr_SetString(PyExc_TypeError, "callback must be callable"); + return NULL; + } + ssl_callback = psk_server_callback; + } + + if (SSL_CTX_use_psk_identity_hint(self->ctx, identity_hint) != 1) { + PyErr_SetString(PyExc_ValueError, "failed to set identity hint"); + return NULL; + } + + Py_XDECREF(self->psk_server_callback); + Py_XINCREF(callback); + + self->psk_server_callback = callback; + SSL_CTX_set_psk_server_callback(self->ctx, ssl_callback); + + Py_RETURN_NONE; +} + static PyGetSetDef context_getsetlist[] = { {"check_hostname", (getter) get_check_hostname, @@ -4716,6 +4938,8 @@ static struct PyMethodDef context_methods[] = { _SSL__SSLCONTEXT_CERT_STORE_STATS_METHODDEF _SSL__SSLCONTEXT_GET_CA_CERTS_METHODDEF _SSL__SSLCONTEXT_GET_CIPHERS_METHODDEF + _SSL__SSLCONTEXT_SET_PSK_CLIENT_CALLBACK_METHODDEF + _SSL__SSLCONTEXT_SET_PSK_SERVER_CALLBACK_METHODDEF {NULL, NULL} /* sentinel */ }; diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 88401b0490a1bb..19c0f619b92f45 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -1014,6 +1014,141 @@ _ssl__SSLContext_get_ca_certs(PySSLContext *self, PyObject *const *args, Py_ssiz return return_value; } +PyDoc_STRVAR(_ssl__SSLContext_set_psk_client_callback__doc__, +"set_psk_client_callback($self, /, callback)\n" +"--\n" +"\n"); + +#define _SSL__SSLCONTEXT_SET_PSK_CLIENT_CALLBACK_METHODDEF \ + {"set_psk_client_callback", _PyCFunction_CAST(_ssl__SSLContext_set_psk_client_callback), METH_FASTCALL|METH_KEYWORDS, _ssl__SSLContext_set_psk_client_callback__doc__}, + +static PyObject * +_ssl__SSLContext_set_psk_client_callback_impl(PySSLContext *self, + PyObject *callback); + +static PyObject * +_ssl__SSLContext_set_psk_client_callback(PySSLContext *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(callback), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"callback", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set_psk_client_callback", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *callback; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + callback = args[0]; + return_value = _ssl__SSLContext_set_psk_client_callback_impl(self, callback); + +exit: + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLContext_set_psk_server_callback__doc__, +"set_psk_server_callback($self, /, callback, identity_hint=None)\n" +"--\n" +"\n"); + +#define _SSL__SSLCONTEXT_SET_PSK_SERVER_CALLBACK_METHODDEF \ + {"set_psk_server_callback", _PyCFunction_CAST(_ssl__SSLContext_set_psk_server_callback), METH_FASTCALL|METH_KEYWORDS, _ssl__SSLContext_set_psk_server_callback__doc__}, + +static PyObject * +_ssl__SSLContext_set_psk_server_callback_impl(PySSLContext *self, + PyObject *callback, + const char *identity_hint); + +static PyObject * +_ssl__SSLContext_set_psk_server_callback(PySSLContext *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(callback), &_Py_ID(identity_hint), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"callback", "identity_hint", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set_psk_server_callback", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *callback; + const char *identity_hint = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + callback = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1] == Py_None) { + identity_hint = NULL; + } + else if (PyUnicode_Check(args[1])) { + Py_ssize_t identity_hint_length; + identity_hint = PyUnicode_AsUTF8AndSize(args[1], &identity_hint_length); + if (identity_hint == NULL) { + goto exit; + } + if (strlen(identity_hint) != (size_t)identity_hint_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + } + else { + _PyArg_BadArgument("set_psk_server_callback", "argument 'identity_hint'", "str or None", args[1]); + goto exit; + } +skip_optional_pos: + return_value = _ssl__SSLContext_set_psk_server_callback_impl(self, callback, identity_hint); + +exit: + return return_value; +} + static PyObject * _ssl_MemoryBIO_impl(PyTypeObject *type); @@ -1527,4 +1662,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=aa6b0a898b6077fe input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6342ea0062ab16c7 input=a9049054013a1b77]*/ From 0622839cfedacbb48eba27180fd0f0586fe97771 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 27 Nov 2023 08:19:29 +0000 Subject: [PATCH 02/37] gh-112414: Fix `AttributeError` when calling `repr()` on a namespace package imported with a custom loader (#112425) --- Lib/importlib/_bootstrap.py | 10 ++++++++-- Lib/test/test_importlib/import_/test___loader__.py | 4 ++++ Lib/test/test_importlib/test_namespace_pkgs.py | 2 +- .../2023-11-26-13-44-19.gh-issue-112414.kx2E7S.rst | 3 +++ 4 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-11-26-13-44-19.gh-issue-112414.kx2E7S.rst diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index ec2e56f6ea9ca1..d942045f3de666 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -824,10 +824,16 @@ def _module_repr_from_spec(spec): """Return the repr to use for the module.""" name = '?' if spec.name is None else spec.name if spec.origin is None: - if spec.loader is None: + loader = spec.loader + if loader is None: return f'' + elif ( + _bootstrap_external is not None + and isinstance(loader, _bootstrap_external.NamespaceLoader) + ): + return f'' else: - return f'' + return f'' else: if spec.has_location: return f'' diff --git a/Lib/test/test_importlib/import_/test___loader__.py b/Lib/test/test_importlib/import_/test___loader__.py index 858b37effc64bd..c6996a42534676 100644 --- a/Lib/test/test_importlib/import_/test___loader__.py +++ b/Lib/test/test_importlib/import_/test___loader__.py @@ -23,6 +23,10 @@ def test___loader__(self): with util.uncache('blah'), util.import_state(meta_path=[loader]): module = self.__import__('blah') self.assertEqual(loader, module.__loader__) + expected_repr_pattern = ( + r"\)>" + ) + self.assertRegex(repr(module), expected_repr_pattern) (Frozen_SpecTests, diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py index 9b3bef02c66820..072e198795d394 100644 --- a/Lib/test/test_importlib/test_namespace_pkgs.py +++ b/Lib/test/test_importlib/test_namespace_pkgs.py @@ -80,7 +80,7 @@ def test_cant_import_other(self): def test_simple_repr(self): import foo.one - assert repr(foo).startswith(" Date: Mon, 27 Nov 2023 22:27:47 +1300 Subject: [PATCH 03/37] Docs: fix typo in doc for sqlite3.Cursor.execute (#112442) --- Doc/library/sqlite3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index a5b3474f4bd39a..6dbb34a84a4c40 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1502,7 +1502,7 @@ Cursor objects .. method:: execute(sql, parameters=(), /) - Execute SQL a single SQL statement, + Execute a single SQL statement, optionally binding Python values using :ref:`placeholders `. From 7ac49e74c3db35365968cd2cbd395cf063d2050d Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 27 Nov 2023 13:01:26 +0300 Subject: [PATCH 04/37] gh-111147: Fix `test_set_of_sets_reprs` in `test_pprint` (GH-111148) Make it stable and not depending on implementation details. --- Lib/test/test_pprint.py | 292 +++++++++++++--------------------------- 1 file changed, 95 insertions(+), 197 deletions(-) diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py index 4d1766fb16ce9f..4e6fed1ab969ac 100644 --- a/Lib/test/test_pprint.py +++ b/Lib/test/test_pprint.py @@ -9,7 +9,6 @@ import random import re import test.support -import test.test_set import types import unittest @@ -623,9 +622,6 @@ def test_set_reprs(self): self.assertEqual(pprint.pformat(frozenset3(range(7)), width=20), 'frozenset3({0, 1, 2, 3, 4, 5, 6})') - @unittest.expectedFailure - #See http://bugs.python.org/issue13907 - @test.support.cpython_only def test_set_of_sets_reprs(self): # This test creates a complex arrangement of frozensets and # compares the pretty-printed repr against a string hard-coded in @@ -636,204 +632,106 @@ def test_set_of_sets_reprs(self): # partial ordering (subset relationships), the output of the # list.sort() method is undefined for lists of sets." # - # In a nutshell, the test assumes frozenset({0}) will always - # sort before frozenset({1}), but: - # # >>> frozenset({0}) < frozenset({1}) # False # >>> frozenset({1}) < frozenset({0}) # False # - # Consequently, this test is fragile and - # implementation-dependent. Small changes to Python's sort - # algorithm cause the test to fail when it should pass. - # XXX Or changes to the dictionary implementation... - - cube_repr_tgt = """\ -{frozenset(): frozenset({frozenset({2}), frozenset({0}), frozenset({1})}), - frozenset({0}): frozenset({frozenset(), - frozenset({0, 2}), - frozenset({0, 1})}), - frozenset({1}): frozenset({frozenset(), - frozenset({1, 2}), - frozenset({0, 1})}), - frozenset({2}): frozenset({frozenset(), - frozenset({1, 2}), - frozenset({0, 2})}), - frozenset({1, 2}): frozenset({frozenset({2}), - frozenset({1}), - frozenset({0, 1, 2})}), - frozenset({0, 2}): frozenset({frozenset({2}), - frozenset({0}), - frozenset({0, 1, 2})}), - frozenset({0, 1}): frozenset({frozenset({0}), - frozenset({1}), - frozenset({0, 1, 2})}), - frozenset({0, 1, 2}): frozenset({frozenset({1, 2}), - frozenset({0, 2}), - frozenset({0, 1})})}""" - cube = test.test_set.cube(3) - self.assertEqual(pprint.pformat(cube), cube_repr_tgt) - cubo_repr_tgt = """\ -{frozenset({frozenset({0, 2}), frozenset({0})}): frozenset({frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset({2}), - frozenset({0, - 2})})}), - frozenset({frozenset({0, 1}), frozenset({1})}): frozenset({frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset({1}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({1})})}), - frozenset({frozenset({1, 2}), frozenset({1})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({1})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({1, 2}), frozenset({2})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({1}), - frozenset({1, - 2})}), - frozenset({frozenset({2}), - frozenset({0, - 2})}), - frozenset({frozenset(), - frozenset({2})})}), - frozenset({frozenset(), frozenset({0})}): frozenset({frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset(), - frozenset({1})}), - frozenset({frozenset(), - frozenset({2})})}), - frozenset({frozenset(), frozenset({1})}): frozenset({frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset({1}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({2})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({2}), frozenset()}): frozenset({frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset(), - frozenset({1})}), - frozenset({frozenset({2}), - frozenset({0, - 2})})}), - frozenset({frozenset({0, 1, 2}), frozenset({0, 1})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({0}), frozenset({0, 1})}): frozenset({frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({2}), frozenset({0, 2})}): frozenset({frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset(), - frozenset({2})})}), - frozenset({frozenset({0, 1, 2}), frozenset({0, 2})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset({2}), - frozenset({0, - 2})})}), - frozenset({frozenset({1, 2}), frozenset({0, 1, 2})}): frozenset({frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset({1}), - frozenset({1, - 2})})})}""" - - cubo = test.test_set.linegraph(cube) - self.assertEqual(pprint.pformat(cubo), cubo_repr_tgt) + # In this test we list all possible invariants of the result + # for unordered frozensets. + # + # This test has a long history, see: + # - https://github.com/python/cpython/commit/969fe57baa0eb80332990f9cda936a33e13fabef + # - https://github.com/python/cpython/issues/58115 + # - https://github.com/python/cpython/issues/111147 + + import textwrap + + # Single-line, always ordered: + fs0 = frozenset() + fs1 = frozenset(('abc', 'xyz')) + data = frozenset((fs0, fs1)) + self.assertEqual(pprint.pformat(data), + 'frozenset({%r, %r})' % (fs0, fs1)) + self.assertEqual(pprint.pformat(data), repr(data)) + + fs2 = frozenset(('one', 'two')) + data = {fs2: frozenset((fs0, fs1))} + self.assertEqual(pprint.pformat(data), + "{%r: frozenset({%r, %r})}" % (fs2, fs0, fs1)) + self.assertEqual(pprint.pformat(data), repr(data)) + + # Single-line, unordered: + fs1 = frozenset(("xyz", "qwerty")) + fs2 = frozenset(("abcd", "spam")) + fs = frozenset((fs1, fs2)) + self.assertEqual(pprint.pformat(fs), repr(fs)) + + # Multiline, unordered: + def check(res, invariants): + self.assertIn(res, [textwrap.dedent(i).strip() for i in invariants]) + + # Inner-most frozensets are singleline, result is multiline, unordered: + fs1 = frozenset(('regular string', 'other string')) + fs2 = frozenset(('third string', 'one more string')) + check( + pprint.pformat(frozenset((fs1, fs2))), + [ + """ + frozenset({%r, + %r}) + """ % (fs1, fs2), + """ + frozenset({%r, + %r}) + """ % (fs2, fs1), + ], + ) + + # Everything is multiline, unordered: + check( + pprint.pformat( + frozenset(( + frozenset(( + "xyz very-very long string", + "qwerty is also absurdly long", + )), + frozenset(( + "abcd is even longer that before", + "spam is not so long", + )), + )), + ), + [ + """ + frozenset({frozenset({'abcd is even longer that before', + 'spam is not so long'}), + frozenset({'qwerty is also absurdly long', + 'xyz very-very long string'})}) + """, + + """ + frozenset({frozenset({'abcd is even longer that before', + 'spam is not so long'}), + frozenset({'xyz very-very long string', + 'qwerty is also absurdly long'})}) + """, + + """ + frozenset({frozenset({'qwerty is also absurdly long', + 'xyz very-very long string'}), + frozenset({'abcd is even longer that before', + 'spam is not so long'})}) + """, + + """ + frozenset({frozenset({'qwerty is also absurdly long', + 'xyz very-very long string'}), + frozenset({'spam is not so long', + 'abcd is even longer that before'})}) + """, + ], + ) def test_depth(self): nested_tuple = (1, (2, (3, (4, (5, 6))))) From d44ee42cd7c9a40e1d7096b95476fe47156f571f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 27 Nov 2023 12:55:52 +0100 Subject: [PATCH 05/37] Move What's New In Python 3.12 entries to the right section (#112447) Jython and ctypes removals are unrelated to C API Removals. --- Doc/whatsnew/3.12.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 8b7a043d068e5c..a4b3a6d12970b4 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1773,6 +1773,14 @@ Others (*ssl_context* in :mod:`imaplib`) instead. (Contributed by Victor Stinner in :gh:`94172`.) +* Remove ``Jython`` compatibility hacks from several stdlib modules and tests. + (Contributed by Nikita Sobolev in :gh:`99482`.) + +* Remove ``_use_broken_old_ctypes_structure_semantics_`` flag + from :mod:`ctypes` module. + (Contributed by Nikita Sobolev in :gh:`99285`.) + + .. _whatsnew312-porting-to-python312: Porting to Python 3.12 @@ -2424,10 +2432,3 @@ Removed * Remove the ``PyUnicode_InternImmortal()`` function macro. (Contributed by Victor Stinner in :gh:`85858`.) - -* Remove ``Jython`` compatibility hacks from several stdlib modules and tests. - (Contributed by Nikita Sobolev in :gh:`99482`.) - -* Remove ``_use_broken_old_ctypes_structure_semantics_`` flag - from :mod:`ctypes` module. - (Contributed by Nikita Sobolev in :gh:`99285`.) From ffe1b2d07b88e185f373ad696fbea5a7f2a315c1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 27 Nov 2023 14:36:54 +0000 Subject: [PATCH 06/37] GH-101100: Fix reference warnings for ``socket`` methods (#110114) Co-authored-by: Serhiy Storchaka --- Doc/library/socket.rst | 24 ++++++++++++------------ Doc/whatsnew/2.0.rst | 6 +++--- Doc/whatsnew/2.7.rst | 4 ++-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 9ff1aa3984e828..e36fc17f89de24 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -23,7 +23,7 @@ all modern Unix systems, Windows, MacOS, and probably additional platforms. The Python interface is a straightforward transliteration of the Unix system call and library interface for sockets to Python's object-oriented style: the -:func:`.socket` function returns a :dfn:`socket object` whose methods implement +:func:`~socket.socket` function returns a :dfn:`socket object` whose methods implement the various socket system calls. Parameter types are somewhat higher-level than in the C interface: as with :meth:`read` and :meth:`write` operations on Python files, buffer allocation on receive operations is automatic, and buffer length @@ -348,7 +348,7 @@ Constants AF_INET6 These constants represent the address (and protocol) families, used for the - first argument to :func:`.socket`. If the :const:`AF_UNIX` constant is not + first argument to :func:`~socket.socket`. If the :const:`AF_UNIX` constant is not defined then this protocol is unsupported. More constants may be available depending on the system. @@ -365,7 +365,7 @@ Constants SOCK_SEQPACKET These constants represent the socket types, used for the second argument to - :func:`.socket`. More constants may be available depending on the system. + :func:`~socket.socket`. More constants may be available depending on the system. (Only :const:`SOCK_STREAM` and :const:`SOCK_DGRAM` appear to be generally useful.) @@ -404,7 +404,7 @@ Constants Many constants of these forms, documented in the Unix documentation on sockets and/or the IP protocol, are also defined in the socket module. They are - generally used in arguments to the :meth:`setsockopt` and :meth:`getsockopt` + generally used in arguments to the :meth:`~socket.setsockopt` and :meth:`~socket.getsockopt` methods of socket objects. In most cases, only those symbols that are defined in the Unix header files are defined; for a few symbols, default values are provided. @@ -770,7 +770,7 @@ The following functions all create :ref:`socket objects `. Build a pair of connected socket objects using the given address family, socket type, and protocol number. Address family, socket type, and protocol number are - as for the :func:`.socket` function above. The default family is :const:`AF_UNIX` + as for the :func:`~socket.socket` function above. The default family is :const:`AF_UNIX` if defined on the platform; otherwise, the default is :const:`AF_INET`. The newly created sockets are :ref:`non-inheritable `. @@ -866,7 +866,7 @@ The following functions all create :ref:`socket objects `. Duplicate the file descriptor *fd* (an integer as returned by a file object's :meth:`~io.IOBase.fileno` method) and build a socket object from the result. Address - family, socket type and protocol number are as for the :func:`.socket` function + family, socket type and protocol number are as for the :func:`~socket.socket` function above. The file descriptor should refer to a socket, but this is not checked --- subsequent operations on the object may fail if the file descriptor is invalid. This function is rarely needed, but can be used to get or set socket options on @@ -931,7 +931,7 @@ The :mod:`socket` module also offers various network-related services: ``(family, type, proto, canonname, sockaddr)`` In these tuples, *family*, *type*, *proto* are all integers and are - meant to be passed to the :func:`.socket` function. *canonname* will be + meant to be passed to the :func:`~socket.socket` function. *canonname* will be a string representing the canonical name of the *host* if :const:`AI_CANONNAME` is part of the *flags* argument; else *canonname* will be empty. *sockaddr* is a tuple describing a socket address, whose @@ -1047,7 +1047,7 @@ The :mod:`socket` module also offers various network-related services: .. function:: getprotobyname(protocolname) Translate an internet protocol name (for example, ``'icmp'``) to a constant - suitable for passing as the (optional) third argument to the :func:`.socket` + suitable for passing as the (optional) third argument to the :func:`~socket.socket` function. This is usually only needed for sockets opened in "raw" mode (:const:`SOCK_RAW`); for the normal socket modes, the correct protocol is chosen automatically if the protocol is omitted or zero. @@ -1331,7 +1331,7 @@ The :mod:`socket` module also offers various network-related services: Send the list of file descriptors *fds* over an :const:`AF_UNIX` socket *sock*. The *fds* parameter is a sequence of file descriptors. - Consult :meth:`sendmsg` for the documentation of these parameters. + Consult :meth:`~socket.sendmsg` for the documentation of these parameters. .. availability:: Unix, Windows, not Emscripten, not WASI. @@ -1345,7 +1345,7 @@ The :mod:`socket` module also offers various network-related services: Receive up to *maxfds* file descriptors from an :const:`AF_UNIX` socket *sock*. Return ``(msg, list(fds), flags, addr)``. - Consult :meth:`recvmsg` for the documentation of these parameters. + Consult :meth:`~socket.recvmsg` for the documentation of these parameters. .. availability:: Unix, Windows, not Emscripten, not WASI. @@ -2064,10 +2064,10 @@ Example Here are four minimal example programs using the TCP/IP protocol: a server that echoes all data that it receives back (servicing only one client), and a client -using it. Note that a server must perform the sequence :func:`.socket`, +using it. Note that a server must perform the sequence :func:`~socket.socket`, :meth:`~socket.bind`, :meth:`~socket.listen`, :meth:`~socket.accept` (possibly repeating the :meth:`~socket.accept` to service more than one client), while a -client only needs the sequence :func:`.socket`, :meth:`~socket.connect`. Also +client only needs the sequence :func:`~socket.socket`, :meth:`~socket.connect`. Also note that the server does not :meth:`~socket.sendall`/:meth:`~socket.recv` on the socket it is listening on but on the new socket returned by :meth:`~socket.accept`. diff --git a/Doc/whatsnew/2.0.rst b/Doc/whatsnew/2.0.rst index c2b0ae8c76302a..6d6e51006d5bd8 100644 --- a/Doc/whatsnew/2.0.rst +++ b/Doc/whatsnew/2.0.rst @@ -671,9 +671,9 @@ errors. If you absolutely must use 2.0 but can't fix your code, you can edit ``NO_STRICT_LIST_APPEND`` to preserve the old behaviour; this isn't recommended. Some of the functions in the :mod:`socket` module are still forgiving in this -way. For example, :func:`socket.connect( ('hostname', 25) )` is the correct -form, passing a tuple representing an IP address, but :func:`socket.connect( -'hostname', 25 )` also works. :func:`socket.connect_ex` and :func:`socket.bind` +way. For example, ``socket.connect( ('hostname', 25) )`` is the correct +form, passing a tuple representing an IP address, but ``socket.connect('hostname', 25)`` +also works. :meth:`socket.connect_ex ` and :meth:`socket.bind ` are similarly easy-going. 2.0alpha1 tightened these functions up, but because the documentation actually used the erroneous multiple argument form, many people wrote code which would break with the stricter checking. GvR backed out diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 61934ab1a2df56..da66dd731831bc 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -2383,8 +2383,8 @@ Port-Specific Changes: Mac OS X Port-Specific Changes: FreeBSD ----------------------------------- -* FreeBSD 7.1's :const:`SO_SETFIB` constant, used with - :func:`~socket.getsockopt`/:func:`~socket.setsockopt` to select an +* FreeBSD 7.1's :const:`SO_SETFIB` constant, used with the :func:`~socket.socket` methods + :func:`~socket.socket.getsockopt`/:func:`~socket.socket.setsockopt` to select an alternate routing table, is now available in the :mod:`socket` module. (Added by Kyle VanderBeek; :issue:`8235`.) From 22e411e1d107f79a0904d41a489a82355a39b5de Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 27 Nov 2023 16:34:44 +0000 Subject: [PATCH 07/37] gh-111874: Call `__set_name__` on objects that define the method inside a `typing.NamedTuple` class dictionary as part of the creation of that class (#111876) Co-authored-by: Jelle Zijlstra --- Lib/test/test_typing.py | 77 +++++++++++++++++++ Lib/typing.py | 21 ++++- ...-11-09-11-07-34.gh-issue-111874.dzYc3j.rst | 4 + 3 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-11-09-11-07-34.gh-issue-111874.dzYc3j.rst diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 31d7fda2f978da..669803177315e3 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -7535,6 +7535,83 @@ class GenericNamedTuple(NamedTuple, Generic[T]): self.assertEqual(CallNamedTuple.__orig_bases__, (NamedTuple,)) + def test_setname_called_on_values_in_class_dictionary(self): + class Vanilla: + def __set_name__(self, owner, name): + self.name = name + + class Foo(NamedTuple): + attr = Vanilla() + + foo = Foo() + self.assertEqual(len(foo), 0) + self.assertNotIn('attr', Foo._fields) + self.assertIsInstance(foo.attr, Vanilla) + self.assertEqual(foo.attr.name, "attr") + + class Bar(NamedTuple): + attr: Vanilla = Vanilla() + + bar = Bar() + self.assertEqual(len(bar), 1) + self.assertIn('attr', Bar._fields) + self.assertIsInstance(bar.attr, Vanilla) + self.assertEqual(bar.attr.name, "attr") + + def test_setname_raises_the_same_as_on_other_classes(self): + class CustomException(BaseException): pass + + class Annoying: + def __set_name__(self, owner, name): + raise CustomException + + annoying = Annoying() + + with self.assertRaises(CustomException) as cm: + class NormalClass: + attr = annoying + normal_exception = cm.exception + + with self.assertRaises(CustomException) as cm: + class NamedTupleClass(NamedTuple): + attr = annoying + namedtuple_exception = cm.exception + + self.assertIs(type(namedtuple_exception), CustomException) + self.assertIs(type(namedtuple_exception), type(normal_exception)) + + self.assertEqual(len(namedtuple_exception.__notes__), 1) + self.assertEqual( + len(namedtuple_exception.__notes__), len(normal_exception.__notes__) + ) + + expected_note = ( + "Error calling __set_name__ on 'Annoying' instance " + "'attr' in 'NamedTupleClass'" + ) + self.assertEqual(namedtuple_exception.__notes__[0], expected_note) + self.assertEqual( + namedtuple_exception.__notes__[0], + normal_exception.__notes__[0].replace("NormalClass", "NamedTupleClass") + ) + + def test_strange_errors_when_accessing_set_name_itself(self): + class CustomException(Exception): pass + + class Meta(type): + def __getattribute__(self, attr): + if attr == "__set_name__": + raise CustomException + return object.__getattribute__(self, attr) + + class VeryAnnoying(metaclass=Meta): pass + + very_annoying = VeryAnnoying() + + with self.assertRaises(CustomException): + class Foo(NamedTuple): + attr = very_annoying + class TypedDictTests(BaseTestCase): def test_basics_functional_syntax(self): diff --git a/Lib/typing.py b/Lib/typing.py index 872aca82c4e779..216f0c141b62af 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2743,11 +2743,26 @@ def __new__(cls, typename, bases, ns): class_getitem = _generic_class_getitem nm_tpl.__class_getitem__ = classmethod(class_getitem) # update from user namespace without overriding special namedtuple attributes - for key in ns: + for key, val in ns.items(): if key in _prohibited: raise AttributeError("Cannot overwrite NamedTuple attribute " + key) - elif key not in _special and key not in nm_tpl._fields: - setattr(nm_tpl, key, ns[key]) + elif key not in _special: + if key not in nm_tpl._fields: + setattr(nm_tpl, key, val) + try: + set_name = type(val).__set_name__ + except AttributeError: + pass + else: + try: + set_name(val, nm_tpl, key) + except BaseException as e: + e.add_note( + f"Error calling __set_name__ on {type(val).__name__!r} " + f"instance {key!r} in {typename!r}" + ) + raise + if Generic in bases: nm_tpl.__init_subclass__() return nm_tpl diff --git a/Misc/NEWS.d/next/Library/2023-11-09-11-07-34.gh-issue-111874.dzYc3j.rst b/Misc/NEWS.d/next/Library/2023-11-09-11-07-34.gh-issue-111874.dzYc3j.rst new file mode 100644 index 00000000000000..50408202a7a5a1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-09-11-07-34.gh-issue-111874.dzYc3j.rst @@ -0,0 +1,4 @@ +When creating a :class:`typing.NamedTuple` class, ensure +:func:`~object.__set_name__` is called on all objects that define +``__set_name__`` and exist in the values of the ``NamedTuple`` class's class +dictionary. Patch by Alex Waygood. From 812360fddda86d7aff5823f529ab720f57ddc411 Mon Sep 17 00:00:00 2001 From: Zackery Spytz Date: Mon, 27 Nov 2023 09:15:39 -0800 Subject: [PATCH 08/37] gh-84443: SSLSocket.recv_into() now support buffer protocol with itemsize != 1 (GH-20310) It is also no longer use __len__(). Co-authored-by: Serhiy Storchaka --- Lib/ssl.py | 12 ++++++---- Lib/test/test_ssl.py | 22 +++++++++++++++++++ .../2020-05-21-23-32-46.bpo-40262.z4fQv1.rst | 2 ++ 3 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-05-21-23-32-46.bpo-40262.z4fQv1.rst diff --git a/Lib/ssl.py b/Lib/ssl.py index 62e55857141dfc..36fca9d4aa93d1 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -1270,10 +1270,14 @@ def recv(self, buflen=1024, flags=0): def recv_into(self, buffer, nbytes=None, flags=0): self._checkClosed() - if buffer and (nbytes is None): - nbytes = len(buffer) - elif nbytes is None: - nbytes = 1024 + if nbytes is None: + if buffer is not None: + with memoryview(buffer) as view: + nbytes = view.nbytes + if not nbytes: + nbytes = 1024 + else: + nbytes = 1024 if self._sslobj is not None: if flags != 0: raise ValueError( diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 9ade595ef8ae7e..aecba89cde1495 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -10,6 +10,7 @@ from test.support import threading_helper from test.support import warnings_helper from test.support import asyncore +import array import re import socket import select @@ -3517,6 +3518,27 @@ def test_recv_zero(self): self.assertEqual(s.recv(0), b"") self.assertEqual(s.recv_into(bytearray()), 0) + def test_recv_into_buffer_protocol_len(self): + server = ThreadedEchoServer(CERTFILE) + self.enterContext(server) + s = socket.create_connection((HOST, server.port)) + self.addCleanup(s.close) + s = test_wrap_socket(s, suppress_ragged_eofs=False) + self.addCleanup(s.close) + + s.send(b"data") + buf = array.array('I', [0, 0]) + self.assertEqual(s.recv_into(buf), 4) + self.assertEqual(bytes(buf)[:4], b"data") + + class B(bytearray): + def __len__(self): + 1/0 + s.send(b"data") + buf = B(6) + self.assertEqual(s.recv_into(buf), 4) + self.assertEqual(bytes(buf), b"data\0\0") + def test_nonblocking_send(self): server = ThreadedEchoServer(CERTFILE, certreqs=ssl.CERT_NONE, diff --git a/Misc/NEWS.d/next/Library/2020-05-21-23-32-46.bpo-40262.z4fQv1.rst b/Misc/NEWS.d/next/Library/2020-05-21-23-32-46.bpo-40262.z4fQv1.rst new file mode 100644 index 00000000000000..c017a1c8df09d8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-05-21-23-32-46.bpo-40262.z4fQv1.rst @@ -0,0 +1,2 @@ +The :meth:`ssl.SSLSocket.recv_into` method no longer requires the *buffer* +argument to implement ``__len__`` and supports buffers with arbitrary item size. From 4eea1e82369fbf7a795d1956e7a8212a1b58009f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 27 Nov 2023 19:32:55 +0200 Subject: [PATCH 09/37] gh-112438: Fix support of format units with the "e" prefix in nested tuples in PyArg_Parse (gh-112439) --- Lib/test/test_capi/test_getargs.py | 28 +++++++++++++++++++ ...-11-27-09-44-16.gh-issue-112438.GdNZiI.rst | 2 ++ Modules/_testcapi/getargs.c | 14 ++++++---- Python/getargs.c | 2 +- 4 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-11-27-09-44-16.gh-issue-112438.GdNZiI.rst diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index c964b1efd577ba..9b6aef27625ad0 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -1314,6 +1314,34 @@ def test_nonascii_keywords(self): f"'{name2}' is an invalid keyword argument"): parse((), {name2: 1, name3: 2}, '|OO', [name, name3]) + def test_nested_tuple(self): + parse = _testcapi.parse_tuple_and_keywords + + self.assertEqual(parse(((1, 2, 3),), {}, '(OOO)', ['a']), (1, 2, 3)) + self.assertEqual(parse((1, (2, 3), 4), {}, 'O(OO)O', ['a', 'b', 'c']), + (1, 2, 3, 4)) + parse(((1, 2, 3),), {}, '(iii)', ['a']) + + with self.assertRaisesRegex(TypeError, + "argument 1 must be sequence of length 2, not 3"): + parse(((1, 2, 3),), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be sequence of length 2, not 1"): + parse(((1,),), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be 2-item sequence, not int"): + parse((1,), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be 2-item sequence, not bytes"): + parse((b'ab',), {}, '(ii)', ['a']) + + for f in 'es', 'et', 'es#', 'et#': + with self.assertRaises(LookupError): # empty encoding "" + parse((('a',),), {}, '(' + f + ')', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be sequence of length 1, not 0"): + parse(((),), {}, '(' + f + ')', ['a']) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/C API/2023-11-27-09-44-16.gh-issue-112438.GdNZiI.rst b/Misc/NEWS.d/next/C API/2023-11-27-09-44-16.gh-issue-112438.GdNZiI.rst new file mode 100644 index 00000000000000..113119efd6aebb --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-11-27-09-44-16.gh-issue-112438.GdNZiI.rst @@ -0,0 +1,2 @@ +Fix support of format units "es", "et", "es#", and "et#" in nested tuples in +:c:func:`PyArg_ParseTuple`-like functions. diff --git a/Modules/_testcapi/getargs.c b/Modules/_testcapi/getargs.c index e4cd15503fd11f..33e8af7d7bbb39 100644 --- a/Modules/_testcapi/getargs.c +++ b/Modules/_testcapi/getargs.c @@ -71,18 +71,22 @@ parse_tuple_and_keywords(PyObject *self, PyObject *args) if (result) { int objects_only = 1; + int count = 0; for (const char *f = sub_format; *f; f++) { - if (Py_ISALNUM(*f) && strchr("OSUY", *f) == NULL) { - objects_only = 0; - break; + if (Py_ISALNUM(*f)) { + if (strchr("OSUY", *f) == NULL) { + objects_only = 0; + break; + } + count++; } } if (objects_only) { - return_value = PyTuple_New(size); + return_value = PyTuple_New(count); if (return_value == NULL) { goto exit; } - for (Py_ssize_t i = 0; i < size; i++) { + for (Py_ssize_t i = 0; i < count; i++) { PyObject *arg = *(PyObject **)(buffers + i); if (arg == NULL) { arg = Py_None; diff --git a/Python/getargs.c b/Python/getargs.c index c0c2eb27184e3c..5ff8f7473609f5 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -477,7 +477,7 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } else if (c == ':' || c == ';' || c == '\0') break; - else if (level == 0 && Py_ISALPHA(c)) + else if (level == 0 && Py_ISALPHA(c) && c != 'e') n++; } From b14e5df120ca8ce968a67df2e00e7a764dd703a0 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 27 Nov 2023 19:35:52 +0200 Subject: [PATCH 10/37] gh-111789: Use PyDict_GetItemRef() in Modules/_csv.c (gh-112073) --- Modules/_csv.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Modules/_csv.c b/Modules/_csv.c index 714fbef08d22c9..ae6b6457ffad9a 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -160,15 +160,9 @@ static PyObject * get_dialect_from_registry(PyObject *name_obj, _csvstate *module_state) { PyObject *dialect_obj; - - dialect_obj = PyDict_GetItemWithError(module_state->dialects, name_obj); - if (dialect_obj == NULL) { - if (!PyErr_Occurred()) - PyErr_Format(module_state->error_obj, "unknown dialect"); + if (PyDict_GetItemRef(module_state->dialects, name_obj, &dialect_obj) == 0) { + PyErr_SetString(module_state->error_obj, "unknown dialect"); } - else - Py_INCREF(dialect_obj); - return dialect_obj; } From 0f009033202b339375944f613aa6d0597a2841de Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 27 Nov 2023 19:41:47 +0200 Subject: [PATCH 11/37] gh-111789: Use PyDict_GetItemRef() in Modules/_struct.c (gh-112076) --- Modules/_struct.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Modules/_struct.c b/Modules/_struct.c index 24a4cb3b6413f1..bd16fa89f18945 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -2257,14 +2257,13 @@ cache_struct_converter(PyObject *module, PyObject *fmt, PyStructObject **ptr) return 1; } - s_object = PyDict_GetItemWithError(state->cache, fmt); + if (PyDict_GetItemRef(state->cache, fmt, &s_object) < 0) { + return 0; + } if (s_object != NULL) { - *ptr = (PyStructObject *)Py_NewRef(s_object); + *ptr = (PyStructObject *)s_object; return Py_CLEANUP_SUPPORTED; } - else if (PyErr_Occurred()) { - return 0; - } s_object = PyObject_CallOneArg(state->PyStructType, fmt); if (s_object != NULL) { From ef9b2fc9b0378aee87328fadce73b3fefb6aca4a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 27 Nov 2023 19:46:43 +0200 Subject: [PATCH 12/37] gh-111789: Use PyDict_GetItemRef() in Modules/_threadmodule.c (gh-112077) --- Modules/_threadmodule.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index c608789b5fbd5c..afcf646e3bc19e 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1115,12 +1115,10 @@ local_getattro(localobject *self, PyObject *name) } /* Optimization: just look in dict ourselves */ - PyObject *value = PyDict_GetItemWithError(ldict, name); - if (value != NULL) { - return Py_NewRef(value); - } - if (PyErr_Occurred()) { - return NULL; + PyObject *value; + if (PyDict_GetItemRef(ldict, name, &value) != 0) { + // found or error + return value; } /* Fall back on generic to get __class__ and __dict__ */ From d8908932fc03e064ba8df03d17d8cc7ffa5f171f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 27 Nov 2023 19:51:31 +0200 Subject: [PATCH 13/37] gh-111789: Use PyDict_GetItemRef() in Modules/pyexpat.c (gh-112079) --- Modules/pyexpat.c | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 21579a80dd7f70..9d95309dbb7aa6 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -240,19 +240,12 @@ string_intern(xmlparseobject *self, const char* str) return result; if (!self->intern) return result; - value = PyDict_GetItemWithError(self->intern, result); - if (!value) { - if (!PyErr_Occurred() && - PyDict_SetItem(self->intern, result, result) == 0) - { - return result; - } - else { - Py_DECREF(result); - return NULL; - } + if (PyDict_GetItemRef(self->intern, result, &value) == 0 && + PyDict_SetItem(self->intern, result, result) == 0) + { + return result; } - Py_INCREF(value); + assert((value != NULL) == !PyErr_Occurred()); Py_DECREF(result); return value; } From 395fd9c1808fa0babc96540744d2c915178a452b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 27 Nov 2023 19:52:54 +0200 Subject: [PATCH 14/37] gh-111789: Use PyDict_GetItemRef() in Python/bltinmodule.c (gh-112081) --- Python/bltinmodule.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index ff07c498263cd3..7a9625134761f9 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -5,7 +5,6 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_Vector() #include "pycore_compile.h" // _PyAST_Compile() -#include "pycore_dict.h" // _PyDict_GetItemWithError() #include "pycore_long.h" // _PyLong_CompactValue #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _Py_AddToAllObjects() @@ -141,18 +140,16 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, goto error; } - meta = _PyDict_GetItemWithError(mkw, &_Py_ID(metaclass)); + if (PyDict_GetItemRef(mkw, &_Py_ID(metaclass), &meta) < 0) { + goto error; + } if (meta != NULL) { - Py_INCREF(meta); if (PyDict_DelItem(mkw, &_Py_ID(metaclass)) < 0) { goto error; } /* metaclass is explicitly given, check if it's indeed a class */ isclass = PyType_Check(meta); } - else if (PyErr_Occurred()) { - goto error; - } } if (meta == NULL) { /* if there are no bases, use type: */ From aa438bdd6deed225d30d87dc3a77602ffc924213 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 27 Nov 2023 19:53:43 +0200 Subject: [PATCH 15/37] gh-111789: Use PyDict_GetItemRef() in Python/codecs.c (gh-112082) --- Python/codecs.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Python/codecs.c b/Python/codecs.c index b79bf555f2f22a..545bf82e00dca1 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -146,15 +146,14 @@ PyObject *_PyCodec_Lookup(const char *encoding) PyUnicode_InternInPlace(&v); /* First, try to lookup the name in the registry dictionary */ - PyObject *result = PyDict_GetItemWithError(interp->codec_search_cache, v); + PyObject *result; + if (PyDict_GetItemRef(interp->codec_search_cache, v, &result) < 0) { + goto onError; + } if (result != NULL) { - Py_INCREF(result); Py_DECREF(v); return result; } - else if (PyErr_Occurred()) { - goto onError; - } /* Next, scan the search functions in order of registration */ const Py_ssize_t len = PyList_Size(interp->codec_search_path); From befbad3663a48a8de2e1263afe18ec9fa47dfc6d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 27 Nov 2023 19:55:30 +0200 Subject: [PATCH 16/37] gh-111789: Use PyDict_GetItemRef() in Python/symtable.c (gh-112084) --- Python/symtable.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Python/symtable.c b/Python/symtable.c index da7fec0ee7cf0c..52d5932896b263 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -497,18 +497,14 @@ _PySymtable_Lookup(struct symtable *st, void *key) k = PyLong_FromVoidPtr(key); if (k == NULL) return NULL; - v = PyDict_GetItemWithError(st->st_blocks, k); - Py_DECREF(k); - - if (v) { - assert(PySTEntry_Check(v)); - } - else if (!PyErr_Occurred()) { + if (PyDict_GetItemRef(st->st_blocks, k, &v) == 0) { PyErr_SetString(PyExc_KeyError, "unknown symbol table entry"); } + Py_DECREF(k); - return (PySTEntryObject *)Py_XNewRef(v); + assert(v == NULL || PySTEntry_Check(v)); + return (PySTEntryObject *)v; } long From 936c503a442ee062c837e334f237796554c792ff Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 27 Nov 2023 19:58:43 +0200 Subject: [PATCH 17/37] gh-111789: Use PyDict_GetItemRef() in Python/_warnings.c (gh-112080) --- Python/_warnings.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/_warnings.c b/Python/_warnings.c index 4b7fb888247145..d4765032824e56 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -425,15 +425,15 @@ already_warned(PyInterpreterState *interp, PyObject *registry, PyObject *key, Py_DECREF(version_obj); } else { - already_warned = PyDict_GetItemWithError(registry, key); + if (PyDict_GetItemRef(registry, key, &already_warned) < 0) { + return -1; + } if (already_warned != NULL) { int rc = PyObject_IsTrue(already_warned); + Py_DECREF(already_warned); if (rc != 0) return rc; } - else if (PyErr_Occurred()) { - return -1; - } } /* This warning wasn't found in the registry, set it. */ From 99a73c3465a45fe57cac01a917fc50e0743b5964 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Mon, 27 Nov 2023 13:05:55 -0500 Subject: [PATCH 18/37] gh-76912: Raise OSError from any failure in getpass.getuser() (#29739) * bpo-32731: Raise OSError from any failure in getpass.getuser() Previously, if the username was not set in certain environment variables, ImportError escaped on Windows systems, and it was possible for KeyError to escape on other systems if getpwuid() failed. --- Doc/library/getpass.rst | 7 +++++-- Doc/whatsnew/3.13.rst | 4 ++++ Lib/getpass.py | 13 ++++++++++--- Lib/test/test_getpass.py | 4 ++-- .../2021-11-23-22-22-49.bpo-32731.kNOASr.rst | 3 +++ 5 files changed, 24 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-11-23-22-22-49.bpo-32731.kNOASr.rst diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst index 5c79daf0f47d8e..54c84d45a59856 100644 --- a/Doc/library/getpass.rst +++ b/Doc/library/getpass.rst @@ -46,7 +46,10 @@ The :mod:`getpass` module provides two functions: :envvar:`USER`, :envvar:`!LNAME` and :envvar:`USERNAME`, in order, and returns the value of the first one which is set to a non-empty string. If none are set, the login name from the password database is returned on - systems which support the :mod:`pwd` module, otherwise, an exception is - raised. + systems which support the :mod:`pwd` module, otherwise, an :exc:`OSError` + is raised. In general, this function should be preferred over :func:`os.getlogin()`. + + .. versionchanged:: 3.13 + Previously, various exceptions beyond just :exc:`OSError` were raised. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 3fd0f5e165f018..ec09dfea4aad3c 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1032,6 +1032,10 @@ Changes in the Python API recomended in the documentation. (Contributed by Serhiy Storchaka in :gh:`106672`.) +* An :exc:`OSError` is now raised by :func:`getpass.getuser` for any failure to + retrieve a username, instead of :exc:`ImportError` on non-Unix platforms or + :exc:`KeyError` on Unix platforms where the password database is empty. + Build Changes ============= diff --git a/Lib/getpass.py b/Lib/getpass.py index 8b42c0a536b4c4..bd0097ced94c5e 100644 --- a/Lib/getpass.py +++ b/Lib/getpass.py @@ -156,7 +156,11 @@ def getuser(): First try various environment variables, then the password database. This works on Windows as long as USERNAME is set. + Any failure to find a username raises OSError. + .. versionchanged:: 3.13 + Previously, various exceptions beyond just :exc:`OSError` + were raised. """ for name in ('LOGNAME', 'USER', 'LNAME', 'USERNAME'): @@ -164,9 +168,12 @@ def getuser(): if user: return user - # If this fails, the exception will "explain" why - import pwd - return pwd.getpwuid(os.getuid())[0] + try: + import pwd + return pwd.getpwuid(os.getuid())[0] + except (ImportError, KeyError) as e: + raise OSError('No username set in the environment') from e + # Bind the name getpass to the appropriate function try: diff --git a/Lib/test/test_getpass.py b/Lib/test/test_getpass.py index 98ecec94336e32..80dda2caaa3331 100644 --- a/Lib/test/test_getpass.py +++ b/Lib/test/test_getpass.py @@ -26,7 +26,7 @@ def test_username_priorities_of_env_values(self, environ): environ.get.return_value = None try: getpass.getuser() - except ImportError: # in case there's no pwd module + except OSError: # in case there's no pwd module pass except KeyError: # current user has no pwd entry @@ -47,7 +47,7 @@ def test_username_falls_back_to_pwd(self, environ): getpass.getuser()) getpw.assert_called_once_with(42) else: - self.assertRaises(ImportError, getpass.getuser) + self.assertRaises(OSError, getpass.getuser) class GetpassRawinputTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2021-11-23-22-22-49.bpo-32731.kNOASr.rst b/Misc/NEWS.d/next/Library/2021-11-23-22-22-49.bpo-32731.kNOASr.rst new file mode 100644 index 00000000000000..92f3b870c11131 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-11-23-22-22-49.bpo-32731.kNOASr.rst @@ -0,0 +1,3 @@ +:func:`getpass.getuser` now raises :exc:`OSError` for all failures rather +than :exc:`ImportError` on systems lacking the :mod:`pwd` module or +:exc:`KeyError` if the password database is empty. From 967f2a3052c2d22e31564b428a9aa8cc63dc2a9f Mon Sep 17 00:00:00 2001 From: kale-smoothie <34165060+kale-smoothie@users.noreply.github.com> Date: Mon, 27 Nov 2023 18:09:41 +0000 Subject: [PATCH 19/37] bpo-41422: Visit the Pickler's and Unpickler's memo in tp_traverse (GH-21664) Co-authored-by: Serhiy Storchaka --- .../2020-07-28-20-48-05.bpo-41422.iMwnMu.rst | 2 ++ Modules/_pickle.c | 15 +++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2020-07-28-20-48-05.bpo-41422.iMwnMu.rst diff --git a/Misc/NEWS.d/next/Library/2020-07-28-20-48-05.bpo-41422.iMwnMu.rst b/Misc/NEWS.d/next/Library/2020-07-28-20-48-05.bpo-41422.iMwnMu.rst new file mode 100644 index 00000000000000..8bde68f8f2afc8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-07-28-20-48-05.bpo-41422.iMwnMu.rst @@ -0,0 +1,2 @@ +Fixed memory leaks of :class:`pickle.Pickler` and :class:`pickle.Unpickler` involving cyclic references via the +internal memo mapping. diff --git a/Modules/_pickle.c b/Modules/_pickle.c index a3cf34699ba509..227e5378e42285 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -4707,6 +4707,14 @@ Pickler_traverse(PicklerObject *self, visitproc visit, void *arg) Py_VISIT(self->fast_memo); Py_VISIT(self->reducer_override); Py_VISIT(self->buffer_callback); + PyMemoTable *memo = self->memo; + if (memo && memo->mt_table) { + Py_ssize_t i = memo->mt_allocated; + while (--i >= 0) { + Py_VISIT(memo->mt_table[i].me_key); + } + } + return 0; } @@ -7175,6 +7183,13 @@ Unpickler_traverse(UnpicklerObject *self, visitproc visit, void *arg) Py_VISIT(self->stack); Py_VISIT(self->pers_func); Py_VISIT(self->buffers); + PyObject **memo = self->memo; + if (memo) { + Py_ssize_t i = self->memo_size; + while (--i >= 0) { + Py_VISIT(memo[i]); + } + } return 0; } From 2c8b19174274c183eb652932871f60570123fe99 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 27 Nov 2023 18:36:11 +0000 Subject: [PATCH 20/37] gh-112388: Fix an error that was causing the parser to try to overwrite tokenizer errors (#112410) Signed-off-by: Pablo Galindo --- Lib/test/test_syntax.py | 1 + .../2023-11-25-22-58-49.gh-issue-112388.MU3cIM.rst | 2 ++ Parser/pegen_errors.c | 4 ++++ 3 files changed, 7 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-58-49.gh-issue-112388.MU3cIM.rst diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index f6fa6495508d2c..e80e95383b897d 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -2349,6 +2349,7 @@ def test_error_string_literal(self): def test_invisible_characters(self): self._check_error('print\x17("Hello")', "invalid non-printable character") + self._check_error(b"with(0,,):\n\x01", "invalid non-printable character") def test_match_call_does_not_raise_syntax_error(self): code = """ diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-58-49.gh-issue-112388.MU3cIM.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-58-49.gh-issue-112388.MU3cIM.rst new file mode 100644 index 00000000000000..1c82be2febda4f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-58-49.gh-issue-112388.MU3cIM.rst @@ -0,0 +1,2 @@ +Fix an error that was causing the parser to try to overwrite tokenizer +errors. Patch by pablo Galindo diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index e2bc3b91c80718..2528d4502b3c0c 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -219,6 +219,10 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) { void * _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *errmsg, ...) { + // Bail out if we already have an error set. + if (p->error_indicator && PyErr_Occurred()) { + return NULL; + } if (p->fill == 0) { va_list va; va_start(va, errmsg); From 45d648597b1146431bf3d91041e60d7f040e70bf Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 27 Nov 2023 18:37:48 +0000 Subject: [PATCH 21/37] gh-112387: Fix error positions for decoded strings with backwards tokenize errors (#112409) Signed-off-by: Pablo Galindo --- Lib/test/test_syntax.py | 4 ++++ .../2023-11-25-22-39-44.gh-issue-112387.AbBq5W.rst | 2 ++ Parser/pegen_errors.c | 4 ++++ 3 files changed, 10 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-39-44.gh-issue-112387.AbBq5W.rst diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index e80e95383b897d..99433df73387d0 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -2334,6 +2334,10 @@ def test_error_parenthesis(self): """ self._check_error(code, "parenthesis '\\)' does not match opening parenthesis '\\['") + # Examples with dencodings + s = b'# coding=latin\n(aaaaaaaaaaaaaaaaa\naaaaaaaaaaa\xb5' + self._check_error(s, "'\(' was never closed") + def test_error_string_literal(self): self._check_error("'blech", r"unterminated string literal \(.*\)$") diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-39-44.gh-issue-112387.AbBq5W.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-39-44.gh-issue-112387.AbBq5W.rst new file mode 100644 index 00000000000000..adac11bf4c90a1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-39-44.gh-issue-112387.AbBq5W.rst @@ -0,0 +1,2 @@ +Fix error positions for decoded strings with backwards tokenize errors. +Patch by Pablo Galindo diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 2528d4502b3c0c..20232f3a26a2cc 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -282,6 +282,10 @@ get_error_line_from_tokenizer_buffers(Parser *p, Py_ssize_t lineno) Py_ssize_t relative_lineno = p->starting_lineno ? lineno - p->starting_lineno + 1 : lineno; const char* buf_end = p->tok->fp_interactive ? p->tok->interactive_src_end : p->tok->inp; + if (buf_end < cur_line) { + buf_end = cur_line + strlen(cur_line); + } + for (int i = 0; i < relative_lineno - 1; i++) { char *new_line = strchr(cur_line, '\n'); // The assert is here for debug builds but the conditional that From 4dcfd02bed0d7958703ef44baa79a4a98475be2e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 27 Nov 2023 20:57:33 +0200 Subject: [PATCH 22/37] gh-68166: Add support of "vsapi" in ttk.Style.element_create() (GH-111393) --- Doc/library/tkinter.ttk.rst | 60 +++++++++++++- Doc/whatsnew/3.13.rst | 5 ++ Lib/test/test_ttk/test_style.py | 82 +++++++++++++++++++ Lib/test/test_ttk_textonly.py | 32 ++++++-- Lib/tkinter/ttk.py | 55 +++++++------ ...3-11-08-18-53-07.gh-issue-68166.1iTh4Y.rst | 2 + 6 files changed, 204 insertions(+), 32 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-11-08-18-53-07.gh-issue-68166.1iTh4Y.rst diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index 5fab1454944989..6e01ec7b291255 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -1391,7 +1391,8 @@ option. If you don't know the class name of a widget, use the method .. method:: element_create(elementname, etype, *args, **kw) Create a new element in the current theme, of the given *etype* which is - expected to be either "image" or "from". + expected to be either "image", "from" or "vsapi". + The latter is only available in Tk 8.6 on Windows. If "image" is used, *args* should contain the default image name followed by statespec/value pairs (this is the imagespec), and *kw* may have the @@ -1439,6 +1440,63 @@ option. If you don't know the class name of a widget, use the method style = ttk.Style(root) style.element_create('plain.background', 'from', 'default') + If "vsapi" is used as the value of *etype*, :meth:`element_create` + will create a new element in the current theme whose visual appearance + is drawn using the Microsoft Visual Styles API which is responsible + for the themed styles on Windows XP and Vista. + *args* is expected to contain the Visual Styles class and part as + given in the Microsoft documentation followed by an optional sequence + of tuples of ttk states and the corresponding Visual Styles API state + value. + *kw* may have the following options: + + padding=padding + Specify the element's interior padding. + *padding* is a list of up to four integers specifying the left, + top, right and bottom padding quantities respectively. + If fewer than four elements are specified, bottom defaults to top, + right defaults to left, and top defaults to left. + In other words, a list of three numbers specify the left, vertical, + and right padding; a list of two numbers specify the horizontal + and the vertical padding; a single number specifies the same + padding all the way around the widget. + This option may not be mixed with any other options. + + margins=padding + Specifies the elements exterior padding. + *padding* is a list of up to four integers specifying the left, top, + right and bottom padding quantities respectively. + This option may not be mixed with any other options. + + width=width + Specifies the width for the element. + If this option is set then the Visual Styles API will not be queried + for the recommended size or the part. + If this option is set then *height* should also be set. + The *width* and *height* options cannot be mixed with the *padding* + or *margins* options. + + height=height + Specifies the height of the element. + See the comments for *width*. + + Example:: + + style = ttk.Style(root) + style.element_create('pin', 'vsapi', 'EXPLORERBAR', 3, [ + ('pressed', '!selected', 3), + ('active', '!selected', 2), + ('pressed', 'selected', 6), + ('active', 'selected', 5), + ('selected', 4), + ('', 1)]) + style.layout('Explorer.Pin', + [('Explorer.Pin.pin', {'sticky': 'news'})]) + pin = ttk.Checkbutton(style='Explorer.Pin') + pin.pack(expand=True, fill='both') + + .. versionchanged:: 3.13 + Added support of the "vsapi" element factory. .. method:: element_names() diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index ec09dfea4aad3c..dad49f43d9090f 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -301,6 +301,11 @@ tkinter :meth:`!tk_busy_current`, and :meth:`!tk_busy_status`. (Contributed by Miguel, klappnase and Serhiy Storchaka in :gh:`72684`.) +* Add support of the "vsapi" element type in + the :meth:`~tkinter.ttk.Style.element_create` method of + :class:`tkinter.ttk.Style`. + (Contributed by Serhiy Storchaka in :gh:`68166`.) + traceback --------- diff --git a/Lib/test/test_ttk/test_style.py b/Lib/test/test_ttk/test_style.py index 52c4b7a0beabd0..9a04a95dc40d65 100644 --- a/Lib/test/test_ttk/test_style.py +++ b/Lib/test/test_ttk/test_style.py @@ -258,6 +258,55 @@ def test_element_create_image_errors(self): with self.assertRaisesRegex(TclError, 'bad option'): style.element_create('block2', 'image', image, spam=1) + def test_element_create_vsapi_1(self): + style = self.style + if 'xpnative' not in style.theme_names(): + self.skipTest("requires 'xpnative' theme") + style.element_create('smallclose', 'vsapi', 'WINDOW', 19, [ + ('disabled', 4), + ('pressed', 3), + ('active', 2), + ('', 1)]) + style.layout('CloseButton', + [('CloseButton.smallclose', {'sticky': 'news'})]) + b = ttk.Button(self.root, style='CloseButton') + b.pack(expand=True, fill='both') + self.assertEqual(b.winfo_reqwidth(), 13) + self.assertEqual(b.winfo_reqheight(), 13) + + def test_element_create_vsapi_2(self): + style = self.style + if 'xpnative' not in style.theme_names(): + self.skipTest("requires 'xpnative' theme") + style.element_create('pin', 'vsapi', 'EXPLORERBAR', 3, [ + ('pressed', '!selected', 3), + ('active', '!selected', 2), + ('pressed', 'selected', 6), + ('active', 'selected', 5), + ('selected', 4), + ('', 1)]) + style.layout('Explorer.Pin', + [('Explorer.Pin.pin', {'sticky': 'news'})]) + pin = ttk.Checkbutton(self.root, style='Explorer.Pin') + pin.pack(expand=True, fill='both') + self.assertEqual(pin.winfo_reqwidth(), 16) + self.assertEqual(pin.winfo_reqheight(), 16) + + def test_element_create_vsapi_3(self): + style = self.style + if 'xpnative' not in style.theme_names(): + self.skipTest("requires 'xpnative' theme") + style.element_create('headerclose', 'vsapi', 'EXPLORERBAR', 2, [ + ('pressed', 3), + ('active', 2), + ('', 1)]) + style.layout('Explorer.CloseButton', + [('Explorer.CloseButton.headerclose', {'sticky': 'news'})]) + b = ttk.Button(self.root, style='Explorer.CloseButton') + b.pack(expand=True, fill='both') + self.assertEqual(b.winfo_reqwidth(), 16) + self.assertEqual(b.winfo_reqheight(), 16) + def test_theme_create(self): style = self.style curr_theme = style.theme_use() @@ -358,6 +407,39 @@ def test_theme_create_image(self): style.theme_use(curr_theme) + def test_theme_create_vsapi(self): + style = self.style + if 'xpnative' not in style.theme_names(): + self.skipTest("requires 'xpnative' theme") + curr_theme = style.theme_use() + new_theme = 'testtheme5' + style.theme_create(new_theme, settings={ + 'pin' : { + 'element create': ['vsapi', 'EXPLORERBAR', 3, [ + ('pressed', '!selected', 3), + ('active', '!selected', 2), + ('pressed', 'selected', 6), + ('active', 'selected', 5), + ('selected', 4), + ('', 1)]], + }, + 'Explorer.Pin' : { + 'layout': [('Explorer.Pin.pin', {'sticky': 'news'})], + }, + }) + + style.theme_use(new_theme) + self.assertIn('pin', style.element_names()) + self.assertEqual(style.layout('Explorer.Pin'), + [('Explorer.Pin.pin', {'sticky': 'nswe'})]) + + pin = ttk.Checkbutton(self.root, style='Explorer.Pin') + pin.pack(expand=True, fill='both') + self.assertEqual(pin.winfo_reqwidth(), 16) + self.assertEqual(pin.winfo_reqheight(), 16) + + style.theme_use(curr_theme) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ttk_textonly.py b/Lib/test/test_ttk_textonly.py index 96dc179a69ecac..e6525c4d6c6982 100644 --- a/Lib/test/test_ttk_textonly.py +++ b/Lib/test/test_ttk_textonly.py @@ -179,7 +179,7 @@ def test_format_elemcreate(self): # don't format returned values as a tcl script # minimum acceptable for image type self.assertEqual(ttk._format_elemcreate('image', False, 'test'), - ("test ", ())) + ("test", ())) # specifying a state spec self.assertEqual(ttk._format_elemcreate('image', False, 'test', ('', 'a')), ("test {} a", ())) @@ -203,17 +203,19 @@ def test_format_elemcreate(self): # don't format returned values as a tcl script # minimum acceptable for vsapi self.assertEqual(ttk._format_elemcreate('vsapi', False, 'a', 'b'), - ("a b ", ())) + ('a', 'b', ('', 1), ())) # now with a state spec with multiple states self.assertEqual(ttk._format_elemcreate('vsapi', False, 'a', 'b', - ('a', 'b', 'c')), ("a b {a b} c", ())) + [('a', 'b', 'c')]), ('a', 'b', ('a b', 'c'), ())) # state spec and option self.assertEqual(ttk._format_elemcreate('vsapi', False, 'a', 'b', - ('a', 'b'), opt='x'), ("a b a b", ("-opt", "x"))) + [('a', 'b')], opt='x'), ('a', 'b', ('a', 'b'), ("-opt", "x"))) # format returned values as a tcl script # state spec with a multivalue and an option self.assertEqual(ttk._format_elemcreate('vsapi', True, 'a', 'b', - ('a', 'b', [1, 2]), opt='x'), ("{a b {a b} {1 2}}", "-opt x")) + opt='x'), ("a b {{} 1}", "-opt x")) + self.assertEqual(ttk._format_elemcreate('vsapi', True, 'a', 'b', + [('a', 'b', [1, 2])], opt='x'), ("a b {{a b} {1 2}}", "-opt x")) # Testing type = from # from type expects at least a type name @@ -222,9 +224,9 @@ def test_format_elemcreate(self): self.assertEqual(ttk._format_elemcreate('from', False, 'a'), ('a', ())) self.assertEqual(ttk._format_elemcreate('from', False, 'a', 'b'), - ('a', ('b', ))) + ('a', ('b',))) self.assertEqual(ttk._format_elemcreate('from', True, 'a', 'b'), - ('{a}', 'b')) + ('a', 'b')) def test_format_layoutlist(self): @@ -326,6 +328,22 @@ def test_script_from_settings(self): "ttk::style element create thing image {name {state1 state2} val} " "-opt {3 2m}") + vsapi = {'pin': {'element create': + ['vsapi', 'EXPLORERBAR', 3, [ + ('pressed', '!selected', 3), + ('active', '!selected', 2), + ('pressed', 'selected', 6), + ('active', 'selected', 5), + ('selected', 4), + ('', 1)]]}} + self.assertEqual(ttk._script_from_settings(vsapi), + "ttk::style element create pin vsapi EXPLORERBAR 3 {" + "{pressed !selected} 3 " + "{active !selected} 2 " + "{pressed selected} 6 " + "{active selected} 5 " + "selected 4 " + "{} 1} ") def test_tclobj_to_py(self): self.assertEqual( diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py index efeabb7a92c627..5ca938a670831a 100644 --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -95,40 +95,47 @@ def _format_mapdict(mapdict, script=False): def _format_elemcreate(etype, script=False, *args, **kw): """Formats args and kw according to the given element factory etype.""" - spec = None + specs = () opts = () - if etype in ("image", "vsapi"): - if etype == "image": # define an element based on an image - # first arg should be the default image name - iname = args[0] - # next args, if any, are statespec/value pairs which is almost - # a mapdict, but we just need the value - imagespec = _join(_mapdict_values(args[1:])) - spec = "%s %s" % (iname, imagespec) - + if etype == "image": # define an element based on an image + # first arg should be the default image name + iname = args[0] + # next args, if any, are statespec/value pairs which is almost + # a mapdict, but we just need the value + imagespec = (iname, *_mapdict_values(args[1:])) + if script: + specs = (imagespec,) else: - # define an element whose visual appearance is drawn using the - # Microsoft Visual Styles API which is responsible for the - # themed styles on Windows XP and Vista. - # Availability: Tk 8.6, Windows XP and Vista. - class_name, part_id = args[:2] - statemap = _join(_mapdict_values(args[2:])) - spec = "%s %s %s" % (class_name, part_id, statemap) + specs = (_join(imagespec),) + opts = _format_optdict(kw, script) + if etype == "vsapi": + # define an element whose visual appearance is drawn using the + # Microsoft Visual Styles API which is responsible for the + # themed styles on Windows XP and Vista. + # Availability: Tk 8.6, Windows XP and Vista. + if len(args) < 3: + class_name, part_id = args + statemap = (((), 1),) + else: + class_name, part_id, statemap = args + specs = (class_name, part_id, tuple(_mapdict_values(statemap))) opts = _format_optdict(kw, script) elif etype == "from": # clone an element # it expects a themename and optionally an element to clone from, # otherwise it will clone {} (empty element) - spec = args[0] # theme name + specs = (args[0],) # theme name if len(args) > 1: # elementfrom specified opts = (_format_optvalue(args[1], script),) if script: - spec = '{%s}' % spec + specs = _join(specs) opts = ' '.join(opts) + return specs, opts + else: + return *specs, opts - return spec, opts def _format_layoutlist(layout, indent=0, indent_size=2): """Formats a layout list so we can pass the result to ttk::style @@ -214,10 +221,10 @@ def _script_from_settings(settings): elemargs = eopts[1:argc] elemkw = eopts[argc] if argc < len(eopts) and eopts[argc] else {} - spec, opts = _format_elemcreate(etype, True, *elemargs, **elemkw) + specs, eopts = _format_elemcreate(etype, True, *elemargs, **elemkw) script.append("ttk::style element create %s %s %s %s" % ( - name, etype, spec, opts)) + name, etype, specs, eopts)) return '\n'.join(script) @@ -434,9 +441,9 @@ def layout(self, style, layoutspec=None): def element_create(self, elementname, etype, *args, **kw): """Create a new element in the current theme of given etype.""" - spec, opts = _format_elemcreate(etype, False, *args, **kw) + *specs, opts = _format_elemcreate(etype, False, *args, **kw) self.tk.call(self._name, "element", "create", elementname, etype, - spec, *opts) + *specs, *opts) def element_names(self): diff --git a/Misc/NEWS.d/next/Library/2023-11-08-18-53-07.gh-issue-68166.1iTh4Y.rst b/Misc/NEWS.d/next/Library/2023-11-08-18-53-07.gh-issue-68166.1iTh4Y.rst new file mode 100644 index 00000000000000..30379b8fa1afaf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-08-18-53-07.gh-issue-68166.1iTh4Y.rst @@ -0,0 +1,2 @@ +Add support of the "vsapi" element type in +:meth:`tkinter.ttk.Style.element_create`. From 8f71b349de1ff2b11223ff7a8241c62a5a932339 Mon Sep 17 00:00:00 2001 From: apaz Date: Mon, 27 Nov 2023 15:13:27 -0600 Subject: [PATCH 23/37] gh-112217: Add check to call result for `do_raise()` where cause is a type. (#112216) --- Lib/test/test_raise.py | 14 ++++++++++++++ .../2023-11-27-18-55-30.gh-issue-112217.SwFLMj.rst | 1 + Python/ceval.c | 7 +++++++ 3 files changed, 22 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-11-27-18-55-30.gh-issue-112217.SwFLMj.rst diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py index 5936d7535edd5f..6d26a61bee4292 100644 --- a/Lib/test/test_raise.py +++ b/Lib/test/test_raise.py @@ -185,6 +185,20 @@ def test_class_cause(self): else: self.fail("No exception raised") + def test_class_cause_nonexception_result(self): + class ConstructsNone(BaseException): + @classmethod + def __new__(*args, **kwargs): + return None + try: + raise IndexError from ConstructsNone + except TypeError as e: + self.assertIn("should have returned an instance of BaseException", str(e)) + except IndexError: + self.fail("Wrong kind of exception raised") + else: + self.fail("No exception raised") + def test_instance_cause(self): cause = KeyError() try: diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-27-18-55-30.gh-issue-112217.SwFLMj.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-27-18-55-30.gh-issue-112217.SwFLMj.rst new file mode 100644 index 00000000000000..d4efbab6b2d128 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-11-27-18-55-30.gh-issue-112217.SwFLMj.rst @@ -0,0 +1 @@ +Add check for the type of ``__cause__`` returned from calling the type ``T`` in ``raise from T``. diff --git a/Python/ceval.c b/Python/ceval.c index 76ab5df42f63db..def75fd114cb86 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1920,6 +1920,13 @@ do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) fixed_cause = _PyObject_CallNoArgs(cause); if (fixed_cause == NULL) goto raise_error; + if (!PyExceptionInstance_Check(fixed_cause)) { + _PyErr_Format(tstate, PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + cause, Py_TYPE(fixed_cause)); + goto raise_error; + } Py_DECREF(cause); } else if (PyExceptionInstance_Check(cause)) { From b90a5cf11cdb69e60aed7be732e80113bca7bbe4 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 27 Nov 2023 14:11:40 -0900 Subject: [PATCH 24/37] gh-99367: Do not mangle sys.path[0] in pdb if safe_path is set (#111762) Co-authored-by: Christian Walther --- Doc/whatsnew/3.13.rst | 5 +++ Lib/pdb.py | 6 ++-- Lib/test/test_pdb.py | 33 ++++++++++++++++--- ...3-11-05-20-09-27.gh-issue-99367.HLaWKo.rst | 1 + 4 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-11-05-20-09-27.gh-issue-99367.HLaWKo.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index dad49f43d9090f..bf6a70f2009b8e 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -285,6 +285,11 @@ pdb identified and executed. (Contributed by Tian Gao in :gh:`108464`.) +* ``sys.path[0]`` will no longer be replaced by the directory of the script + being debugged when ``sys.flags.safe_path`` is set (via the :option:`-P` + command line option or :envvar:`PYTHONSAFEPATH` environment variable). + (Contributed by Tian Gao and Christian Walther in :gh:`111762`.) + sqlite3 ------- diff --git a/Lib/pdb.py b/Lib/pdb.py index ed78d749a47fa8..9d124189df11cf 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -142,8 +142,10 @@ def check(self): print('Error:', self.orig, 'is a directory') sys.exit(1) - # Replace pdb's dir with script's dir in front of module search path. - sys.path[0] = os.path.dirname(self) + # If safe_path(-P) is not set, sys.path[0] is the directory + # of pdb, and we should replace it with the directory of the script + if not sys.flags.safe_path: + sys.path[0] = os.path.dirname(self) @property def filename(self): diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 67a4053a2ac8bc..2a279ca869e9c7 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2520,15 +2520,21 @@ def tearDown(self): @unittest.skipIf(sys.flags.safe_path, 'PYTHONSAFEPATH changes default sys.path') - def _run_pdb(self, pdb_args, commands, expected_returncode=0): + def _run_pdb(self, pdb_args, commands, + expected_returncode=0, + extra_env=None): self.addCleanup(os_helper.rmtree, '__pycache__') cmd = [sys.executable, '-m', 'pdb'] + pdb_args + if extra_env is not None: + env = os.environ | extra_env + else: + env = os.environ with subprocess.Popen( cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT, - env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'} + env = {**env, 'PYTHONIOENCODING': 'utf-8'} ) as proc: stdout, stderr = proc.communicate(str.encode(commands)) stdout = stdout and bytes.decode(stdout) @@ -2540,13 +2546,15 @@ def _run_pdb(self, pdb_args, commands, expected_returncode=0): ) return stdout, stderr - def run_pdb_script(self, script, commands, expected_returncode=0): + def run_pdb_script(self, script, commands, + expected_returncode=0, + extra_env=None): """Run 'script' lines with pdb and the pdb 'commands'.""" filename = 'main.py' with open(filename, 'w') as f: f.write(textwrap.dedent(script)) self.addCleanup(os_helper.unlink, filename) - return self._run_pdb([filename], commands, expected_returncode) + return self._run_pdb([filename], commands, expected_returncode, extra_env) def run_pdb_module(self, script, commands): """Runs the script code as part of a module""" @@ -3131,6 +3139,23 @@ def test_issue42384_symlink(self): self.assertEqual(stdout.split('\n')[2].rstrip('\r'), expected) + def test_safe_path(self): + """ With safe_path set, pdb should not mangle sys.path[0]""" + + script = textwrap.dedent(""" + import sys + import random + print('sys.path[0] is', sys.path[0]) + """) + commands = 'c\n' + + + with os_helper.temp_cwd() as cwd: + stdout, _ = self.run_pdb_script(script, commands, extra_env={'PYTHONSAFEPATH': '1'}) + + unexpected = f'sys.path[0] is {os.path.realpath(cwd)}' + self.assertNotIn(unexpected, stdout) + def test_issue42383(self): with os_helper.temp_cwd() as cwd: with open('foo.py', 'w') as f: diff --git a/Misc/NEWS.d/next/Library/2023-11-05-20-09-27.gh-issue-99367.HLaWKo.rst b/Misc/NEWS.d/next/Library/2023-11-05-20-09-27.gh-issue-99367.HLaWKo.rst new file mode 100644 index 00000000000000..0920da221e423f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-05-20-09-27.gh-issue-99367.HLaWKo.rst @@ -0,0 +1 @@ +Do not mangle ``sys.path[0]`` in :mod:`pdb` if safe_path is set From 562d7149c6944fb9e4c7be80664b2f2d5a12a3ea Mon Sep 17 00:00:00 2001 From: Eugene Toder Date: Mon, 27 Nov 2023 18:42:37 -0500 Subject: [PATCH 25/37] Correct documentation for AF_PACKET (#112339) Protocol in the address tuple should *not* be in the network-byte-order, because it is converted internally[1]. [1] https://github.com/python/cpython/blob/89ddea4886942b0c27a778a0ad3f0d5ac5f518f0/Modules/socketmodule.c#L2144 network byte order doesn't make sense for a python level int anyways. It's a fixed size C serialization concept. --- Doc/library/socket.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index e36fc17f89de24..e0a75304ef1606 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -185,7 +185,7 @@ created. Socket addresses are represented as follows: .. versionadded:: 3.7 - :const:`AF_PACKET` is a low-level interface directly to network devices. - The packets are represented by the tuple + The addresses are represented by the tuple ``(ifname, proto[, pkttype[, hatype[, addr]]])`` where: - *ifname* - String specifying the device name. @@ -193,7 +193,6 @@ created. Socket addresses are represented as follows: May be :data:`ETH_P_ALL` to capture all protocols, one of the :ref:`ETHERTYPE_* constants ` or any other Ethernet protocol number. - Value must be in network-byte-order. - *pkttype* - Optional integer specifying the packet type: - ``PACKET_HOST`` (the default) - Packet addressed to the local host. From cf2054059c08ef1c5546f24874191f341dc94eb9 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 28 Nov 2023 00:09:59 +0000 Subject: [PATCH 26/37] gh-112414: Add additional unit tests for calling `repr()` on a namespace package (#112475) Co-authored-by: Eric Snow --- .../test_importlib/import_/test___loader__.py | 4 --- Lib/test/test_module/__init__.py | 30 +++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_importlib/import_/test___loader__.py b/Lib/test/test_importlib/import_/test___loader__.py index c6996a42534676..858b37effc64bd 100644 --- a/Lib/test/test_importlib/import_/test___loader__.py +++ b/Lib/test/test_importlib/import_/test___loader__.py @@ -23,10 +23,6 @@ def test___loader__(self): with util.uncache('blah'), util.import_state(meta_path=[loader]): module = self.__import__('blah') self.assertEqual(loader, module.__loader__) - expected_repr_pattern = ( - r"\)>" - ) - self.assertRegex(repr(module), expected_repr_pattern) (Frozen_SpecTests, diff --git a/Lib/test/test_module/__init__.py b/Lib/test/test_module/__init__.py index db2133a9e8d17b..d49c44df4d839d 100644 --- a/Lib/test/test_module/__init__.py +++ b/Lib/test/test_module/__init__.py @@ -1,4 +1,5 @@ # Test the module type +import importlib.machinery import unittest import weakref from test.support import gc_collect @@ -264,6 +265,35 @@ def test_module_repr_source(self): self.assertEqual(r[-len(ends_with):], ends_with, '{!r} does not end with {!r}'.format(r, ends_with)) + def test_module_repr_with_namespace_package(self): + m = ModuleType('foo') + loader = importlib.machinery.NamespaceLoader('foo', ['bar'], 'baz') + spec = importlib.machinery.ModuleSpec('foo', loader) + m.__loader__ = loader + m.__spec__ = spec + self.assertEqual(repr(m), "") + + def test_module_repr_with_namespace_package_and_custom_loader(self): + m = ModuleType('foo') + loader = BareLoader() + spec = importlib.machinery.ModuleSpec('foo', loader) + m.__loader__ = loader + m.__spec__ = spec + expected_repr_pattern = r"\)>" + self.assertRegex(repr(m), expected_repr_pattern) + self.assertNotIn('from', repr(m)) + + def test_module_repr_with_fake_namespace_package(self): + m = ModuleType('foo') + loader = BareLoader() + loader._path = ['spam'] + spec = importlib.machinery.ModuleSpec('foo', loader) + m.__loader__ = loader + m.__spec__ = spec + expected_repr_pattern = r"\)>" + self.assertRegex(repr(m), expected_repr_pattern) + self.assertNotIn('from', repr(m)) + def test_module_finalization_at_shutdown(self): # Module globals and builtins should still be available during shutdown rc, out, err = assert_python_ok("-c", "from test.test_module import final_a") From 2e632fa07d13a58be62f59be4e656ad58b378f9b Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 28 Nov 2023 00:15:23 +0000 Subject: [PATCH 27/37] Docs: fix markup for `importlib.machinery.NamespaceLoader` (#112479) --- Doc/library/importlib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index fc954724bb72fe..2402bc5cd3ee2c 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1145,7 +1145,7 @@ find and load modules. .. versionadded:: 3.4 -.. class:: NamespaceLoader(name, path, path_finder): +.. class:: NamespaceLoader(name, path, path_finder) A concrete implementation of :class:`importlib.abc.InspectLoader` for namespace packages. This is an alias for a private class and is only made From 154f099e611cea74daa755c77df3b8003861cc76 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 28 Nov 2023 12:58:53 +1100 Subject: [PATCH 28/37] gh-112292 : Catch import error conditions with readline hooks (gh-112313) Prevents a segmentation fault in registered hooks for the readline library, but only when the readline module is loaded inside an isolated sub interpreter. The module is single-phase init so loading it fails, but not until the module init function has already run, where the readline hooks get registered. The readlinestate_global macro was error-prone to PyImport_FindModule returning NULL and crashing in about 18 places. I could reproduce 1 easily, but this PR replaces the macro with a function and adds error conditions to the other functions. --- ...-11-22-19-43-54.gh-issue-112292.5nDU87.rst | 2 + Modules/readline.c | 91 ++++++++++++++----- 2 files changed, 71 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-11-22-19-43-54.gh-issue-112292.5nDU87.rst diff --git a/Misc/NEWS.d/next/Library/2023-11-22-19-43-54.gh-issue-112292.5nDU87.rst b/Misc/NEWS.d/next/Library/2023-11-22-19-43-54.gh-issue-112292.5nDU87.rst new file mode 100644 index 00000000000000..8345e33791cde0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-22-19-43-54.gh-issue-112292.5nDU87.rst @@ -0,0 +1,2 @@ +Fix a crash in :mod:`readline` when imported from a sub interpreter. Patch +by Anthony Shaw diff --git a/Modules/readline.c b/Modules/readline.c index fde552d124bc77..209ac8bbcfbe78 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -147,8 +147,19 @@ readline_free(void *m) static PyModuleDef readlinemodule; -#define readlinestate_global ((readlinestate *)PyModule_GetState(PyState_FindModule(&readlinemodule))) - +static inline readlinestate* +get_hook_module_state(void) +{ + PyObject *mod = PyState_FindModule(&readlinemodule); + if (mod == NULL){ + PyErr_Clear(); + return NULL; + } + Py_INCREF(mod); + readlinestate *state = get_readline_state(mod); + Py_DECREF(mod); + return state; +} /* Convert to/from multibyte C strings */ @@ -438,14 +449,15 @@ readline_set_completion_display_matches_hook_impl(PyObject *module, PyObject *function) /*[clinic end generated code: output=516e5cb8db75a328 input=4f0bfd5ab0179a26]*/ { + readlinestate *state = get_readline_state(module); PyObject *result = set_hook("completion_display_matches_hook", - &readlinestate_global->completion_display_matches_hook, + &state->completion_display_matches_hook, function); #ifdef HAVE_RL_COMPLETION_DISPLAY_MATCHES_HOOK /* We cannot set this hook globally, since it replaces the default completion display. */ rl_completion_display_matches_hook = - readlinestate_global->completion_display_matches_hook ? + state->completion_display_matches_hook ? #if defined(HAVE_RL_COMPDISP_FUNC_T) (rl_compdisp_func_t *)on_completion_display_matches_hook : 0; #else @@ -472,7 +484,8 @@ static PyObject * readline_set_startup_hook_impl(PyObject *module, PyObject *function) /*[clinic end generated code: output=02cd0e0c4fa082ad input=7783b4334b26d16d]*/ { - return set_hook("startup_hook", &readlinestate_global->startup_hook, + readlinestate *state = get_readline_state(module); + return set_hook("startup_hook", &state->startup_hook, function); } @@ -497,7 +510,8 @@ static PyObject * readline_set_pre_input_hook_impl(PyObject *module, PyObject *function) /*[clinic end generated code: output=fe1a96505096f464 input=4f3eaeaf7ce1fdbe]*/ { - return set_hook("pre_input_hook", &readlinestate_global->pre_input_hook, + readlinestate *state = get_readline_state(module); + return set_hook("pre_input_hook", &state->pre_input_hook, function); } #endif @@ -530,7 +544,8 @@ static PyObject * readline_get_begidx_impl(PyObject *module) /*[clinic end generated code: output=362616ee8ed1b2b1 input=e083b81c8eb4bac3]*/ { - return Py_NewRef(readlinestate_global->begidx); + readlinestate *state = get_readline_state(module); + return Py_NewRef(state->begidx); } /* Get the ending index for the scope of the tab-completion */ @@ -545,7 +560,8 @@ static PyObject * readline_get_endidx_impl(PyObject *module) /*[clinic end generated code: output=7f763350b12d7517 input=d4c7e34a625fd770]*/ { - return Py_NewRef(readlinestate_global->endidx); + readlinestate *state = get_readline_state(module); + return Py_NewRef(state->endidx); } /* Set the tab-completion word-delimiters that readline uses */ @@ -772,7 +788,8 @@ static PyObject * readline_set_completer_impl(PyObject *module, PyObject *function) /*[clinic end generated code: output=171a2a60f81d3204 input=51e81e13118eb877]*/ { - return set_hook("completer", &readlinestate_global->completer, function); + readlinestate *state = get_readline_state(module); + return set_hook("completer", &state->completer, function); } /*[clinic input] @@ -785,10 +802,11 @@ static PyObject * readline_get_completer_impl(PyObject *module) /*[clinic end generated code: output=6e6bbd8226d14475 input=6457522e56d70d13]*/ { - if (readlinestate_global->completer == NULL) { + readlinestate *state = get_readline_state(module); + if (state->completer == NULL) { Py_RETURN_NONE; } - return Py_NewRef(readlinestate_global->completer); + return Py_NewRef(state->completer); } /* Private function to get current length of history. XXX It may be @@ -1026,7 +1044,12 @@ on_startup_hook(void) { int r; PyGILState_STATE gilstate = PyGILState_Ensure(); - r = on_hook(readlinestate_global->startup_hook); + readlinestate *state = get_hook_module_state(); + if (state == NULL) { + PyGILState_Release(gilstate); + return -1; + } + r = on_hook(state->startup_hook); PyGILState_Release(gilstate); return r; } @@ -1043,7 +1066,12 @@ on_pre_input_hook(void) { int r; PyGILState_STATE gilstate = PyGILState_Ensure(); - r = on_hook(readlinestate_global->pre_input_hook); + readlinestate *state = get_hook_module_state(); + if (state == NULL) { + PyGILState_Release(gilstate); + return -1; + } + r = on_hook(state->pre_input_hook); PyGILState_Release(gilstate); return r; } @@ -1060,6 +1088,11 @@ on_completion_display_matches_hook(char **matches, int i; PyObject *sub, *m=NULL, *s=NULL, *r=NULL; PyGILState_STATE gilstate = PyGILState_Ensure(); + readlinestate *state = get_hook_module_state(); + if (state == NULL) { + PyGILState_Release(gilstate); + return; + } m = PyList_New(num_matches); if (m == NULL) goto error; @@ -1070,7 +1103,7 @@ on_completion_display_matches_hook(char **matches, PyList_SET_ITEM(m, i, s); } sub = decode(matches[0]); - r = PyObject_CallFunction(readlinestate_global->completion_display_matches_hook, + r = PyObject_CallFunction(state->completion_display_matches_hook, "NNi", sub, m, max_length); m=NULL; @@ -1118,12 +1151,17 @@ static char * on_completion(const char *text, int state) { char *result = NULL; - if (readlinestate_global->completer != NULL) { + PyGILState_STATE gilstate = PyGILState_Ensure(); + readlinestate *module_state = get_hook_module_state(); + if (module_state == NULL) { + PyGILState_Release(gilstate); + return NULL; + } + if (module_state->completer != NULL) { PyObject *r = NULL, *t; - PyGILState_STATE gilstate = PyGILState_Ensure(); rl_attempted_completion_over = 1; t = decode(text); - r = PyObject_CallFunction(readlinestate_global->completer, "Ni", t, state); + r = PyObject_CallFunction(module_state->completer, "Ni", t, state); if (r == NULL) goto error; if (r == Py_None) { @@ -1145,6 +1183,7 @@ on_completion(const char *text, int state) PyGILState_Release(gilstate); return result; } + PyGILState_Release(gilstate); return result; } @@ -1160,6 +1199,7 @@ flex_complete(const char *text, int start, int end) size_t start_size, end_size; wchar_t *s; PyGILState_STATE gilstate = PyGILState_Ensure(); + readlinestate *state = get_hook_module_state(); #ifdef HAVE_RL_COMPLETION_APPEND_CHARACTER rl_completion_append_character ='\0'; #endif @@ -1187,10 +1227,12 @@ flex_complete(const char *text, int start, int end) end = start + (int)end_size; done: - Py_XDECREF(readlinestate_global->begidx); - Py_XDECREF(readlinestate_global->endidx); - readlinestate_global->begidx = PyLong_FromLong((long) start); - readlinestate_global->endidx = PyLong_FromLong((long) end); + if (state) { + Py_XDECREF(state->begidx); + Py_XDECREF(state->endidx); + state->begidx = PyLong_FromLong((long) start); + state->endidx = PyLong_FromLong((long) end); + } result = completion_matches((char *)text, *on_completion); PyGILState_Release(gilstate); return result; @@ -1511,12 +1553,17 @@ PyInit_readline(void) } mod_state = (readlinestate *) PyModule_GetState(m); + if (mod_state == NULL){ + goto error; + } PyOS_ReadlineFunctionPointer = call_readline; if (setup_readline(mod_state) < 0) { PyErr_NoMemory(); goto error; } - + if (PyErr_Occurred()){ + goto error; + } return m; error: From ac4b44266d61651aea5928ce7d3fae4de226f83d Mon Sep 17 00:00:00 2001 From: Radislav Chugunov <52372310+chgnrdv@users.noreply.github.com> Date: Tue, 28 Nov 2023 06:27:39 +0300 Subject: [PATCH 29/37] gh-112071: Make `_random.Random` methods thread-safe in `--disable-gil` builds (gh-112128) Co-authored-by: Donghee Na --- Modules/_randommodule.c | 18 +++++++++------ Modules/clinic/_randommodule.c.h | 38 +++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 514bec16a347f4..4403e1d132c057 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -175,6 +175,7 @@ genrand_uint32(RandomObject *self) */ /*[clinic input] +@critical_section _random.Random.random self: self(type="RandomObject *") @@ -184,7 +185,7 @@ random() -> x in the interval [0, 1). static PyObject * _random_Random_random_impl(RandomObject *self) -/*[clinic end generated code: output=117ff99ee53d755c input=afb2a59cbbb00349]*/ +/*[clinic end generated code: output=117ff99ee53d755c input=26492e52d26e8b7b]*/ { uint32_t a=genrand_uint32(self)>>5, b=genrand_uint32(self)>>6; return PyFloat_FromDouble((a*67108864.0+b)*(1.0/9007199254740992.0)); @@ -368,6 +369,7 @@ random_seed(RandomObject *self, PyObject *arg) } /*[clinic input] +@critical_section _random.Random.seed self: self(type="RandomObject *") @@ -382,7 +384,7 @@ of the current time and the process identifier. static PyObject * _random_Random_seed_impl(RandomObject *self, PyObject *n) -/*[clinic end generated code: output=0fad1e16ba883681 input=78d6ef0d52532a54]*/ +/*[clinic end generated code: output=0fad1e16ba883681 input=46d01d2ba938c7b1]*/ { if (random_seed(self, n) < 0) { return NULL; @@ -391,6 +393,7 @@ _random_Random_seed_impl(RandomObject *self, PyObject *n) } /*[clinic input] +@critical_section _random.Random.getstate self: self(type="RandomObject *") @@ -400,7 +403,7 @@ getstate() -> tuple containing the current state. static PyObject * _random_Random_getstate_impl(RandomObject *self) -/*[clinic end generated code: output=bf6cef0c092c7180 input=b937a487928c0e89]*/ +/*[clinic end generated code: output=bf6cef0c092c7180 input=b6621f31eb639694]*/ { PyObject *state; PyObject *element; @@ -428,6 +431,7 @@ _random_Random_getstate_impl(RandomObject *self) /*[clinic input] +@critical_section _random.Random.setstate self: self(type="RandomObject *") @@ -438,8 +442,8 @@ setstate(state) -> None. Restores generator state. [clinic start generated code]*/ static PyObject * -_random_Random_setstate(RandomObject *self, PyObject *state) -/*[clinic end generated code: output=fd1c3cd0037b6681 input=b3b4efbb1bc66af8]*/ +_random_Random_setstate_impl(RandomObject *self, PyObject *state) +/*[clinic end generated code: output=babfc2c2eac6b027 input=358e898ec07469b7]*/ { int i; unsigned long element; @@ -479,7 +483,7 @@ _random_Random_setstate(RandomObject *self, PyObject *state) } /*[clinic input] - +@critical_section _random.Random.getrandbits self: self(type="RandomObject *") @@ -491,7 +495,7 @@ getrandbits(k) -> x. Generates an int with k random bits. static PyObject * _random_Random_getrandbits_impl(RandomObject *self, int k) -/*[clinic end generated code: output=b402f82a2158887f input=8c0e6396dd176fc0]*/ +/*[clinic end generated code: output=b402f82a2158887f input=87603cd60f79f730]*/ { int i, words; uint32_t r; diff --git a/Modules/clinic/_randommodule.c.h b/Modules/clinic/_randommodule.c.h index 757e49e23cacfb..6193acac67e7ac 100644 --- a/Modules/clinic/_randommodule.c.h +++ b/Modules/clinic/_randommodule.c.h @@ -2,6 +2,7 @@ preserve [clinic start generated code]*/ +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_random_Random_random__doc__, @@ -19,7 +20,13 @@ _random_Random_random_impl(RandomObject *self); static PyObject * _random_Random_random(RandomObject *self, PyObject *Py_UNUSED(ignored)) { - return _random_Random_random_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _random_Random_random_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_random_Random_seed__doc__, @@ -51,7 +58,9 @@ _random_Random_seed(RandomObject *self, PyObject *const *args, Py_ssize_t nargs) } n = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _random_Random_seed_impl(self, n); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -72,7 +81,13 @@ _random_Random_getstate_impl(RandomObject *self); static PyObject * _random_Random_getstate(RandomObject *self, PyObject *Py_UNUSED(ignored)) { - return _random_Random_getstate_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _random_Random_getstate_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_random_Random_setstate__doc__, @@ -84,6 +99,21 @@ PyDoc_STRVAR(_random_Random_setstate__doc__, #define _RANDOM_RANDOM_SETSTATE_METHODDEF \ {"setstate", (PyCFunction)_random_Random_setstate, METH_O, _random_Random_setstate__doc__}, +static PyObject * +_random_Random_setstate_impl(RandomObject *self, PyObject *state); + +static PyObject * +_random_Random_setstate(RandomObject *self, PyObject *state) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _random_Random_setstate_impl(self, state); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_random_Random_getrandbits__doc__, "getrandbits($self, k, /)\n" "--\n" @@ -106,9 +136,11 @@ _random_Random_getrandbits(RandomObject *self, PyObject *arg) if (k == -1 && PyErr_Occurred()) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _random_Random_getrandbits_impl(self, k); + Py_END_CRITICAL_SECTION(); exit: return return_value; } -/*[clinic end generated code: output=5c800a28c2d7b9d1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bf49ece1d341b1b6 input=a9049054013a1b77]*/ From 2df26d83486b8f9ac6b7df2a9a4669508aa61983 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 27 Nov 2023 21:23:23 -0900 Subject: [PATCH 30/37] gh-112105: Make completer delims work on libedit (gh-112106) --- Lib/test/test_readline.py | 20 +++++++++++++++++++ ...-11-15-04-53-37.gh-issue-112105.I3RcVN.rst | 1 + Modules/readline.c | 16 +++++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-11-15-04-53-37.gh-issue-112105.I3RcVN.rst diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 835280f2281cde..6c2726d3209ecf 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -5,6 +5,7 @@ import os import sys import tempfile +import textwrap import unittest from test.support import verbose from test.support.import_helper import import_module @@ -163,6 +164,25 @@ def test_auto_history_disabled(self): # end, so don't expect it in the output. self.assertIn(b"History length: 0", output) + def test_set_complete_delims(self): + script = textwrap.dedent(""" + import readline + def complete(text, state): + if state == 0 and text == "$": + return "$complete" + return None + if "libedit" in getattr(readline, "__doc__", ""): + readline.parse_and_bind(r'bind "\\t" rl_complete') + else: + readline.parse_and_bind(r'"\\t": complete') + readline.set_completer_delims(" \\t\\n") + readline.set_completer(complete) + print(input()) + """) + + output = run_pty(script, input=b"$\t\n") + self.assertIn(b"$complete", output) + def test_nonascii(self): loc = locale.setlocale(locale.LC_CTYPE, None) if loc in ('C', 'POSIX'): diff --git a/Misc/NEWS.d/next/Library/2023-11-15-04-53-37.gh-issue-112105.I3RcVN.rst b/Misc/NEWS.d/next/Library/2023-11-15-04-53-37.gh-issue-112105.I3RcVN.rst new file mode 100644 index 00000000000000..4243dcb190434f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-15-04-53-37.gh-issue-112105.I3RcVN.rst @@ -0,0 +1 @@ +Make :func:`readline.set_completer_delims` work with libedit diff --git a/Modules/readline.c b/Modules/readline.c index 209ac8bbcfbe78..eb9a3d4693ee90 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -592,6 +592,13 @@ readline_set_completer_delims(PyObject *module, PyObject *string) if (break_chars) { free(completer_word_break_characters); completer_word_break_characters = break_chars; +#ifdef WITH_EDITLINE + rl_basic_word_break_characters = break_chars; +#else + if (using_libedit_emulation) { + rl_basic_word_break_characters = break_chars; + } +#endif rl_completer_word_break_characters = break_chars; Py_RETURN_NONE; } @@ -1309,6 +1316,15 @@ setup_readline(readlinestate *mod_state) completer_word_break_characters = strdup(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?"); /* All nonalphanums except '.' */ +#ifdef WITH_EDITLINE + // libedit uses rl_basic_word_break_characters instead of + // rl_completer_word_break_characters as complete delimiter + rl_basic_word_break_characters = completer_word_break_characters; +#else + if (using_libedit_emulation) { + rl_basic_word_break_characters = completer_word_break_characters; + } +#endif rl_completer_word_break_characters = completer_word_break_characters; mod_state->begidx = PyLong_FromLong(0L); From 2c68011780bd68463f5183601ea9c10af368dff6 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 28 Nov 2023 08:03:25 +0000 Subject: [PATCH 31/37] gh-112332: Deprecate TracebackException.exc_type, add exc_type_str. (#112333) --- Doc/library/traceback.rst | 8 +++ Doc/whatsnew/3.13.rst | 11 ++++ Lib/test/test_traceback.py | 51 ++++++++++++++++--- Lib/traceback.py | 50 +++++++++++++----- ...-11-23-10-41-21.gh-issue-112332.rhTBaa.rst | 2 + 5 files changed, 102 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-11-23-10-41-21.gh-issue-112332.rhTBaa.rst diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 408da7fc5f0645..80dda5ec520d7a 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -287,6 +287,14 @@ capture data for later printing in a lightweight fashion. The class of the original traceback. + .. deprecated:: 3.13 + + .. attribute:: exc_type_str + + String display of the class of the original exception. + + .. versionadded:: 3.13 + .. attribute:: filename For syntax errors - the file name where the error occurred. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index bf6a70f2009b8e..198ea3a4b57bde 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -318,6 +318,12 @@ traceback to format the nested exceptions of a :exc:`BaseExceptionGroup` instance, recursively. (Contributed by Irit Katriel in :gh:`105292`.) +* Add the field *exc_type_str* to :class:`~traceback.TracebackException`, which + holds a string display of the *exc_type*. Deprecate the field *exc_type* + which holds the type object itself. Add parameter *save_exc_type* (default + ``True``) to indicate whether ``exc_type`` should be saved. + (Contributed by Irit Katriel in :gh:`112332`.) + typing ------ @@ -377,6 +383,11 @@ Deprecated security and functionality bugs. This includes removal of the ``--cgi`` flag to the ``python -m http.server`` command line in 3.15. +* :mod:`traceback`: + + * The field *exc_type* of :class:`traceback.TracebackException` is + deprecated. Use *exc_type_str* instead. + * :mod:`typing`: * Creating a :class:`typing.NamedTuple` class using keyword arguments to denote diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index b43dca6f640b9a..c58d979bdd0115 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -2715,9 +2715,9 @@ def __repr__(self) -> str: class TestTracebackException(unittest.TestCase): - def test_smoke(self): + def do_test_smoke(self, exc, expected_type_str): try: - 1/0 + raise exc except Exception as e: exc_obj = e exc = traceback.TracebackException.from_exception(e) @@ -2727,9 +2727,23 @@ def test_smoke(self): self.assertEqual(None, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(type(exc_obj), exc.exc_type) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(expected_type_str, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) + def test_smoke_builtin(self): + self.do_test_smoke(ValueError(42), 'ValueError') + + def test_smoke_user_exception(self): + class MyException(Exception): + pass + + self.do_test_smoke( + MyException('bad things happened'), + ('test.test_traceback.TestTracebackException.' + 'test_smoke_user_exception..MyException')) + def test_from_exception(self): # Check all the parameters are accepted. def foo(): @@ -2750,7 +2764,9 @@ def foo(): self.assertEqual(None, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(type(exc_obj), exc.exc_type) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) def test_cause(self): @@ -2772,7 +2788,9 @@ def test_cause(self): self.assertEqual(exc_context, exc.__context__) self.assertEqual(True, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(type(exc_obj), exc.exc_type) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) def test_context(self): @@ -2792,7 +2810,9 @@ def test_context(self): self.assertEqual(exc_context, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(type(exc_obj), exc.exc_type) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) def test_long_context_chain(self): @@ -2837,7 +2857,9 @@ def test_compact_with_cause(self): self.assertEqual(None, exc.__context__) self.assertEqual(True, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(type(exc_obj), exc.exc_type) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) def test_compact_no_cause(self): @@ -2857,9 +2879,22 @@ def test_compact_no_cause(self): self.assertEqual(exc_context, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(type(exc_obj), exc.exc_type) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) + def test_no_save_exc_type(self): + try: + 1/0 + except Exception as e: + exc = e + + te = traceback.TracebackException.from_exception( + exc, save_exc_type=False) + with self.assertWarns(DeprecationWarning): + self.assertIsNone(te.exc_type) + def test_no_refs_to_exception_and_traceback_objects(self): try: 1/0 diff --git a/Lib/traceback.py b/Lib/traceback.py index b25a7291f6be51..5d83f85ac3edb0 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -5,6 +5,7 @@ import linecache import sys import textwrap +import warnings from contextlib import suppress __all__ = ['extract_stack', 'extract_tb', 'format_exception', @@ -719,7 +720,8 @@ class TracebackException: - :attr:`__suppress_context__` The *__suppress_context__* value from the original exception. - :attr:`stack` A `StackSummary` representing the traceback. - - :attr:`exc_type` The class of the original traceback. + - :attr:`exc_type` (deprecated) The class of the original traceback. + - :attr:`exc_type_str` String display of exc_type - :attr:`filename` For syntax errors - the filename where the error occurred. - :attr:`lineno` For syntax errors - the linenumber where the error @@ -737,7 +739,7 @@ class TracebackException: def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, - max_group_width=15, max_group_depth=10, _seen=None): + max_group_width=15, max_group_depth=10, save_exc_type=True, _seen=None): # NB: we need to accept exc_traceback, exc_value, exc_traceback to # permit backwards compat with the existing API, otherwise we # need stub thunk objects just to glue it together. @@ -754,12 +756,23 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, _walk_tb_with_full_positions(exc_traceback), limit=limit, lookup_lines=lookup_lines, capture_locals=capture_locals) - self.exc_type = exc_type + + self._exc_type = exc_type if save_exc_type else None + # Capture now to permit freeing resources: only complication is in the # unofficial API _format_final_exc_line self._str = _safe_string(exc_value, 'exception') self.__notes__ = getattr(exc_value, '__notes__', None) + self._is_syntax_error = False + self._have_exc_type = exc_type is not None + if exc_type is not None: + self.exc_type_qualname = exc_type.__qualname__ + self.exc_type_module = exc_type.__module__ + else: + self.exc_type_qualname = None + self.exc_type_module = None + if exc_type and issubclass(exc_type, SyntaxError): # Handle SyntaxError's specially self.filename = exc_value.filename @@ -771,6 +784,7 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, self.offset = exc_value.offset self.end_offset = exc_value.end_offset self.msg = exc_value.msg + self._is_syntax_error = True elif exc_type and issubclass(exc_type, ImportError) and \ getattr(exc_value, "name_from", None) is not None: wrong_name = getattr(exc_value, "name_from", None) @@ -869,6 +883,24 @@ def from_exception(cls, exc, *args, **kwargs): """Create a TracebackException from an exception.""" return cls(type(exc), exc, exc.__traceback__, *args, **kwargs) + @property + def exc_type(self): + warnings.warn('Deprecated in 3.13. Use exc_type_str instead.', + DeprecationWarning, stacklevel=2) + return self._exc_type + + @property + def exc_type_str(self): + if not self._have_exc_type: + return None + stype = self.exc_type_qualname + smod = self.exc_type_module + if smod not in ("__main__", "builtins"): + if not isinstance(smod, str): + smod = "" + stype = smod + '.' + stype + return stype + def _load_lines(self): """Private API. force all lines in the stack to be loaded.""" for frame in self.stack: @@ -901,18 +933,12 @@ def format_exception_only(self, *, show_group=False, _depth=0): """ indent = 3 * _depth * ' ' - if self.exc_type is None: + if not self._have_exc_type: yield indent + _format_final_exc_line(None, self._str) return - stype = self.exc_type.__qualname__ - smod = self.exc_type.__module__ - if smod not in ("__main__", "builtins"): - if not isinstance(smod, str): - smod = "" - stype = smod + '.' + stype - - if not issubclass(self.exc_type, SyntaxError): + stype = self.exc_type_str + if not self._is_syntax_error: if _depth > 0: # Nested exceptions needs correct handling of multiline messages. formatted = _format_final_exc_line( diff --git a/Misc/NEWS.d/next/Library/2023-11-23-10-41-21.gh-issue-112332.rhTBaa.rst b/Misc/NEWS.d/next/Library/2023-11-23-10-41-21.gh-issue-112332.rhTBaa.rst new file mode 100644 index 00000000000000..bd686ad052e5b2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-23-10-41-21.gh-issue-112332.rhTBaa.rst @@ -0,0 +1,2 @@ +Deprecate the ``exc_type`` field of :class:`traceback.TracebackException`. +Add ``exc_type_str`` to replace it. From f14d741daa1b9e5b9c9fc1edba93d0fa92b5ba8d Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Tue, 28 Nov 2023 11:18:33 +0300 Subject: [PATCH 32/37] gh-109802: Increase test coverage for complexobject.c (GH-112452) --- Lib/test/test_capi/test_complex.py | 87 ++++++++++++++++++++++++++++++ Lib/test/test_complex.py | 47 ++++++++++++++++ Modules/_testcapi/complex.c | 59 ++++++++++++++++++++ 3 files changed, 193 insertions(+) diff --git a/Lib/test/test_capi/test_complex.py b/Lib/test/test_capi/test_complex.py index 9f51efb091dea5..d6fc1f077c40aa 100644 --- a/Lib/test/test_capi/test_complex.py +++ b/Lib/test/test_capi/test_complex.py @@ -1,3 +1,5 @@ +from math import isnan +import errno import unittest import warnings @@ -10,6 +12,10 @@ _testcapi = import_helper.import_module('_testcapi') NULL = None +INF = float("inf") +NAN = float("nan") +DBL_MAX = _testcapi.DBL_MAX + class BadComplex3: def __complex__(self): @@ -141,6 +147,87 @@ def test_asccomplex(self): # CRASHES asccomplex(NULL) + def test_py_c_sum(self): + # Test _Py_c_sum() + _py_c_sum = _testcapi._py_c_sum + + self.assertEqual(_py_c_sum(1, 1j), (1+1j, 0)) + + def test_py_c_diff(self): + # Test _Py_c_diff() + _py_c_diff = _testcapi._py_c_diff + + self.assertEqual(_py_c_diff(1, 1j), (1-1j, 0)) + + def test_py_c_neg(self): + # Test _Py_c_neg() + _py_c_neg = _testcapi._py_c_neg + + self.assertEqual(_py_c_neg(1+1j), -1-1j) + + def test_py_c_prod(self): + # Test _Py_c_prod() + _py_c_prod = _testcapi._py_c_prod + + self.assertEqual(_py_c_prod(2, 1j), (2j, 0)) + + def test_py_c_quot(self): + # Test _Py_c_quot() + _py_c_quot = _testcapi._py_c_quot + + self.assertEqual(_py_c_quot(1, 1j), (-1j, 0)) + self.assertEqual(_py_c_quot(1, -1j), (1j, 0)) + self.assertEqual(_py_c_quot(1j, 2), (0.5j, 0)) + self.assertEqual(_py_c_quot(1j, -2), (-0.5j, 0)) + self.assertEqual(_py_c_quot(1, 2j), (-0.5j, 0)) + + z, e = _py_c_quot(NAN, 1j) + self.assertTrue(isnan(z.real)) + self.assertTrue(isnan(z.imag)) + self.assertEqual(e, 0) + + z, e = _py_c_quot(1j, NAN) + self.assertTrue(isnan(z.real)) + self.assertTrue(isnan(z.imag)) + self.assertEqual(e, 0) + + self.assertEqual(_py_c_quot(1, 0j)[1], errno.EDOM) + + def test_py_c_pow(self): + # Test _Py_c_pow() + _py_c_pow = _testcapi._py_c_pow + + self.assertEqual(_py_c_pow(1j, 0j), (1+0j, 0)) + self.assertEqual(_py_c_pow(1, 1j), (1+0j, 0)) + self.assertEqual(_py_c_pow(0j, 1), (0j, 0)) + self.assertAlmostEqual(_py_c_pow(1j, 2)[0], -1.0+0j) + + r, e = _py_c_pow(1+1j, -1) + self.assertAlmostEqual(r, 0.5-0.5j) + self.assertEqual(e, 0) + + self.assertEqual(_py_c_pow(0j, -1)[1], errno.EDOM) + self.assertEqual(_py_c_pow(0j, 1j)[1], errno.EDOM) + self.assertEqual(_py_c_pow(*[DBL_MAX+1j]*2)[0], complex(*[INF]*2)) + + + def test_py_c_abs(self): + # Test _Py_c_abs() + _py_c_abs = _testcapi._py_c_abs + + self.assertEqual(_py_c_abs(-1), (1.0, 0)) + self.assertEqual(_py_c_abs(1j), (1.0, 0)) + + self.assertEqual(_py_c_abs(complex('+inf+1j')), (INF, 0)) + self.assertEqual(_py_c_abs(complex('-inf+1j')), (INF, 0)) + self.assertEqual(_py_c_abs(complex('1.25+infj')), (INF, 0)) + self.assertEqual(_py_c_abs(complex('1.25-infj')), (INF, 0)) + + self.assertTrue(isnan(_py_c_abs(complex('1.25+nanj'))[0])) + self.assertTrue(isnan(_py_c_abs(complex('nan-1j'))[0])) + + self.assertEqual(_py_c_abs(complex(*[DBL_MAX]*2))[1], errno.ERANGE) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index 9180cca62b28b8..b057121f285dc7 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -109,6 +109,8 @@ def test_truediv(self): complex(random(), random())) self.assertAlmostEqual(complex.__truediv__(2+0j, 1+1j), 1-1j) + self.assertRaises(TypeError, operator.truediv, 1j, None) + self.assertRaises(TypeError, operator.truediv, None, 1j) for denom_real, denom_imag in [(0, NAN), (NAN, 0), (NAN, NAN)]: z = complex(0, 0) / complex(denom_real, denom_imag) @@ -140,6 +142,7 @@ def test_floordiv_zero_division(self): def test_richcompare(self): self.assertIs(complex.__eq__(1+1j, 1<<10000), False) self.assertIs(complex.__lt__(1+1j, None), NotImplemented) + self.assertIs(complex.__eq__(1+1j, None), NotImplemented) self.assertIs(complex.__eq__(1+1j, 1+1j), True) self.assertIs(complex.__eq__(1+1j, 2+2j), False) self.assertIs(complex.__ne__(1+1j, 1+1j), False) @@ -162,6 +165,7 @@ def test_richcompare(self): self.assertIs(operator.eq(1+1j, 2+2j), False) self.assertIs(operator.ne(1+1j, 1+1j), False) self.assertIs(operator.ne(1+1j, 2+2j), True) + self.assertIs(operator.eq(1+1j, 2.0), False) def test_richcompare_boundaries(self): def check(n, deltas, is_equal, imag = 0.0): @@ -180,6 +184,27 @@ def check(n, deltas, is_equal, imag = 0.0): check(2 ** pow, range(1, 101), lambda delta: False, float(i)) check(2 ** 53, range(-100, 0), lambda delta: True) + def test_add(self): + self.assertEqual(1j + int(+1), complex(+1, 1)) + self.assertEqual(1j + int(-1), complex(-1, 1)) + self.assertRaises(OverflowError, operator.add, 1j, 10**1000) + self.assertRaises(TypeError, operator.add, 1j, None) + self.assertRaises(TypeError, operator.add, None, 1j) + + def test_sub(self): + self.assertEqual(1j - int(+1), complex(-1, 1)) + self.assertEqual(1j - int(-1), complex(1, 1)) + self.assertRaises(OverflowError, operator.sub, 1j, 10**1000) + self.assertRaises(TypeError, operator.sub, 1j, None) + self.assertRaises(TypeError, operator.sub, None, 1j) + + def test_mul(self): + self.assertEqual(1j * int(20), complex(0, 20)) + self.assertEqual(1j * int(-1), complex(0, -1)) + self.assertRaises(OverflowError, operator.mul, 1j, 10**1000) + self.assertRaises(TypeError, operator.mul, 1j, None) + self.assertRaises(TypeError, operator.mul, None, 1j) + def test_mod(self): # % is no longer supported on complex numbers with self.assertRaises(TypeError): @@ -212,11 +237,18 @@ def test_divmod_zero_division(self): def test_pow(self): self.assertAlmostEqual(pow(1+1j, 0+0j), 1.0) self.assertAlmostEqual(pow(0+0j, 2+0j), 0.0) + self.assertEqual(pow(0+0j, 2000+0j), 0.0) + self.assertEqual(pow(0, 0+0j), 1.0) + self.assertEqual(pow(-1, 0+0j), 1.0) self.assertRaises(ZeroDivisionError, pow, 0+0j, 1j) + self.assertRaises(ZeroDivisionError, pow, 0+0j, -1000) self.assertAlmostEqual(pow(1j, -1), 1/1j) self.assertAlmostEqual(pow(1j, 200), 1) self.assertRaises(ValueError, pow, 1+1j, 1+1j, 1+1j) self.assertRaises(OverflowError, pow, 1e200+1j, 1e200+1j) + self.assertRaises(TypeError, pow, 1j, None) + self.assertRaises(TypeError, pow, None, 1j) + self.assertAlmostEqual(pow(1j, 0.5), 0.7071067811865476+0.7071067811865475j) a = 3.33+4.43j self.assertEqual(a ** 0j, 1) @@ -301,6 +333,7 @@ def test_boolcontext(self): for i in range(100): self.assertTrue(complex(random() + 1e-6, random() + 1e-6)) self.assertTrue(not complex(0.0, 0.0)) + self.assertTrue(1j) def test_conjugate(self): self.assertClose(complex(5.3, 9.8).conjugate(), 5.3-9.8j) @@ -314,6 +347,8 @@ def __complex__(self): return self.value self.assertRaises(TypeError, complex, {}) self.assertRaises(TypeError, complex, NS(1.5)) self.assertRaises(TypeError, complex, NS(1)) + self.assertRaises(TypeError, complex, object()) + self.assertRaises(TypeError, complex, NS(4.25+0.5j), object()) self.assertAlmostEqual(complex("1+10j"), 1+10j) self.assertAlmostEqual(complex(10), 10+0j) @@ -359,6 +394,8 @@ def __complex__(self): return self.value self.assertAlmostEqual(complex('1e-500'), 0.0 + 0.0j) self.assertAlmostEqual(complex('-1e-500j'), 0.0 - 0.0j) self.assertAlmostEqual(complex('-1e-500+1e-500j'), -0.0 + 0.0j) + self.assertEqual(complex('1-1j'), 1.0 - 1j) + self.assertEqual(complex('1J'), 1j) class complex2(complex): pass self.assertAlmostEqual(complex(complex2(1+1j)), 1+1j) @@ -553,6 +590,8 @@ def test_hash(self): x /= 3.0 # now check against floating point self.assertEqual(hash(x), hash(complex(x, 0.))) + self.assertNotEqual(hash(2000005 - 1j), -1) + def test_abs(self): nums = [complex(x/3., y/7.) for x in range(-9,9) for y in range(-9,9)] for num in nums: @@ -602,6 +641,14 @@ def test(v, expected, test_fn=self.assertEqual): test(complex(-0., 0.), "(-0+0j)") test(complex(-0., -0.), "(-0-0j)") + def test_pos(self): + class ComplexSubclass(complex): + pass + + self.assertEqual(+(1+6j), 1+6j) + self.assertEqual(+ComplexSubclass(1, 6), 1+6j) + self.assertIs(type(+ComplexSubclass(1, 6)), complex) + def test_neg(self): self.assertEqual(-(1+6j), -1-6j) diff --git a/Modules/_testcapi/complex.c b/Modules/_testcapi/complex.c index 400f4054c613ee..4a70217eb90d62 100644 --- a/Modules/_testcapi/complex.c +++ b/Modules/_testcapi/complex.c @@ -85,6 +85,58 @@ complex_asccomplex(PyObject *Py_UNUSED(module), PyObject *obj) return PyComplex_FromCComplex(complex); } +static PyObject* +_py_c_neg(PyObject *Py_UNUSED(module), PyObject *num) +{ + Py_complex complex; + + complex = PyComplex_AsCComplex(num); + if (complex.real == -1. && PyErr_Occurred()) { + return NULL; + } + + return PyComplex_FromCComplex(_Py_c_neg(complex)); +} + +#define _PY_C_FUNC2(suffix) \ + static PyObject * \ + _py_c_##suffix(PyObject *Py_UNUSED(module), PyObject *args) \ + { \ + Py_complex num, exp, res; \ + \ + if (!PyArg_ParseTuple(args, "DD", &num, &exp)) { \ + return NULL; \ + } \ + \ + errno = 0; \ + res = _Py_c_##suffix(num, exp); \ + return Py_BuildValue("Di", &res, errno); \ + }; + +_PY_C_FUNC2(sum) +_PY_C_FUNC2(diff) +_PY_C_FUNC2(prod) +_PY_C_FUNC2(quot) +_PY_C_FUNC2(pow) + +static PyObject* +_py_c_abs(PyObject *Py_UNUSED(module), PyObject* obj) +{ + Py_complex complex; + double res; + + NULLABLE(obj); + complex = PyComplex_AsCComplex(obj); + + if (complex.real == -1. && PyErr_Occurred()) { + return NULL; + } + + errno = 0; + res = _Py_c_abs(complex); + return Py_BuildValue("di", res, errno); +} + static PyMethodDef test_methods[] = { {"complex_check", complex_check, METH_O}, @@ -94,6 +146,13 @@ static PyMethodDef test_methods[] = { {"complex_realasdouble", complex_realasdouble, METH_O}, {"complex_imagasdouble", complex_imagasdouble, METH_O}, {"complex_asccomplex", complex_asccomplex, METH_O}, + {"_py_c_sum", _py_c_sum, METH_VARARGS}, + {"_py_c_diff", _py_c_diff, METH_VARARGS}, + {"_py_c_neg", _py_c_neg, METH_O}, + {"_py_c_prod", _py_c_prod, METH_VARARGS}, + {"_py_c_quot", _py_c_quot, METH_VARARGS}, + {"_py_c_pow", _py_c_pow, METH_VARARGS}, + {"_py_c_abs", _py_c_abs, METH_O}, {NULL}, }; From a194938f33a71e727e53490815bae874eece1460 Mon Sep 17 00:00:00 2001 From: James Morris <6653392+J-M0@users.noreply.github.com> Date: Tue, 28 Nov 2023 04:24:59 -0500 Subject: [PATCH 33/37] gh-112431: Unconditionally call `hash -r` (GH-112432) The `activate` script calls `hash -r` in two places to make sure the shell picks up the environment changes the script makes. Before that, it checks to see if the shell running the script is bash or zsh. `hash -r` is specified by POSIX and is not exclusive to bash and zsh. This guard prevents the script from calling `hash -r` in other `#!/bin/sh`-compatible shells like dash. --- Lib/venv/scripts/common/activate | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index 8398981ce53b9c..6fdf423a1c516a 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -14,12 +14,9 @@ deactivate () { unset _OLD_VIRTUAL_PYTHONHOME fi - # This should detect bash and zsh, which have a hash command that must - # be called to get it to forget past commands. Without forgetting + # Call hash to forget past commands. Without forgetting # past commands the $PATH changes we made may not be respected - if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null - fi + hash -r 2> /dev/null if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then PS1="${_OLD_VIRTUAL_PS1:-}" @@ -69,9 +66,6 @@ if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then export PS1 fi -# This should detect bash and zsh, which have a hash command that must -# be called to get it to forget past commands. Without forgetting +# Call hash to forget past commands. Without forgetting # past commands the $PATH changes we made may not be respected -if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null -fi +hash -r 2> /dev/null From 3fdf7ae3d1a08894e53c263945fba67fe62ac05b Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Tue, 28 Nov 2023 16:46:00 +0200 Subject: [PATCH 34/37] gh-110930: Correct book title by Alan D. Moore (#112490) --- Doc/library/tkinter.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index a8c67b02a23e4d..e084d8554c7c09 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -58,8 +58,8 @@ details that are unchanged. * `Modern Tkinter for Busy Python Developers `_ By Mark Roseman. (ISBN 978-1999149567) - * `Python and Tkinter Programming `_ - By Alan Moore. (ISBN 978-1788835886) + * `Python GUI programming with Tkinter `_ + By Alan D. Moore. (ISBN 978-1788835886) * `Programming Python `_ By Mark Lutz; has excellent coverage of Tkinter. (ISBN 978-0596158101) From 48dfd74a9db9d4aa9c6f23b4a67b461e5d977173 Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Tue, 28 Nov 2023 07:45:03 -0800 Subject: [PATCH 35/37] GH-112245: Promote free threaded CI (#112246) --- .github/workflows/build.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4210194b978749..cfb36c8c32e18d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -190,7 +190,7 @@ jobs: build_windows_free_threaded: name: 'Windows (free-threaded)' needs: check_source - if: needs.check_source.outputs.run_tests == 'true' && contains(github.event.pull_request.labels.*.name, 'topic-free-threaded') + if: needs.check_source.outputs.run_tests == 'true' uses: ./.github/workflows/reusable-windows.yml with: free-threaded: true @@ -206,7 +206,7 @@ jobs: build_macos_free_threaded: name: 'macOS (free-threaded)' needs: check_source - if: needs.check_source.outputs.run_tests == 'true' && contains(github.event.pull_request.labels.*.name, 'topic-free-threaded') + if: needs.check_source.outputs.run_tests == 'true' uses: ./.github/workflows/reusable-macos.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} @@ -228,7 +228,7 @@ jobs: build_ubuntu_free_threaded: name: 'Ubuntu (free-threaded)' needs: check_source - if: needs.check_source.outputs.run_tests == 'true' && contains(github.event.pull_request.labels.*.name, 'topic-free-threaded') + if: needs.check_source.outputs.run_tests == 'true' uses: ./.github/workflows/reusable-ubuntu.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} @@ -521,10 +521,7 @@ jobs: uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe with: allowed-failures: >- - build_macos_free_threaded, - build_ubuntu_free_threaded, build_ubuntu_ssltests, - build_windows_free_threaded, cifuzz, test_hypothesis, allowed-skips: >- From e413daf5f6b983bdb4e1965d76b5313cb93b266e Mon Sep 17 00:00:00 2001 From: Grant Ramsay Date: Wed, 29 Nov 2023 13:15:39 +1300 Subject: [PATCH 36/37] gh-112454: Disable TLS-PSK if OpenSSL was built without PSK support (#112491) If OpenSSL was built without PSK support, the python TLS-PSK methods will raise "NotImplementedError" if called. Add a constant "ssl.HAS_PSK" to check if TLS-PSK is supported --- Doc/library/ssl.rst | 12 ++++++++++++ Lib/ssl.py | 2 +- Lib/test/test_ssl.py | 2 ++ Modules/_ssl.c | 28 ++++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 206294528e0016..0db233e2dde33c 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -908,6 +908,12 @@ Constants .. versionadded:: 3.7 +.. data:: HAS_PSK + + Whether the OpenSSL library has built-in support for TLS-PSK. + + .. versionadded:: 3.13 + .. data:: CHANNEL_BINDING_TYPES List of supported TLS channel binding types. Strings in this list @@ -2050,6 +2056,9 @@ to speed up repeated connections from the same clients. return 'ClientId_1', psk_table.get(hint, b'') context.set_psk_client_callback(callback) + This method will raise :exc:`NotImplementedError` if :data:`HAS_PSK` is + ``False``. + .. versionadded:: 3.13 .. method:: SSLContext.set_psk_server_callback(callback, identity_hint=None) @@ -2092,6 +2101,9 @@ to speed up repeated connections from the same clients. return psk_table.get(identity, b'') context.set_psk_server_callback(callback, 'ServerId_1') + This method will raise :exc:`NotImplementedError` if :data:`HAS_PSK` is + ``False``. + .. versionadded:: 3.13 .. index:: single: certificates diff --git a/Lib/ssl.py b/Lib/ssl.py index 36fca9d4aa93d1..d01484964b6895 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -116,7 +116,7 @@ from _ssl import ( HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN, HAS_SSLv2, HAS_SSLv3, HAS_TLSv1, - HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3 + HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3, HAS_PSK ) from _ssl import _DEFAULT_CIPHERS, _OPENSSL_API_VERSION diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index aecba89cde1495..3fdfa2960503b8 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -4259,6 +4259,7 @@ def test_session_handling(self): 'Session refers to a different SSLContext.') @requires_tls_version('TLSv1_2') + @unittest.skipUnless(ssl.HAS_PSK, 'TLS-PSK disabled on this OpenSSL build') def test_psk(self): psk = bytes.fromhex('deadbeef') @@ -4326,6 +4327,7 @@ def server_callback(identity): s.connect((HOST, server.port)) @requires_tls_version('TLSv1_3') + @unittest.skipUnless(ssl.HAS_PSK, 'TLS-PSK disabled on this OpenSSL build') def test_psk_tls1_3(self): psk = bytes.fromhex('deadbeef') identity_hint = 'identity-hint' diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 707e7ad9543acb..90b600f4b776a3 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -301,8 +301,10 @@ typedef struct { BIO *keylog_bio; /* Cached module state, also used in SSLSocket and SSLSession code. */ _sslmodulestate *state; +#ifndef OPENSSL_NO_PSK PyObject *psk_client_callback; PyObject *psk_server_callback; +#endif } PySSLContext; typedef struct { @@ -3125,8 +3127,10 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) self->alpn_protocols = NULL; self->set_sni_cb = NULL; self->state = get_ssl_state(module); +#ifndef OPENSSL_NO_PSK self->psk_client_callback = NULL; self->psk_server_callback = NULL; +#endif /* Don't check host name by default */ if (proto_version == PY_SSL_VERSION_TLS_CLIENT) { @@ -3239,8 +3243,10 @@ context_clear(PySSLContext *self) Py_CLEAR(self->set_sni_cb); Py_CLEAR(self->msg_cb); Py_CLEAR(self->keylog_filename); +#ifndef OPENSSL_NO_PSK Py_CLEAR(self->psk_client_callback); Py_CLEAR(self->psk_server_callback); +#endif if (self->keylog_bio != NULL) { PySSL_BEGIN_ALLOW_THREADS BIO_free_all(self->keylog_bio); @@ -4668,6 +4674,7 @@ _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) return NULL; } +#ifndef OPENSSL_NO_PSK static unsigned int psk_client_callback(SSL *s, const char *hint, char *identity, @@ -4735,6 +4742,7 @@ static unsigned int psk_client_callback(SSL *s, PyGILState_Release(gstate); return 0; } +#endif /*[clinic input] _ssl._SSLContext.set_psk_client_callback @@ -4747,6 +4755,7 @@ _ssl__SSLContext_set_psk_client_callback_impl(PySSLContext *self, PyObject *callback) /*[clinic end generated code: output=0aba86f6ed75119e input=7627bae0e5ee7635]*/ { +#ifndef OPENSSL_NO_PSK if (self->protocol == PY_SSL_VERSION_TLS_SERVER) { _setSSLError(get_state_ctx(self), "Cannot add PSK client callback to a " @@ -4774,8 +4783,14 @@ _ssl__SSLContext_set_psk_client_callback_impl(PySSLContext *self, SSL_CTX_set_psk_client_callback(self->ctx, ssl_callback); Py_RETURN_NONE; +#else + PyErr_SetString(PyExc_NotImplementedError, + "TLS-PSK is not supported by your OpenSSL version."); + return NULL; +#endif } +#ifndef OPENSSL_NO_PSK static unsigned int psk_server_callback(SSL *s, const char *identity, unsigned char *psk, @@ -4835,6 +4850,7 @@ static unsigned int psk_server_callback(SSL *s, PyGILState_Release(gstate); return 0; } +#endif /*[clinic input] _ssl._SSLContext.set_psk_server_callback @@ -4849,6 +4865,7 @@ _ssl__SSLContext_set_psk_server_callback_impl(PySSLContext *self, const char *identity_hint) /*[clinic end generated code: output=1f4d6a4e09a92b03 input=65d4b6022aa85ea3]*/ { +#ifndef OPENSSL_NO_PSK if (self->protocol == PY_SSL_VERSION_TLS_CLIENT) { _setSSLError(get_state_ctx(self), "Cannot add PSK server callback to a " @@ -4882,6 +4899,11 @@ _ssl__SSLContext_set_psk_server_callback_impl(PySSLContext *self, SSL_CTX_set_psk_server_callback(self->ctx, ssl_callback); Py_RETURN_NONE; +#else + PyErr_SetString(PyExc_NotImplementedError, + "TLS-PSK is not supported by your OpenSSL version."); + return NULL; +#endif } @@ -6243,6 +6265,12 @@ sslmodule_init_constants(PyObject *m) addbool(m, "HAS_TLSv1_3", 0); #endif +#ifdef OPENSSL_NO_PSK + addbool(m, "HAS_PSK", 0); +#else + addbool(m, "HAS_PSK", 1); +#endif + #undef addbool #undef ADD_INT_CONST From e723700190ba497d1601cb423ee48d5d222a9d26 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 29 Nov 2023 10:10:11 +0900 Subject: [PATCH 37/37] Rename ...Uop... to ...UOp... (uppercase O) for consistency (#112327) * Rename _PyUopExecute to _PyUOpExecute (uppercase O) for consistency * Also rename _PyUopName and _PyUOp_Replacements, and some output strings --- Include/internal/pycore_uops.h | 2 +- Python/bytecodes.c | 2 +- Python/ceval.c | 12 ++++++------ Python/generated_cases.c.h | 2 +- Python/optimizer.c | 28 ++++++++++++++-------------- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Include/internal/pycore_uops.h b/Include/internal/pycore_uops.h index e2b94894681f44..ea8f90bf8c1d8f 100644 --- a/Include/internal/pycore_uops.h +++ b/Include/internal/pycore_uops.h @@ -24,7 +24,7 @@ typedef struct { _PyUOpInstruction trace[1]; } _PyUOpExecutorObject; -_PyInterpreterFrame *_PyUopExecute( +_PyInterpreterFrame *_PyUOpExecute( _PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a1ca66e457e47d..2e5f6c8d0d10e2 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2357,7 +2357,7 @@ dummy_func( JUMPBY(1-original_oparg); frame->instr_ptr = next_instr; Py_INCREF(executor); - if (executor->execute == _PyUopExecute) { + if (executor->execute == _PyUOpExecute) { current_executor = (_PyUOpExecutorObject *)executor; GOTO_TIER_TWO(); } diff --git a/Python/ceval.c b/Python/ceval.c index def75fd114cb86..1806ceb7fa9681 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -647,7 +647,7 @@ static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { extern const struct _PyCode_DEF(8) _Py_InitCleanup; -extern const char *_PyUopName(int index); +extern const char *_PyUOpName(int index); /* Disable unused label warnings. They are handy for debugging, even if computed gotos aren't used. */ @@ -1002,7 +1002,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DPRINTF(3, "%4d: uop %s, oparg %d, operand %" PRIu64 ", target %d, stack_level %d\n", (int)(next_uop - current_executor->trace), - _PyUopName(uopcode), + _PyUOpName(uopcode), next_uop->oparg, next_uop->operand, next_uop->target, @@ -1051,8 +1051,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int pop_1_error_tier_two: STACK_SHRINK(1); error_tier_two: - DPRINTF(2, "Error: [Uop %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d]\n", - uopcode, _PyUopName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, + DPRINTF(2, "Error: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d]\n", + uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, (int)(next_uop - current_executor->trace - 1)); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); frame->return_offset = 0; // Don't leave this random @@ -1064,8 +1064,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int deoptimize: // On DEOPT_IF we just repeat the last instruction. // This presumes nothing was popped from the stack (nor pushed). - DPRINTF(2, "DEOPT: [Uop %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d]\n", - uopcode, _PyUopName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, + DPRINTF(2, "DEOPT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d]\n", + uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, (int)(next_uop - current_executor->trace - 1)); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); UOP_STAT_INC(uopcode, miss); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index dedd793111b7ff..0ac99e759deb12 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2331,7 +2331,7 @@ JUMPBY(1-original_oparg); frame->instr_ptr = next_instr; Py_INCREF(executor); - if (executor->execute == _PyUopExecute) { + if (executor->execute == _PyUOpExecute) { current_executor = (_PyUOpExecutorObject *)executor; GOTO_TIER_TWO(); } diff --git a/Python/optimizer.c b/Python/optimizer.c index ec59fea26fbc70..dd24fbebbfd2a9 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -325,7 +325,7 @@ uop_dealloc(_PyUOpExecutorObject *self) { } const char * -_PyUopName(int index) +_PyUOpName(int index) { if (index <= MAX_REAL_OPCODE) { return _PyOpcode_OpName[index]; @@ -347,7 +347,7 @@ uop_item(_PyUOpExecutorObject *self, Py_ssize_t index) PyErr_SetNone(PyExc_IndexError); return NULL; } - const char *name = _PyUopName(self->trace[index].opcode); + const char *name = _PyUOpName(self->trace[index].opcode); if (name == NULL) { name = ""; } @@ -388,7 +388,7 @@ PyTypeObject _PyUOpExecutor_Type = { /* TO DO -- Generate these tables */ static const uint16_t -_PyUop_Replacements[OPCODE_METADATA_SIZE] = { +_PyUOp_Replacements[OPCODE_METADATA_SIZE] = { [_ITER_JUMP_RANGE] = _GUARD_NOT_EXHAUSTED_RANGE, [_ITER_JUMP_LIST] = _GUARD_NOT_EXHAUSTED_LIST, [_ITER_JUMP_TUPLE] = _GUARD_NOT_EXHAUSTED_TUPLE, @@ -451,7 +451,7 @@ translate_bytecode_to_trace( #define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ DPRINTF(2, \ " ADD_TO_TRACE(%s, %d, %" PRIu64 ")\n", \ - _PyUopName(OPCODE), \ + _PyUOpName(OPCODE), \ (OPARG), \ (uint64_t)(OPERAND)); \ assert(trace_length < max_length); \ @@ -474,7 +474,7 @@ translate_bytecode_to_trace( } // Reserve space for N uops, plus 3 for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE -#define RESERVE(needed) RESERVE_RAW((needed) + 3, _PyUopName(opcode)) +#define RESERVE(needed) RESERVE_RAW((needed) + 3, _PyUOpName(opcode)) // Trace stack operations (used by _PUSH_FRAME, _POP_FRAME) #define TRACE_STACK_PUSH() \ @@ -546,8 +546,8 @@ translate_bytecode_to_trace( uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; DPRINTF(4, "%s(%d): counter=%x, bitcount=%d, likely=%d, uopcode=%s\n", - _PyUopName(opcode), oparg, - counter, bitcount, jump_likely, _PyUopName(uopcode)); + _PyUOpName(opcode), oparg, + counter, bitcount, jump_likely, _PyUOpName(uopcode)); ADD_TO_TRACE(uopcode, max_length, 0, target); if (jump_likely) { _Py_CODEUNIT *target_instr = next_instr + oparg; @@ -615,8 +615,8 @@ translate_bytecode_to_trace( oparg += extras; } } - if (_PyUop_Replacements[uop]) { - uop = _PyUop_Replacements[uop]; + if (_PyUOp_Replacements[uop]) { + uop = _PyUOp_Replacements[uop]; if (uop == _FOR_ITER_TIER_TWO) { target += 1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; assert(_PyCode_CODE(code)[target-1].op.code == END_FOR || @@ -712,7 +712,7 @@ translate_bytecode_to_trace( } break; } - DPRINTF(2, "Unsupported opcode %s\n", _PyUopName(opcode)); + DPRINTF(2, "Unsupported opcode %s\n", _PyUOpName(opcode)); OPT_UNSUPPORTED_OPCODE(opcode); goto done; // Break out of loop } // End default @@ -832,7 +832,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) dest--; } assert(dest == -1); - executor->base.execute = _PyUopExecute; + executor->base.execute = _PyUOpExecute; _Py_ExecutorInit((_PyExecutorObject *)executor, dependencies); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -845,7 +845,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) for (int i = 0; i < length; i++) { printf("%4d %s(%d, %d, %" PRIu64 ")\n", i, - _PyUopName(executor->trace[i].opcode), + _PyUOpName(executor->trace[i].opcode), executor->trace[i].oparg, executor->trace[i].target, executor->trace[i].operand); @@ -888,11 +888,11 @@ uop_optimize( return 1; } -/* Dummy execute() function for Uop Executor. +/* Dummy execute() function for UOp Executor. * The actual implementation is inlined in ceval.c, * in _PyEval_EvalFrameDefault(). */ _PyInterpreterFrame * -_PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer) +_PyUOpExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer) { Py_FatalError("Tier 2 is now inlined into Tier 1"); }