From 0c6fe81dce9d6bb1dce5e4503f1b42bc5355ba24 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 13 Feb 2023 20:33:48 +0000 Subject: [PATCH 01/76] gh-101849: Add upgrade codes for old versions of launcher that ended up with later version numbers (GH-101877) --- ...-02-13-16-32-50.gh-issue-101849.7lm_53.rst | 1 + Tools/msi/common.wxs | 2 +- Tools/msi/launcher/launcher.wxs | 21 +++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst diff --git a/Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst b/Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst new file mode 100644 index 00000000000000..861d4de9f9a650 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst @@ -0,0 +1 @@ +Ensures installer will correctly upgrade existing ``py.exe`` launcher installs. diff --git a/Tools/msi/common.wxs b/Tools/msi/common.wxs index 55cb44860d02c0..54fa749ab17cdd 100644 --- a/Tools/msi/common.wxs +++ b/Tools/msi/common.wxs @@ -25,7 +25,6 @@ - @@ -42,6 +41,7 @@ UPGRADE + diff --git a/Tools/msi/launcher/launcher.wxs b/Tools/msi/launcher/launcher.wxs index b83058c63bf6d9..49f1f7b8c1762e 100644 --- a/Tools/msi/launcher/launcher.wxs +++ b/Tools/msi/launcher/launcher.wxs @@ -34,13 +34,34 @@ NOT Installed AND NOT ALLUSERS=1 NOT Installed AND ALLUSERS=1 + + UPGRADE or REMOVE_350_LAUNCHER or REMOVE_360A1_LAUNCHER or UPGRADE_3_11_0 or UPGRADE_3_11_1 + UPGRADE or REMOVE_350_LAUNCHER or REMOVE_360A1_LAUNCHER + + + Installed OR NOT DOWNGRADE OR UPGRADE_3_11_0 OR UPGRADE_3_11_1 + + Installed OR NOT DOWNGRADE + + + + + + + From 928752ce4c23f47d3175dd47ecacf08d86a99c9d Mon Sep 17 00:00:00 2001 From: Radek Smejkal Date: Tue, 14 Feb 2023 02:37:34 +0100 Subject: [PATCH 02/76] gh-74895: getaddrinfo no longer raises OverflowError (#2435) `socket.getaddrinfo()` no longer raises `OverflowError` based on the **port** argument. Error reporting (or not) for its value is left up to the underlying C library `getaddrinfo()` implementation. --- Lib/test/test_socket.py | 48 +++++++++++++++++++ Misc/ACKS | 1 + ...3-02-13-22-21-58.gh-issue-74895.esMNtq.rst | 5 ++ Modules/getaddrinfo.c | 6 ++- Modules/socketmodule.c | 14 ++++-- 5 files changed, 68 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index f1b4018c265e18..a313da29b4a4fd 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1600,6 +1600,54 @@ def testGetaddrinfo(self): except socket.gaierror: pass + def test_getaddrinfo_int_port_overflow(self): + # gh-74895: Test that getaddrinfo does not raise OverflowError on port. + # + # POSIX getaddrinfo() never specify the valid range for "service" + # decimal port number values. For IPv4 and IPv6 they are technically + # unsigned 16-bit values, but the API is protocol agnostic. Which values + # trigger an error from the C library function varies by platform as + # they do not all perform validation. + + # The key here is that we don't want to produce OverflowError as Python + # prior to 3.12 did for ints outside of a [LONG_MIN, LONG_MAX] range. + # Leave the error up to the underlying string based platform C API. + + from _testcapi import ULONG_MAX, LONG_MAX, LONG_MIN + try: + socket.getaddrinfo(None, ULONG_MAX + 1) + except OverflowError: + # Platforms differ as to what values consitute a getaddrinfo() error + # return. Some fail for LONG_MAX+1, others ULONG_MAX+1, and Windows + # silently accepts such huge "port" aka "service" numeric values. + self.fail("Either no error or socket.gaierror expected.") + except socket.gaierror: + pass + + try: + socket.getaddrinfo(None, LONG_MAX + 1) + except OverflowError: + self.fail("Either no error or socket.gaierror expected.") + except socket.gaierror: + pass + + try: + socket.getaddrinfo(None, LONG_MAX - 0xffff + 1) + except OverflowError: + self.fail("Either no error or socket.gaierror expected.") + except socket.gaierror: + pass + + try: + socket.getaddrinfo(None, LONG_MIN - 1) + except OverflowError: + self.fail("Either no error or socket.gaierror expected.") + except socket.gaierror: + pass + + socket.getaddrinfo(None, 0) # No error expected. + socket.getaddrinfo(None, 0xffff) # No error expected. + def test_getnameinfo(self): # only IP addresses are allowed self.assertRaises(OSError, socket.getnameinfo, ('mail.python.org',0), 0) diff --git a/Misc/ACKS b/Misc/ACKS index e12cbea0ebd6ed..ca92608868f23f 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1688,6 +1688,7 @@ Roman Skurikhin Ville Skyttä Michael Sloan Nick Sloan +Radek Smejkal Václav Šmilauer Casper W. Smet Allen W. Smith diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst new file mode 100644 index 00000000000000..adbbb601634a60 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst @@ -0,0 +1,5 @@ +:mod:`socket.getaddrinfo` no longer raises :class:`OverflowError` for +:class:`int` **port** values outside of the C long range. Out of range values +are left up to the underlying string based C library API to report. A +:class:`socket.gaierror` ``SAI_SERVICE`` may occur instead, or no error at all +as not all platform C libraries generate an error. diff --git a/Modules/getaddrinfo.c b/Modules/getaddrinfo.c index 0b4620ed683de9..f1c28d7d9312ac 100644 --- a/Modules/getaddrinfo.c +++ b/Modules/getaddrinfo.c @@ -342,7 +342,11 @@ getaddrinfo(const char*hostname, const char*servname, pai->ai_socktype = SOCK_DGRAM; pai->ai_protocol = IPPROTO_UDP; } - port = htons((u_short)atoi(servname)); + long maybe_port = strtol(servname, NULL, 10); + if (maybe_port < 0 || maybe_port > 0xffff) { + ERR(EAI_SERVICE); + } + port = htons((u_short)maybe_port); } else { struct servent *sp; const char *proto; diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 0a9e46512b157b..2d300f19436b1a 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -6650,7 +6650,7 @@ socket_getaddrinfo(PyObject *self, PyObject *args, PyObject* kwargs) struct addrinfo *res0 = NULL; PyObject *hobj = NULL; PyObject *pobj = (PyObject *)NULL; - char pbuf[30]; + PyObject *pstr = NULL; const char *hptr, *pptr; int family, socktype, protocol, flags; int error; @@ -6680,11 +6680,13 @@ socket_getaddrinfo(PyObject *self, PyObject *args, PyObject* kwargs) return NULL; } if (PyLong_CheckExact(pobj)) { - long value = PyLong_AsLong(pobj); - if (value == -1 && PyErr_Occurred()) + pstr = PyObject_Str(pobj); + if (pstr == NULL) + goto err; + assert(PyUnicode_Check(pstr)); + pptr = PyUnicode_AsUTF8(pstr); + if (pptr == NULL) goto err; - PyOS_snprintf(pbuf, sizeof(pbuf), "%ld", value); - pptr = pbuf; } else if (PyUnicode_Check(pobj)) { pptr = PyUnicode_AsUTF8(pobj); if (pptr == NULL) @@ -6750,12 +6752,14 @@ socket_getaddrinfo(PyObject *self, PyObject *args, PyObject* kwargs) Py_DECREF(single); } Py_XDECREF(idna); + Py_XDECREF(pstr); if (res0) freeaddrinfo(res0); return all; err: Py_XDECREF(all); Py_XDECREF(idna); + Py_XDECREF(pstr); if (res0) freeaddrinfo(res0); return (PyObject *)NULL; From 8be8101bca34b60481ec3d7ecaea4a3379fb7dbb Mon Sep 17 00:00:00 2001 From: Sam James Date: Tue, 14 Feb 2023 07:21:58 +0000 Subject: [PATCH 03/76] gh-101857: Allow xattr detection on musl libc (#101858) Previously, we checked exclusively for `__GLIBC__` (AND'd with some other conditions). Checking for `__linux__` instead should be fine. This fixes using e.g. `os.listxattr()` on systems using musl libc. Bug: https://bugs.gentoo.org/894130 Co-authored-by: Gregory P. Smith --- .../2023-02-12-22-40-22.gh-issue-101857._bribG.rst | 1 + Modules/posixmodule.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst new file mode 100644 index 00000000000000..832cc300fa9433 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst @@ -0,0 +1 @@ +Fix xattr support detection on Linux systems by widening the check to linux, not just glibc. This fixes support for musl. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index cba6cea48b77e1..d9e93473aeadaa 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -274,8 +274,9 @@ corresponding Unix manual entries for more information on calls."); # undef HAVE_SCHED_SETAFFINITY #endif -#if defined(HAVE_SYS_XATTR_H) && defined(__GLIBC__) && !defined(__FreeBSD_kernel__) && !defined(__GNU__) +#if defined(HAVE_SYS_XATTR_H) && defined(__linux__) && !defined(__FreeBSD_kernel__) && !defined(__GNU__) # define USE_XATTRS +# include // Needed for XATTR_SIZE_MAX on musl libc. #endif #ifdef USE_XATTRS From e5da9ab2c82c6b4e4f8ffa699a9a609ea1bea255 Mon Sep 17 00:00:00 2001 From: Jonathan Protzenko Date: Tue, 14 Feb 2023 01:25:16 -0800 Subject: [PATCH 04/76] gh-99108: Import SHA2-384/512 from HACL* (#101707) Replace the builtin hashlib implementations of SHA2-384 and SHA2-512 originally from LibTomCrypt with formally verified, side-channel resistant code from the [HACL*](https://github.com/hacl-star/hacl-star/) project. The builtins remain a fallback only used when OpenSSL does not provide them. --- Makefile.pre.in | 2 +- ...3-02-08-12-57-35.gh-issue-99108.6tnmhA.rst | 4 + Modules/Setup.stdlib.in | 2 +- Modules/_hacl/Hacl_Streaming_SHA2.c | 654 ++++++++++++++++++ Modules/_hacl/Hacl_Streaming_SHA2.h | 85 ++- .../include/krml/FStar_UInt128_Verified.h | 347 ++++++++++ .../include/krml/FStar_UInt_8_16_32_64.h | 2 +- .../krml/fstar_uint128_struct_endianness.h | 68 ++ Modules/_hacl/include/krml/types.h | 14 + .../_hacl/include/python_hacl_namespaces.h | 17 + Modules/_hacl/internal/Hacl_SHA2_Generic.h | 5 +- Modules/_hacl/refresh.sh | 45 +- Modules/sha256module.c | 1 + Modules/sha512module.c | 441 ++---------- configure | 2 +- configure.ac | 4 +- 16 files changed, 1259 insertions(+), 434 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst create mode 100644 Modules/_hacl/include/krml/FStar_UInt128_Verified.h create mode 100644 Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h create mode 100644 Modules/_hacl/include/krml/types.h diff --git a/Makefile.pre.in b/Makefile.pre.in index 7a84b953d97962..d42d4d8a3c1c9f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2608,7 +2608,7 @@ MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h MODULE__SHA256_DEPS=$(srcdir)/Modules/hashlib.h $(srcdir)/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h $(srcdir)/Modules/_hacl/include/krml/lowstar_endianness.h $(srcdir)/Modules/_hacl/include/krml/internal/target.h $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.h MODULE__SHA3_DEPS=$(srcdir)/Modules/_sha3/sha3.c $(srcdir)/Modules/_sha3/sha3.h $(srcdir)/Modules/hashlib.h -MODULE__SHA512_DEPS=$(srcdir)/Modules/hashlib.h +MODULE__SHA512_DEPS=$(srcdir)/Modules/hashlib.h $(srcdir)/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h $(srcdir)/Modules/_hacl/include/krml/lowstar_endianness.h $(srcdir)/Modules/_hacl/include/krml/internal/target.h $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.h MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data.h $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h diff --git a/Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst b/Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst new file mode 100644 index 00000000000000..6a7a309dad5d8f --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst @@ -0,0 +1,4 @@ +Replace the builtin :mod:`hashlib` implementations of SHA2-384 and SHA2-512 +originally from LibTomCrypt with formally verified, side-channel resistant +code from the `HACL* `_ project. +The builtins remain a fallback only used when OpenSSL does not provide them. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index f72783810f9415..b6d13e04d3fa87 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -80,7 +80,7 @@ @MODULE__MD5_TRUE@_md5 md5module.c @MODULE__SHA1_TRUE@_sha1 sha1module.c @MODULE__SHA256_TRUE@_sha256 sha256module.c _hacl/Hacl_Streaming_SHA2.c -@MODULE__SHA512_TRUE@_sha512 sha512module.c +@MODULE__SHA512_TRUE@_sha512 sha512module.c _hacl/Hacl_Streaming_SHA2.c @MODULE__SHA3_TRUE@_sha3 _sha3/sha3module.c @MODULE__BLAKE2_TRUE@_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c diff --git a/Modules/_hacl/Hacl_Streaming_SHA2.c b/Modules/_hacl/Hacl_Streaming_SHA2.c index 84566571792a3c..8169c7a356731e 100644 --- a/Modules/_hacl/Hacl_Streaming_SHA2.c +++ b/Modules/_hacl/Hacl_Streaming_SHA2.c @@ -250,6 +250,229 @@ static inline void sha224_finish(uint32_t *st, uint8_t *h) memcpy(h, hbuf, (uint32_t)28U * sizeof (uint8_t)); } +void Hacl_SHA2_Scalar32_sha512_init(uint64_t *hash) +{ + KRML_MAYBE_FOR8(i, + (uint32_t)0U, + (uint32_t)8U, + (uint32_t)1U, + uint64_t *os = hash; + uint64_t x = Hacl_Impl_SHA2_Generic_h512[i]; + os[i] = x;); +} + +static inline void sha512_update(uint8_t *b, uint64_t *hash) +{ + uint64_t hash_old[8U] = { 0U }; + uint64_t ws[16U] = { 0U }; + memcpy(hash_old, hash, (uint32_t)8U * sizeof (uint64_t)); + uint8_t *b10 = b; + uint64_t u = load64_be(b10); + ws[0U] = u; + uint64_t u0 = load64_be(b10 + (uint32_t)8U); + ws[1U] = u0; + uint64_t u1 = load64_be(b10 + (uint32_t)16U); + ws[2U] = u1; + uint64_t u2 = load64_be(b10 + (uint32_t)24U); + ws[3U] = u2; + uint64_t u3 = load64_be(b10 + (uint32_t)32U); + ws[4U] = u3; + uint64_t u4 = load64_be(b10 + (uint32_t)40U); + ws[5U] = u4; + uint64_t u5 = load64_be(b10 + (uint32_t)48U); + ws[6U] = u5; + uint64_t u6 = load64_be(b10 + (uint32_t)56U); + ws[7U] = u6; + uint64_t u7 = load64_be(b10 + (uint32_t)64U); + ws[8U] = u7; + uint64_t u8 = load64_be(b10 + (uint32_t)72U); + ws[9U] = u8; + uint64_t u9 = load64_be(b10 + (uint32_t)80U); + ws[10U] = u9; + uint64_t u10 = load64_be(b10 + (uint32_t)88U); + ws[11U] = u10; + uint64_t u11 = load64_be(b10 + (uint32_t)96U); + ws[12U] = u11; + uint64_t u12 = load64_be(b10 + (uint32_t)104U); + ws[13U] = u12; + uint64_t u13 = load64_be(b10 + (uint32_t)112U); + ws[14U] = u13; + uint64_t u14 = load64_be(b10 + (uint32_t)120U); + ws[15U] = u14; + KRML_MAYBE_FOR5(i0, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + KRML_MAYBE_FOR16(i, + (uint32_t)0U, + (uint32_t)16U, + (uint32_t)1U, + uint64_t k_t = Hacl_Impl_SHA2_Generic_k384_512[(uint32_t)16U * i0 + i]; + uint64_t ws_t = ws[i]; + uint64_t a0 = hash[0U]; + uint64_t b0 = hash[1U]; + uint64_t c0 = hash[2U]; + uint64_t d0 = hash[3U]; + uint64_t e0 = hash[4U]; + uint64_t f0 = hash[5U]; + uint64_t g0 = hash[6U]; + uint64_t h02 = hash[7U]; + uint64_t k_e_t = k_t; + uint64_t + t1 = + h02 + + + ((e0 << (uint32_t)50U | e0 >> (uint32_t)14U) + ^ + ((e0 << (uint32_t)46U | e0 >> (uint32_t)18U) + ^ (e0 << (uint32_t)23U | e0 >> (uint32_t)41U))) + + ((e0 & f0) ^ (~e0 & g0)) + + k_e_t + + ws_t; + uint64_t + t2 = + ((a0 << (uint32_t)36U | a0 >> (uint32_t)28U) + ^ + ((a0 << (uint32_t)30U | a0 >> (uint32_t)34U) + ^ (a0 << (uint32_t)25U | a0 >> (uint32_t)39U))) + + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); + uint64_t a1 = t1 + t2; + uint64_t b1 = a0; + uint64_t c1 = b0; + uint64_t d1 = c0; + uint64_t e1 = d0 + t1; + uint64_t f1 = e0; + uint64_t g1 = f0; + uint64_t h12 = g0; + hash[0U] = a1; + hash[1U] = b1; + hash[2U] = c1; + hash[3U] = d1; + hash[4U] = e1; + hash[5U] = f1; + hash[6U] = g1; + hash[7U] = h12;); + if (i0 < (uint32_t)4U) + { + KRML_MAYBE_FOR16(i, + (uint32_t)0U, + (uint32_t)16U, + (uint32_t)1U, + uint64_t t16 = ws[i]; + uint64_t t15 = ws[(i + (uint32_t)1U) % (uint32_t)16U]; + uint64_t t7 = ws[(i + (uint32_t)9U) % (uint32_t)16U]; + uint64_t t2 = ws[(i + (uint32_t)14U) % (uint32_t)16U]; + uint64_t + s1 = + (t2 << (uint32_t)45U | t2 >> (uint32_t)19U) + ^ ((t2 << (uint32_t)3U | t2 >> (uint32_t)61U) ^ t2 >> (uint32_t)6U); + uint64_t + s0 = + (t15 << (uint32_t)63U | t15 >> (uint32_t)1U) + ^ ((t15 << (uint32_t)56U | t15 >> (uint32_t)8U) ^ t15 >> (uint32_t)7U); + ws[i] = s1 + t7 + s0 + t16;); + }); + KRML_MAYBE_FOR8(i, + (uint32_t)0U, + (uint32_t)8U, + (uint32_t)1U, + uint64_t *os = hash; + uint64_t x = hash[i] + hash_old[i]; + os[i] = x;); +} + +static inline void sha512_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st) +{ + uint32_t blocks = len / (uint32_t)128U; + for (uint32_t i = (uint32_t)0U; i < blocks; i++) + { + uint8_t *b0 = b; + uint8_t *mb = b0 + i * (uint32_t)128U; + sha512_update(mb, st); + } +} + +static inline void +sha512_update_last(FStar_UInt128_uint128 totlen, uint32_t len, uint8_t *b, uint64_t *hash) +{ + uint32_t blocks; + if (len + (uint32_t)16U + (uint32_t)1U <= (uint32_t)128U) + { + blocks = (uint32_t)1U; + } + else + { + blocks = (uint32_t)2U; + } + uint32_t fin = blocks * (uint32_t)128U; + uint8_t last[256U] = { 0U }; + uint8_t totlen_buf[16U] = { 0U }; + FStar_UInt128_uint128 total_len_bits = FStar_UInt128_shift_left(totlen, (uint32_t)3U); + store128_be(totlen_buf, total_len_bits); + uint8_t *b0 = b; + memcpy(last, b0, len * sizeof (uint8_t)); + last[len] = (uint8_t)0x80U; + memcpy(last + fin - (uint32_t)16U, totlen_buf, (uint32_t)16U * sizeof (uint8_t)); + uint8_t *last00 = last; + uint8_t *last10 = last + (uint32_t)128U; + uint8_t *l0 = last00; + uint8_t *l1 = last10; + uint8_t *lb0 = l0; + uint8_t *lb1 = l1; + uint8_t *last0 = lb0; + uint8_t *last1 = lb1; + sha512_update(last0, hash); + if (blocks > (uint32_t)1U) + { + sha512_update(last1, hash); + return; + } +} + +static inline void sha512_finish(uint64_t *st, uint8_t *h) +{ + uint8_t hbuf[64U] = { 0U }; + KRML_MAYBE_FOR8(i, + (uint32_t)0U, + (uint32_t)8U, + (uint32_t)1U, + store64_be(hbuf + i * (uint32_t)8U, st[i]);); + memcpy(h, hbuf, (uint32_t)64U * sizeof (uint8_t)); +} + +static inline void sha384_init(uint64_t *hash) +{ + KRML_MAYBE_FOR8(i, + (uint32_t)0U, + (uint32_t)8U, + (uint32_t)1U, + uint64_t *os = hash; + uint64_t x = Hacl_Impl_SHA2_Generic_h384[i]; + os[i] = x;); +} + +static inline void sha384_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st) +{ + sha512_update_nblocks(len, b, st); +} + +static void +sha384_update_last(FStar_UInt128_uint128 totlen, uint32_t len, uint8_t *b, uint64_t *st) +{ + sha512_update_last(totlen, len, b, st); +} + +static inline void sha384_finish(uint64_t *st, uint8_t *h) +{ + uint8_t hbuf[64U] = { 0U }; + KRML_MAYBE_FOR8(i, + (uint32_t)0U, + (uint32_t)8U, + (uint32_t)1U, + store64_be(hbuf + i * (uint32_t)8U, st[i]);); + memcpy(h, hbuf, (uint32_t)48U * sizeof (uint8_t)); +} + /** Allocate initial state for the SHA2_256 hash. The state is to be freed by calling `free_256`. @@ -680,3 +903,434 @@ void Hacl_Streaming_SHA2_sha224(uint8_t *input, uint32_t input_len, uint8_t *dst sha224_finish(st, rb); } +Hacl_Streaming_SHA2_state_sha2_384 *Hacl_Streaming_SHA2_create_in_512(void) +{ + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)128U, sizeof (uint8_t)); + uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); + Hacl_Streaming_SHA2_state_sha2_384 + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + Hacl_Streaming_SHA2_state_sha2_384 + *p = + (Hacl_Streaming_SHA2_state_sha2_384 *)KRML_HOST_MALLOC(sizeof ( + Hacl_Streaming_SHA2_state_sha2_384 + )); + p[0U] = s; + Hacl_SHA2_Scalar32_sha512_init(block_state); + return p; +} + +/** +Copies the state passed as argument into a newly allocated state (deep copy). +The state is to be freed by calling `free_512`. Cloning the state this way is +useful, for instance, if your control-flow diverges and you need to feed +more (different) data into the hash in each branch. +*/ +Hacl_Streaming_SHA2_state_sha2_384 +*Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_SHA2_state_sha2_384 *s0) +{ + Hacl_Streaming_SHA2_state_sha2_384 scrut = *s0; + uint64_t *block_state0 = scrut.block_state; + uint8_t *buf0 = scrut.buf; + uint64_t total_len0 = scrut.total_len; + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)128U, sizeof (uint8_t)); + memcpy(buf, buf0, (uint32_t)128U * sizeof (uint8_t)); + uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); + memcpy(block_state, block_state0, (uint32_t)8U * sizeof (uint64_t)); + Hacl_Streaming_SHA2_state_sha2_384 + s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; + Hacl_Streaming_SHA2_state_sha2_384 + *p = + (Hacl_Streaming_SHA2_state_sha2_384 *)KRML_HOST_MALLOC(sizeof ( + Hacl_Streaming_SHA2_state_sha2_384 + )); + p[0U] = s; + return p; +} + +void Hacl_Streaming_SHA2_init_512(Hacl_Streaming_SHA2_state_sha2_384 *s) +{ + Hacl_Streaming_SHA2_state_sha2_384 scrut = *s; + uint8_t *buf = scrut.buf; + uint64_t *block_state = scrut.block_state; + Hacl_SHA2_Scalar32_sha512_init(block_state); + Hacl_Streaming_SHA2_state_sha2_384 + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s[0U] = tmp; +} + +static inline uint32_t +update_384_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *data, uint32_t len) +{ + Hacl_Streaming_SHA2_state_sha2_384 s = *p; + uint64_t total_len = s.total_len; + if ((uint64_t)len > (uint64_t)18446744073709551615U - total_len) + { + return (uint32_t)1U; + } + uint32_t sz; + if (total_len % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len > (uint64_t)0U) + { + sz = (uint32_t)128U; + } + else + { + sz = (uint32_t)(total_len % (uint64_t)(uint32_t)128U); + } + if (len <= (uint32_t)128U - sz) + { + Hacl_Streaming_SHA2_state_sha2_384 s1 = *p; + uint64_t *block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = (uint32_t)128U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)128U); + } + uint8_t *buf2 = buf + sz1; + memcpy(buf2, data, len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)len; + *p + = + ( + (Hacl_Streaming_SHA2_state_sha2_384){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len2 + } + ); + } + else if (sz == (uint32_t)0U) + { + Hacl_Streaming_SHA2_state_sha2_384 s1 = *p; + uint64_t *block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = (uint32_t)128U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)128U); + } + if (!(sz1 == (uint32_t)0U)) + { + sha512_update_nblocks((uint32_t)128U, buf, block_state1); + } + uint32_t ite; + if ((uint64_t)len % (uint64_t)(uint32_t)128U == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + { + ite = (uint32_t)128U; + } + else + { + ite = (uint32_t)((uint64_t)len % (uint64_t)(uint32_t)128U); + } + uint32_t n_blocks = (len - ite) / (uint32_t)128U; + uint32_t data1_len = n_blocks * (uint32_t)128U; + uint32_t data2_len = len - data1_len; + uint8_t *data1 = data; + uint8_t *data2 = data + data1_len; + sha512_update_nblocks(data1_len, data1, block_state1); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_SHA2_state_sha2_384){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)len + } + ); + } + else + { + uint32_t diff = (uint32_t)128U - sz; + uint8_t *data1 = data; + uint8_t *data2 = data + diff; + Hacl_Streaming_SHA2_state_sha2_384 s1 = *p; + uint64_t *block_state10 = s1.block_state; + uint8_t *buf0 = s1.buf; + uint64_t total_len10 = s1.total_len; + uint32_t sz10; + if (total_len10 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len10 > (uint64_t)0U) + { + sz10 = (uint32_t)128U; + } + else + { + sz10 = (uint32_t)(total_len10 % (uint64_t)(uint32_t)128U); + } + uint8_t *buf2 = buf0 + sz10; + memcpy(buf2, data1, diff * sizeof (uint8_t)); + uint64_t total_len2 = total_len10 + (uint64_t)diff; + *p + = + ( + (Hacl_Streaming_SHA2_state_sha2_384){ + .block_state = block_state10, + .buf = buf0, + .total_len = total_len2 + } + ); + Hacl_Streaming_SHA2_state_sha2_384 s10 = *p; + uint64_t *block_state1 = s10.block_state; + uint8_t *buf = s10.buf; + uint64_t total_len1 = s10.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = (uint32_t)128U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)128U); + } + if (!(sz1 == (uint32_t)0U)) + { + sha512_update_nblocks((uint32_t)128U, buf, block_state1); + } + uint32_t ite; + if + ( + (uint64_t)(len - diff) + % (uint64_t)(uint32_t)128U + == (uint64_t)0U + && (uint64_t)(len - diff) > (uint64_t)0U + ) + { + ite = (uint32_t)128U; + } + else + { + ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)(uint32_t)128U); + } + uint32_t n_blocks = (len - diff - ite) / (uint32_t)128U; + uint32_t data1_len = n_blocks * (uint32_t)128U; + uint32_t data2_len = len - diff - data1_len; + uint8_t *data11 = data2; + uint8_t *data21 = data2 + data1_len; + sha512_update_nblocks(data1_len, data11, block_state1); + uint8_t *dst = buf; + memcpy(dst, data21, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_SHA2_state_sha2_384){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)(len - diff) + } + ); + } + return (uint32_t)0U; +} + +/** +Feed an arbitrary amount of data into the hash. This function returns 0 for +success, or 1 if the combined length of all of the data passed to `update_512` +(since the last call to `init_512`) exceeds 2^125-1 bytes. + +This function is identical to the update function for SHA2_384. +*/ +uint32_t +Hacl_Streaming_SHA2_update_512( + Hacl_Streaming_SHA2_state_sha2_384 *p, + uint8_t *input, + uint32_t input_len +) +{ + return update_384_512(p, input, input_len); +} + +/** +Write the resulting hash into `dst`, an array of 64 bytes. The state remains +valid after a call to `finish_512`, meaning the user may feed more data into +the hash via `update_512`. (The finish_512 function operates on an internal copy of +the state and therefore does not invalidate the client-held state `p`.) +*/ +void Hacl_Streaming_SHA2_finish_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *dst) +{ + Hacl_Streaming_SHA2_state_sha2_384 scrut = *p; + uint64_t *block_state = scrut.block_state; + uint8_t *buf_ = scrut.buf; + uint64_t total_len = scrut.total_len; + uint32_t r; + if (total_len % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len > (uint64_t)0U) + { + r = (uint32_t)128U; + } + else + { + r = (uint32_t)(total_len % (uint64_t)(uint32_t)128U); + } + uint8_t *buf_1 = buf_; + uint64_t tmp_block_state[8U] = { 0U }; + memcpy(tmp_block_state, block_state, (uint32_t)8U * sizeof (uint64_t)); + uint32_t ite; + if (r % (uint32_t)128U == (uint32_t)0U && r > (uint32_t)0U) + { + ite = (uint32_t)128U; + } + else + { + ite = r % (uint32_t)128U; + } + uint8_t *buf_last = buf_1 + r - ite; + uint8_t *buf_multi = buf_1; + sha512_update_nblocks((uint32_t)0U, buf_multi, tmp_block_state); + uint64_t prev_len_last = total_len - (uint64_t)r; + sha512_update_last(FStar_UInt128_add(FStar_UInt128_uint64_to_uint128(prev_len_last), + FStar_UInt128_uint64_to_uint128((uint64_t)r)), + r, + buf_last, + tmp_block_state); + sha512_finish(tmp_block_state, dst); +} + +/** +Free a state allocated with `create_in_512`. + +This function is identical to the free function for SHA2_384. +*/ +void Hacl_Streaming_SHA2_free_512(Hacl_Streaming_SHA2_state_sha2_384 *s) +{ + Hacl_Streaming_SHA2_state_sha2_384 scrut = *s; + uint8_t *buf = scrut.buf; + uint64_t *block_state = scrut.block_state; + KRML_HOST_FREE(block_state); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(s); +} + +/** +Hash `input`, of len `input_len`, into `dst`, an array of 64 bytes. +*/ +void Hacl_Streaming_SHA2_sha512(uint8_t *input, uint32_t input_len, uint8_t *dst) +{ + uint8_t *ib = input; + uint8_t *rb = dst; + uint64_t st[8U] = { 0U }; + Hacl_SHA2_Scalar32_sha512_init(st); + uint32_t rem = input_len % (uint32_t)128U; + FStar_UInt128_uint128 len_ = FStar_UInt128_uint64_to_uint128((uint64_t)input_len); + sha512_update_nblocks(input_len, ib, st); + uint32_t rem1 = input_len % (uint32_t)128U; + uint8_t *b0 = ib; + uint8_t *lb = b0 + input_len - rem1; + sha512_update_last(len_, rem, lb, st); + sha512_finish(st, rb); +} + +Hacl_Streaming_SHA2_state_sha2_384 *Hacl_Streaming_SHA2_create_in_384(void) +{ + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)128U, sizeof (uint8_t)); + uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); + Hacl_Streaming_SHA2_state_sha2_384 + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + Hacl_Streaming_SHA2_state_sha2_384 + *p = + (Hacl_Streaming_SHA2_state_sha2_384 *)KRML_HOST_MALLOC(sizeof ( + Hacl_Streaming_SHA2_state_sha2_384 + )); + p[0U] = s; + sha384_init(block_state); + return p; +} + +void Hacl_Streaming_SHA2_init_384(Hacl_Streaming_SHA2_state_sha2_384 *s) +{ + Hacl_Streaming_SHA2_state_sha2_384 scrut = *s; + uint8_t *buf = scrut.buf; + uint64_t *block_state = scrut.block_state; + sha384_init(block_state); + Hacl_Streaming_SHA2_state_sha2_384 + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s[0U] = tmp; +} + +uint32_t +Hacl_Streaming_SHA2_update_384( + Hacl_Streaming_SHA2_state_sha2_384 *p, + uint8_t *input, + uint32_t input_len +) +{ + return update_384_512(p, input, input_len); +} + +/** +Write the resulting hash into `dst`, an array of 48 bytes. The state remains +valid after a call to `finish_384`, meaning the user may feed more data into +the hash via `update_384`. +*/ +void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *dst) +{ + Hacl_Streaming_SHA2_state_sha2_384 scrut = *p; + uint64_t *block_state = scrut.block_state; + uint8_t *buf_ = scrut.buf; + uint64_t total_len = scrut.total_len; + uint32_t r; + if (total_len % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len > (uint64_t)0U) + { + r = (uint32_t)128U; + } + else + { + r = (uint32_t)(total_len % (uint64_t)(uint32_t)128U); + } + uint8_t *buf_1 = buf_; + uint64_t tmp_block_state[8U] = { 0U }; + memcpy(tmp_block_state, block_state, (uint32_t)8U * sizeof (uint64_t)); + uint32_t ite; + if (r % (uint32_t)128U == (uint32_t)0U && r > (uint32_t)0U) + { + ite = (uint32_t)128U; + } + else + { + ite = r % (uint32_t)128U; + } + uint8_t *buf_last = buf_1 + r - ite; + uint8_t *buf_multi = buf_1; + sha384_update_nblocks((uint32_t)0U, buf_multi, tmp_block_state); + uint64_t prev_len_last = total_len - (uint64_t)r; + sha384_update_last(FStar_UInt128_add(FStar_UInt128_uint64_to_uint128(prev_len_last), + FStar_UInt128_uint64_to_uint128((uint64_t)r)), + r, + buf_last, + tmp_block_state); + sha384_finish(tmp_block_state, dst); +} + +void Hacl_Streaming_SHA2_free_384(Hacl_Streaming_SHA2_state_sha2_384 *p) +{ + Hacl_Streaming_SHA2_free_512(p); +} + +/** +Hash `input`, of len `input_len`, into `dst`, an array of 48 bytes. +*/ +void Hacl_Streaming_SHA2_sha384(uint8_t *input, uint32_t input_len, uint8_t *dst) +{ + uint8_t *ib = input; + uint8_t *rb = dst; + uint64_t st[8U] = { 0U }; + sha384_init(st); + uint32_t rem = input_len % (uint32_t)128U; + FStar_UInt128_uint128 len_ = FStar_UInt128_uint64_to_uint128((uint64_t)input_len); + sha384_update_nblocks(input_len, ib, st); + uint32_t rem1 = input_len % (uint32_t)128U; + uint8_t *b0 = ib; + uint8_t *lb = b0 + input_len - rem1; + sha384_update_last(len_, rem, lb, st); + sha384_finish(st, rb); +} + diff --git a/Modules/_hacl/Hacl_Streaming_SHA2.h b/Modules/_hacl/Hacl_Streaming_SHA2.h index c83a835afe70fd..2c905854f336fd 100644 --- a/Modules/_hacl/Hacl_Streaming_SHA2.h +++ b/Modules/_hacl/Hacl_Streaming_SHA2.h @@ -32,13 +32,12 @@ extern "C" { #include #include "python_hacl_namespaces.h" -#include "krml/FStar_UInt_8_16_32_64.h" +#include "krml/types.h" #include "krml/lowstar_endianness.h" #include "krml/internal/target.h" - typedef struct Hacl_Streaming_SHA2_state_sha2_224_s { uint32_t *block_state; @@ -49,6 +48,16 @@ Hacl_Streaming_SHA2_state_sha2_224; typedef Hacl_Streaming_SHA2_state_sha2_224 Hacl_Streaming_SHA2_state_sha2_256; +typedef struct Hacl_Streaming_SHA2_state_sha2_384_s +{ + uint64_t *block_state; + uint8_t *buf; + uint64_t total_len; +} +Hacl_Streaming_SHA2_state_sha2_384; + +typedef Hacl_Streaming_SHA2_state_sha2_384 Hacl_Streaming_SHA2_state_sha2_512; + /** Allocate initial state for the SHA2_256 hash. The state is to be freed by calling `free_256`. @@ -128,6 +137,78 @@ Hash `input`, of len `input_len`, into `dst`, an array of 28 bytes. */ void Hacl_Streaming_SHA2_sha224(uint8_t *input, uint32_t input_len, uint8_t *dst); +Hacl_Streaming_SHA2_state_sha2_384 *Hacl_Streaming_SHA2_create_in_512(void); + +/** +Copies the state passed as argument into a newly allocated state (deep copy). +The state is to be freed by calling `free_512`. Cloning the state this way is +useful, for instance, if your control-flow diverges and you need to feed +more (different) data into the hash in each branch. +*/ +Hacl_Streaming_SHA2_state_sha2_384 +*Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_SHA2_state_sha2_384 *s0); + +void Hacl_Streaming_SHA2_init_512(Hacl_Streaming_SHA2_state_sha2_384 *s); + +/** +Feed an arbitrary amount of data into the hash. This function returns 0 for +success, or 1 if the combined length of all of the data passed to `update_512` +(since the last call to `init_512`) exceeds 2^125-1 bytes. + +This function is identical to the update function for SHA2_384. +*/ +uint32_t +Hacl_Streaming_SHA2_update_512( + Hacl_Streaming_SHA2_state_sha2_384 *p, + uint8_t *input, + uint32_t input_len +); + +/** +Write the resulting hash into `dst`, an array of 64 bytes. The state remains +valid after a call to `finish_512`, meaning the user may feed more data into +the hash via `update_512`. (The finish_512 function operates on an internal copy of +the state and therefore does not invalidate the client-held state `p`.) +*/ +void Hacl_Streaming_SHA2_finish_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *dst); + +/** +Free a state allocated with `create_in_512`. + +This function is identical to the free function for SHA2_384. +*/ +void Hacl_Streaming_SHA2_free_512(Hacl_Streaming_SHA2_state_sha2_384 *s); + +/** +Hash `input`, of len `input_len`, into `dst`, an array of 64 bytes. +*/ +void Hacl_Streaming_SHA2_sha512(uint8_t *input, uint32_t input_len, uint8_t *dst); + +Hacl_Streaming_SHA2_state_sha2_384 *Hacl_Streaming_SHA2_create_in_384(void); + +void Hacl_Streaming_SHA2_init_384(Hacl_Streaming_SHA2_state_sha2_384 *s); + +uint32_t +Hacl_Streaming_SHA2_update_384( + Hacl_Streaming_SHA2_state_sha2_384 *p, + uint8_t *input, + uint32_t input_len +); + +/** +Write the resulting hash into `dst`, an array of 48 bytes. The state remains +valid after a call to `finish_384`, meaning the user may feed more data into +the hash via `update_384`. +*/ +void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *dst); + +void Hacl_Streaming_SHA2_free_384(Hacl_Streaming_SHA2_state_sha2_384 *p); + +/** +Hash `input`, of len `input_len`, into `dst`, an array of 48 bytes. +*/ +void Hacl_Streaming_SHA2_sha384(uint8_t *input, uint32_t input_len, uint8_t *dst); + #if defined(__cplusplus) } #endif diff --git a/Modules/_hacl/include/krml/FStar_UInt128_Verified.h b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h new file mode 100644 index 00000000000000..ee160193539e28 --- /dev/null +++ b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h @@ -0,0 +1,347 @@ +/* + Copyright (c) INRIA and Microsoft Corporation. All rights reserved. + Licensed under the Apache 2.0 License. +*/ + + +#ifndef __FStar_UInt128_Verified_H +#define __FStar_UInt128_Verified_H + + + +#include "FStar_UInt_8_16_32_64.h" +#include +#include +#include "krml/types.h" +#include "krml/internal/target.h" +static inline uint64_t FStar_UInt128_constant_time_carry(uint64_t a, uint64_t b) +{ + return (a ^ ((a ^ b) | ((a - b) ^ b))) >> (uint32_t)63U; +} + +static inline uint64_t FStar_UInt128_carry(uint64_t a, uint64_t b) +{ + return FStar_UInt128_constant_time_carry(a, b); +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_add(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = a.low + b.low; + lit.high = a.high + b.high + FStar_UInt128_carry(a.low + b.low, b.low); + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_add_underspec(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = a.low + b.low; + lit.high = a.high + b.high + FStar_UInt128_carry(a.low + b.low, b.low); + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_add_mod(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = a.low + b.low; + lit.high = a.high + b.high + FStar_UInt128_carry(a.low + b.low, b.low); + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_sub(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = a.low - b.low; + lit.high = a.high - b.high - FStar_UInt128_carry(a.low, a.low - b.low); + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_sub_underspec(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = a.low - b.low; + lit.high = a.high - b.high - FStar_UInt128_carry(a.low, a.low - b.low); + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_sub_mod_impl(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = a.low - b.low; + lit.high = a.high - b.high - FStar_UInt128_carry(a.low, a.low - b.low); + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_sub_mod(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + return FStar_UInt128_sub_mod_impl(a, b); +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_logand(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = a.low & b.low; + lit.high = a.high & b.high; + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_logxor(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = a.low ^ b.low; + lit.high = a.high ^ b.high; + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_logor(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = a.low | b.low; + lit.high = a.high | b.high; + return lit; +} + +static inline FStar_UInt128_uint128 FStar_UInt128_lognot(FStar_UInt128_uint128 a) +{ + FStar_UInt128_uint128 lit; + lit.low = ~a.low; + lit.high = ~a.high; + return lit; +} + +static uint32_t FStar_UInt128_u32_64 = (uint32_t)64U; + +static inline uint64_t FStar_UInt128_add_u64_shift_left(uint64_t hi, uint64_t lo, uint32_t s) +{ + return (hi << s) + (lo >> (FStar_UInt128_u32_64 - s)); +} + +static inline uint64_t +FStar_UInt128_add_u64_shift_left_respec(uint64_t hi, uint64_t lo, uint32_t s) +{ + return FStar_UInt128_add_u64_shift_left(hi, lo, s); +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_shift_left_small(FStar_UInt128_uint128 a, uint32_t s) +{ + if (s == (uint32_t)0U) + { + return a; + } + else + { + FStar_UInt128_uint128 lit; + lit.low = a.low << s; + lit.high = FStar_UInt128_add_u64_shift_left_respec(a.high, a.low, s); + return lit; + } +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_shift_left_large(FStar_UInt128_uint128 a, uint32_t s) +{ + FStar_UInt128_uint128 lit; + lit.low = (uint64_t)0U; + lit.high = a.low << (s - FStar_UInt128_u32_64); + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_shift_left(FStar_UInt128_uint128 a, uint32_t s) +{ + if (s < FStar_UInt128_u32_64) + { + return FStar_UInt128_shift_left_small(a, s); + } + else + { + return FStar_UInt128_shift_left_large(a, s); + } +} + +static inline uint64_t FStar_UInt128_add_u64_shift_right(uint64_t hi, uint64_t lo, uint32_t s) +{ + return (lo >> s) + (hi << (FStar_UInt128_u32_64 - s)); +} + +static inline uint64_t +FStar_UInt128_add_u64_shift_right_respec(uint64_t hi, uint64_t lo, uint32_t s) +{ + return FStar_UInt128_add_u64_shift_right(hi, lo, s); +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_shift_right_small(FStar_UInt128_uint128 a, uint32_t s) +{ + if (s == (uint32_t)0U) + { + return a; + } + else + { + FStar_UInt128_uint128 lit; + lit.low = FStar_UInt128_add_u64_shift_right_respec(a.high, a.low, s); + lit.high = a.high >> s; + return lit; + } +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_shift_right_large(FStar_UInt128_uint128 a, uint32_t s) +{ + FStar_UInt128_uint128 lit; + lit.low = a.high >> (s - FStar_UInt128_u32_64); + lit.high = (uint64_t)0U; + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_shift_right(FStar_UInt128_uint128 a, uint32_t s) +{ + if (s < FStar_UInt128_u32_64) + { + return FStar_UInt128_shift_right_small(a, s); + } + else + { + return FStar_UInt128_shift_right_large(a, s); + } +} + +static inline bool FStar_UInt128_eq(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + return a.low == b.low && a.high == b.high; +} + +static inline bool FStar_UInt128_gt(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + return a.high > b.high || (a.high == b.high && a.low > b.low); +} + +static inline bool FStar_UInt128_lt(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + return a.high < b.high || (a.high == b.high && a.low < b.low); +} + +static inline bool FStar_UInt128_gte(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + return a.high > b.high || (a.high == b.high && a.low >= b.low); +} + +static inline bool FStar_UInt128_lte(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + return a.high < b.high || (a.high == b.high && a.low <= b.low); +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_eq_mask(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = FStar_UInt64_eq_mask(a.low, b.low) & FStar_UInt64_eq_mask(a.high, b.high); + lit.high = FStar_UInt64_eq_mask(a.low, b.low) & FStar_UInt64_eq_mask(a.high, b.high); + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_gte_mask(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = + (FStar_UInt64_gte_mask(a.high, b.high) & ~FStar_UInt64_eq_mask(a.high, b.high)) + | (FStar_UInt64_eq_mask(a.high, b.high) & FStar_UInt64_gte_mask(a.low, b.low)); + lit.high = + (FStar_UInt64_gte_mask(a.high, b.high) & ~FStar_UInt64_eq_mask(a.high, b.high)) + | (FStar_UInt64_eq_mask(a.high, b.high) & FStar_UInt64_gte_mask(a.low, b.low)); + return lit; +} + +static inline FStar_UInt128_uint128 FStar_UInt128_uint64_to_uint128(uint64_t a) +{ + FStar_UInt128_uint128 lit; + lit.low = a; + lit.high = (uint64_t)0U; + return lit; +} + +static inline uint64_t FStar_UInt128_uint128_to_uint64(FStar_UInt128_uint128 a) +{ + return a.low; +} + +static inline uint64_t FStar_UInt128_u64_mod_32(uint64_t a) +{ + return a & (uint64_t)0xffffffffU; +} + +static uint32_t FStar_UInt128_u32_32 = (uint32_t)32U; + +static inline uint64_t FStar_UInt128_u32_combine(uint64_t hi, uint64_t lo) +{ + return lo + (hi << FStar_UInt128_u32_32); +} + +static inline FStar_UInt128_uint128 FStar_UInt128_mul32(uint64_t x, uint32_t y) +{ + FStar_UInt128_uint128 lit; + lit.low = + FStar_UInt128_u32_combine((x >> FStar_UInt128_u32_32) + * (uint64_t)y + + (FStar_UInt128_u64_mod_32(x) * (uint64_t)y >> FStar_UInt128_u32_32), + FStar_UInt128_u64_mod_32(FStar_UInt128_u64_mod_32(x) * (uint64_t)y)); + lit.high = + ((x >> FStar_UInt128_u32_32) + * (uint64_t)y + + (FStar_UInt128_u64_mod_32(x) * (uint64_t)y >> FStar_UInt128_u32_32)) + >> FStar_UInt128_u32_32; + return lit; +} + +static inline uint64_t FStar_UInt128_u32_combine_(uint64_t hi, uint64_t lo) +{ + return lo + (hi << FStar_UInt128_u32_32); +} + +static inline FStar_UInt128_uint128 FStar_UInt128_mul_wide(uint64_t x, uint64_t y) +{ + FStar_UInt128_uint128 lit; + lit.low = + FStar_UInt128_u32_combine_(FStar_UInt128_u64_mod_32(x) + * (y >> FStar_UInt128_u32_32) + + + FStar_UInt128_u64_mod_32((x >> FStar_UInt128_u32_32) + * FStar_UInt128_u64_mod_32(y) + + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32)), + FStar_UInt128_u64_mod_32(FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y))); + lit.high = + (x >> FStar_UInt128_u32_32) + * (y >> FStar_UInt128_u32_32) + + + (((x >> FStar_UInt128_u32_32) + * FStar_UInt128_u64_mod_32(y) + + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32)) + >> FStar_UInt128_u32_32) + + + ((FStar_UInt128_u64_mod_32(x) + * (y >> FStar_UInt128_u32_32) + + + FStar_UInt128_u64_mod_32((x >> FStar_UInt128_u32_32) + * FStar_UInt128_u64_mod_32(y) + + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32))) + >> FStar_UInt128_u32_32); + return lit; +} + + +#define __FStar_UInt128_Verified_H_DEFINED +#endif diff --git a/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h b/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h index 3e2e4b32b22f96..965afc836fd12b 100644 --- a/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h +++ b/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h @@ -14,7 +14,7 @@ #include #include "krml/lowstar_endianness.h" -#include "krml/FStar_UInt_8_16_32_64.h" +#include "krml/types.h" #include "krml/internal/target.h" static inline uint64_t FStar_UInt64_eq_mask(uint64_t a, uint64_t b) { diff --git a/Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h b/Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h new file mode 100644 index 00000000000000..e2b6d62859a5f1 --- /dev/null +++ b/Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h @@ -0,0 +1,68 @@ +/* Copyright (c) INRIA and Microsoft Corporation. All rights reserved. + Licensed under the Apache 2.0 License. */ + +#ifndef FSTAR_UINT128_STRUCT_ENDIANNESS_H +#define FSTAR_UINT128_STRUCT_ENDIANNESS_H + +/* Hand-written implementation of endianness-related uint128 functions + * for the extracted uint128 implementation */ + +/* Access 64-bit fields within the int128. */ +#define HIGH64_OF(x) ((x)->high) +#define LOW64_OF(x) ((x)->low) + +/* A series of definitions written using pointers. */ + +inline static void load128_le_(uint8_t *b, uint128_t *r) { + LOW64_OF(r) = load64_le(b); + HIGH64_OF(r) = load64_le(b + 8); +} + +inline static void store128_le_(uint8_t *b, uint128_t *n) { + store64_le(b, LOW64_OF(n)); + store64_le(b + 8, HIGH64_OF(n)); +} + +inline static void load128_be_(uint8_t *b, uint128_t *r) { + HIGH64_OF(r) = load64_be(b); + LOW64_OF(r) = load64_be(b + 8); +} + +inline static void store128_be_(uint8_t *b, uint128_t *n) { + store64_be(b, HIGH64_OF(n)); + store64_be(b + 8, LOW64_OF(n)); +} + +#ifndef KRML_NOSTRUCT_PASSING + +inline static uint128_t load128_le(uint8_t *b) { + uint128_t r; + load128_le_(b, &r); + return r; +} + +inline static void store128_le(uint8_t *b, uint128_t n) { + store128_le_(b, &n); +} + +inline static uint128_t load128_be(uint8_t *b) { + uint128_t r; + load128_be_(b, &r); + return r; +} + +inline static void store128_be(uint8_t *b, uint128_t n) { + store128_be_(b, &n); +} + +#else /* !defined(KRML_STRUCT_PASSING) */ + +# define print128 print128_ +# define load128_le load128_le_ +# define store128_le store128_le_ +# define load128_be load128_be_ +# define store128_be store128_be_ + +#endif /* KRML_STRUCT_PASSING */ + +#endif diff --git a/Modules/_hacl/include/krml/types.h b/Modules/_hacl/include/krml/types.h new file mode 100644 index 00000000000000..509f555536e4c6 --- /dev/null +++ b/Modules/_hacl/include/krml/types.h @@ -0,0 +1,14 @@ +#pragma once + +#include + +typedef struct FStar_UInt128_uint128_s { + uint64_t low; + uint64_t high; +} FStar_UInt128_uint128, uint128_t; + +#define KRML_VERIFIED_UINT128 + +#include "krml/lowstar_endianness.h" +#include "krml/fstar_uint128_struct_endianness.h" +#include "krml/FStar_UInt128_Verified.h" diff --git a/Modules/_hacl/include/python_hacl_namespaces.h b/Modules/_hacl/include/python_hacl_namespaces.h index af390459311fe8..65608d1fd283c4 100644 --- a/Modules/_hacl/include/python_hacl_namespaces.h +++ b/Modules/_hacl/include/python_hacl_namespaces.h @@ -10,19 +10,36 @@ #define Hacl_Streaming_SHA2_state_sha2_224_s python_hashlib_Hacl_Streaming_SHA2_state_sha2_224_s #define Hacl_Streaming_SHA2_state_sha2_224 python_hashlib_Hacl_Streaming_SHA2_state_sha2_224 #define Hacl_Streaming_SHA2_state_sha2_256 python_hashlib_Hacl_Streaming_SHA2_state_sha2_256 +#define Hacl_Streaming_SHA2_state_sha2_384_s python_hashlib_Hacl_Streaming_SHA2_state_sha2_384_s +#define Hacl_Streaming_SHA2_state_sha2_384 python_hashlib_Hacl_Streaming_SHA2_state_sha2_384 +#define Hacl_Streaming_SHA2_state_sha2_512 python_hashlib_Hacl_Streaming_SHA2_state_sha2_512 #define Hacl_Streaming_SHA2_create_in_256 python_hashlib_Hacl_Streaming_SHA2_create_in_256 #define Hacl_Streaming_SHA2_create_in_224 python_hashlib_Hacl_Streaming_SHA2_create_in_224 +#define Hacl_Streaming_SHA2_create_in_512 python_hashlib_Hacl_Streaming_SHA2_create_in_512 +#define Hacl_Streaming_SHA2_create_in_384 python_hashlib_Hacl_Streaming_SHA2_create_in_384 #define Hacl_Streaming_SHA2_copy_256 python_hashlib_Hacl_Streaming_SHA2_copy_256 #define Hacl_Streaming_SHA2_copy_224 python_hashlib_Hacl_Streaming_SHA2_copy_224 +#define Hacl_Streaming_SHA2_copy_512 python_hashlib_Hacl_Streaming_SHA2_copy_512 +#define Hacl_Streaming_SHA2_copy_384 python_hashlib_Hacl_Streaming_SHA2_copy_384 #define Hacl_Streaming_SHA2_init_256 python_hashlib_Hacl_Streaming_SHA2_init_256 #define Hacl_Streaming_SHA2_init_224 python_hashlib_Hacl_Streaming_SHA2_init_224 +#define Hacl_Streaming_SHA2_init_512 python_hashlib_Hacl_Streaming_SHA2_init_512 +#define Hacl_Streaming_SHA2_init_384 python_hashlib_Hacl_Streaming_SHA2_init_384 #define Hacl_Streaming_SHA2_update_256 python_hashlib_Hacl_Streaming_SHA2_update_256 #define Hacl_Streaming_SHA2_update_224 python_hashlib_Hacl_Streaming_SHA2_update_224 +#define Hacl_Streaming_SHA2_update_512 python_hashlib_Hacl_Streaming_SHA2_update_512 +#define Hacl_Streaming_SHA2_update_384 python_hashlib_Hacl_Streaming_SHA2_update_384 #define Hacl_Streaming_SHA2_finish_256 python_hashlib_Hacl_Streaming_SHA2_finish_256 #define Hacl_Streaming_SHA2_finish_224 python_hashlib_Hacl_Streaming_SHA2_finish_224 +#define Hacl_Streaming_SHA2_finish_512 python_hashlib_Hacl_Streaming_SHA2_finish_512 +#define Hacl_Streaming_SHA2_finish_384 python_hashlib_Hacl_Streaming_SHA2_finish_384 #define Hacl_Streaming_SHA2_free_256 python_hashlib_Hacl_Streaming_SHA2_free_256 #define Hacl_Streaming_SHA2_free_224 python_hashlib_Hacl_Streaming_SHA2_free_224 +#define Hacl_Streaming_SHA2_free_512 python_hashlib_Hacl_Streaming_SHA2_free_512 +#define Hacl_Streaming_SHA2_free_384 python_hashlib_Hacl_Streaming_SHA2_free_384 #define Hacl_Streaming_SHA2_sha256 python_hashlib_Hacl_Streaming_SHA2_sha256 #define Hacl_Streaming_SHA2_sha224 python_hashlib_Hacl_Streaming_SHA2_sha224 +#define Hacl_Streaming_SHA2_sha512 python_hashlib_Hacl_Streaming_SHA2_sha512 +#define Hacl_Streaming_SHA2_sha384 python_hashlib_Hacl_Streaming_SHA2_sha384 #endif // _PYTHON_HACL_NAMESPACES_H diff --git a/Modules/_hacl/internal/Hacl_SHA2_Generic.h b/Modules/_hacl/internal/Hacl_SHA2_Generic.h index 23f7cea1eb3884..6ac47f3cf7ed36 100644 --- a/Modules/_hacl/internal/Hacl_SHA2_Generic.h +++ b/Modules/_hacl/internal/Hacl_SHA2_Generic.h @@ -31,13 +31,10 @@ extern "C" { #endif #include -#include "krml/FStar_UInt_8_16_32_64.h" +#include "krml/types.h" #include "krml/lowstar_endianness.h" #include "krml/internal/target.h" - - - static const uint32_t Hacl_Impl_SHA2_Generic_h224[8U] = diff --git a/Modules/_hacl/refresh.sh b/Modules/_hacl/refresh.sh index 594873862a2db0..dba8cb3972ea17 100755 --- a/Modules/_hacl/refresh.sh +++ b/Modules/_hacl/refresh.sh @@ -22,7 +22,7 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. -expected_hacl_star_rev=94aabbb4cf71347d3779a8db486c761403c6d036 +expected_hacl_star_rev=4751fc2b11639f651718abf8522fcc36902ca67c hacl_dir="$(realpath "$1")" cd "$(dirname "$0")" @@ -54,6 +54,8 @@ include_files=( declare -a lib_files lib_files=( krmllib/dist/minimal/FStar_UInt_8_16_32_64.h + krmllib/dist/minimal/fstar_uint128_struct_endianness.h + krmllib/dist/minimal/FStar_UInt128_Verified.h ) # C files for the algorithms themselves: current directory @@ -82,10 +84,27 @@ fi readarray -t all_files < <(find . -name '*.h' -or -name '*.c') -# types.h is a simple wrapper that defines the uint128 type then proceeds to -# include FStar_UInt_8_16_32_64.h; we jump the types.h step since our current -# selection of algorithms does not necessitate the use of uint128 -$sed -i 's!#include.*types.h"!#include "krml/FStar_UInt_8_16_32_64.h"!g' "${all_files[@]}" +# types.h originally contains a complex series of if-defs and auxiliary type +# definitions; here, we just need a proper uint128 type in scope +# is a simple wrapper that defines the uint128 type +cat > include/krml/types.h < + +typedef struct FStar_UInt128_uint128_s { + uint64_t low; + uint64_t high; +} FStar_UInt128_uint128, uint128_t; + +#define KRML_VERIFIED_UINT128 + +#include "krml/lowstar_endianness.h" +#include "krml/fstar_uint128_struct_endianness.h" +#include "krml/FStar_UInt128_Verified.h" +EOF +# Adjust the include path to reflect the local directory structure +$sed -i 's!#include.*types.h"!#include "krml/types.h"!g' "${all_files[@]}" $sed -i 's!#include.*compat.h"!!g' "${all_files[@]}" # FStar_UInt_8_16_32_64 contains definitions useful in the general case, but not @@ -103,22 +122,6 @@ $sed -i 's!#include.*Hacl_Krmllib.h"!!g' "${all_files[@]}" # included in $dist_files). $sed -i 's!#include.*internal/Hacl_Streaming_SHA2.h"!#include "Hacl_Streaming_SHA2.h"!g' "${all_files[@]}" -# The SHA2 file contains all variants of SHA2. We strip 384 and 512 for the time -# being, to be included later. -# This regexp matches a separator (two new lines), followed by: -# -# * -# ... 384 or 512 ... { -# * -# } -# -# The first non-empty lines are the comment block. The second ... may spill over -# the next following lines if the arguments are printed in one-per-line mode. -$sed -i -z 's/\n\n\([^\n]\+\n\)*[^\n]*\(384\|512\)[^{]*{\n\?\( [^\n]*\n\)*}//g' Hacl_Streaming_SHA2.c - -# Same thing with function prototypes -$sed -i -z 's/\n\n\([^\n]\+\n\)*[^\n]*\(384\|512\)[^;]*;//g' Hacl_Streaming_SHA2.h - # Use globally unique names for the Hacl_ C APIs to avoid linkage conflicts. $sed -i -z 's!#include \n!#include \n#include "python_hacl_namespaces.h"\n!' Hacl_Streaming_SHA2.h diff --git a/Modules/sha256module.c b/Modules/sha256module.c index 630e4bf03bbe96..301c9837bb6720 100644 --- a/Modules/sha256module.c +++ b/Modules/sha256module.c @@ -8,6 +8,7 @@ Andrew Kuchling (amk@amk.ca) Greg Stein (gstein@lyra.org) Trevor Perrin (trevp@trevp.net) + Jonathan Protzenko (jonathan@protzenko.fr) Copyright (C) 2005-2007 Gregory P. Smith (greg@krypto.org) Licensed to PSF under a Contributor Agreement. diff --git a/Modules/sha512module.c b/Modules/sha512module.c index bf4408b455f2c4..d7dfed4e5db03a 100644 --- a/Modules/sha512module.c +++ b/Modules/sha512module.c @@ -8,6 +8,7 @@ Andrew Kuchling (amk@amk.ca) Greg Stein (gstein@lyra.org) Trevor Perrin (trevp@trevp.net) + Jonathan Protzenko (jonathan@protzenko.fr) Copyright (C) 2005-2007 Gregory P. Smith (greg@krypto.org) Licensed to PSF under a Contributor Agreement. @@ -31,400 +32,32 @@ class SHA512Type "SHAobject *" "&PyType_Type" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=81a3ccde92bcfe8d]*/ -/* Some useful types */ - -typedef unsigned char SHA_BYTE; -typedef uint32_t SHA_INT32; /* 32-bit integer */ -typedef uint64_t SHA_INT64; /* 64-bit integer */ /* The SHA block size and message digest sizes, in bytes */ #define SHA_BLOCKSIZE 128 #define SHA_DIGESTSIZE 64 -/* The structure for storing SHA info */ +/* The SHA2-384 and SHA2-512 implementations defer to the HACL* verified + * library. */ + +#include "_hacl/Hacl_Streaming_SHA2.h" typedef struct { PyObject_HEAD - SHA_INT64 digest[8]; /* Message digest */ - SHA_INT32 count_lo, count_hi; /* 64-bit bit count */ - SHA_BYTE data[SHA_BLOCKSIZE]; /* SHA data buffer */ - int local; /* unprocessed amount in data */ int digestsize; + Hacl_Streaming_SHA2_state_sha2_512 *state; } SHAobject; #include "clinic/sha512module.c.h" -/* When run on a little-endian CPU we need to perform byte reversal on an - array of longwords. */ - -#if PY_LITTLE_ENDIAN -static void longReverse(SHA_INT64 *buffer, int byteCount) -{ - byteCount /= sizeof(*buffer); - for (; byteCount--; buffer++) { - *buffer = _Py_bswap64(*buffer); - } -} -#endif static void SHAcopy(SHAobject *src, SHAobject *dest) { - dest->local = src->local; dest->digestsize = src->digestsize; - dest->count_lo = src->count_lo; - dest->count_hi = src->count_hi; - memcpy(dest->digest, src->digest, sizeof(src->digest)); - memcpy(dest->data, src->data, sizeof(src->data)); -} - - -/* ------------------------------------------------------------------------ - * - * This code for the SHA-512 algorithm was noted as public domain. The - * original headers are pasted below. - * - * Several changes have been made to make it more compatible with the - * Python environment and desired interface. - * - */ - -/* LibTomCrypt, modular cryptographic library -- Tom St Denis - * - * LibTomCrypt is a library that provides various cryptographic - * algorithms in a highly modular and flexible manner. - * - * The library is free for all purposes without any express - * guarantee it works. - * - * Tom St Denis, tomstdenis@iahu.ca, https://www.libtom.net - */ - - -/* SHA512 by Tom St Denis */ - -/* Various logical functions */ -#define ROR64(x, y) \ - ( ((((x) & 0xFFFFFFFFFFFFFFFFULL)>>((unsigned long long)(y) & 63)) | \ - ((x)<<((unsigned long long)(64-((y) & 63))))) & 0xFFFFFFFFFFFFFFFFULL) -#define Ch(x,y,z) (z ^ (x & (y ^ z))) -#define Maj(x,y,z) (((x | y) & z) | (x & y)) -#define S(x, n) ROR64((x),(n)) -#define R(x, n) (((x) & 0xFFFFFFFFFFFFFFFFULL) >> ((unsigned long long)n)) -#define Sigma0(x) (S(x, 28) ^ S(x, 34) ^ S(x, 39)) -#define Sigma1(x) (S(x, 14) ^ S(x, 18) ^ S(x, 41)) -#define Gamma0(x) (S(x, 1) ^ S(x, 8) ^ R(x, 7)) -#define Gamma1(x) (S(x, 19) ^ S(x, 61) ^ R(x, 6)) - - -static void -sha512_transform(SHAobject *sha_info) -{ - int i; - SHA_INT64 S[8], W[80], t0, t1; - - memcpy(W, sha_info->data, sizeof(sha_info->data)); -#if PY_LITTLE_ENDIAN - longReverse(W, (int)sizeof(sha_info->data)); -#endif - - for (i = 16; i < 80; ++i) { - W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]; - } - for (i = 0; i < 8; ++i) { - S[i] = sha_info->digest[i]; - } - - /* Compress */ -#define RND(a,b,c,d,e,f,g,h,i,ki) \ - t0 = h + Sigma1(e) + Ch(e, f, g) + ki + W[i]; \ - t1 = Sigma0(a) + Maj(a, b, c); \ - d += t0; \ - h = t0 + t1; - - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],0,0x428a2f98d728ae22ULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],1,0x7137449123ef65cdULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],2,0xb5c0fbcfec4d3b2fULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],3,0xe9b5dba58189dbbcULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],4,0x3956c25bf348b538ULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],5,0x59f111f1b605d019ULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],6,0x923f82a4af194f9bULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],7,0xab1c5ed5da6d8118ULL); - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],8,0xd807aa98a3030242ULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],9,0x12835b0145706fbeULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],10,0x243185be4ee4b28cULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],11,0x550c7dc3d5ffb4e2ULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],12,0x72be5d74f27b896fULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],13,0x80deb1fe3b1696b1ULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],14,0x9bdc06a725c71235ULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],15,0xc19bf174cf692694ULL); - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],16,0xe49b69c19ef14ad2ULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],17,0xefbe4786384f25e3ULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],18,0x0fc19dc68b8cd5b5ULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],19,0x240ca1cc77ac9c65ULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],20,0x2de92c6f592b0275ULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],21,0x4a7484aa6ea6e483ULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],22,0x5cb0a9dcbd41fbd4ULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],23,0x76f988da831153b5ULL); - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],24,0x983e5152ee66dfabULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],25,0xa831c66d2db43210ULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],26,0xb00327c898fb213fULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],27,0xbf597fc7beef0ee4ULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],28,0xc6e00bf33da88fc2ULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],29,0xd5a79147930aa725ULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],30,0x06ca6351e003826fULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],31,0x142929670a0e6e70ULL); - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],32,0x27b70a8546d22ffcULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],33,0x2e1b21385c26c926ULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],34,0x4d2c6dfc5ac42aedULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],35,0x53380d139d95b3dfULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],36,0x650a73548baf63deULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],37,0x766a0abb3c77b2a8ULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],38,0x81c2c92e47edaee6ULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],39,0x92722c851482353bULL); - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],40,0xa2bfe8a14cf10364ULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],41,0xa81a664bbc423001ULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],42,0xc24b8b70d0f89791ULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],43,0xc76c51a30654be30ULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],44,0xd192e819d6ef5218ULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],45,0xd69906245565a910ULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],46,0xf40e35855771202aULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],47,0x106aa07032bbd1b8ULL); - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],48,0x19a4c116b8d2d0c8ULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],49,0x1e376c085141ab53ULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],50,0x2748774cdf8eeb99ULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],51,0x34b0bcb5e19b48a8ULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],52,0x391c0cb3c5c95a63ULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],53,0x4ed8aa4ae3418acbULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],54,0x5b9cca4f7763e373ULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],55,0x682e6ff3d6b2b8a3ULL); - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],56,0x748f82ee5defb2fcULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],57,0x78a5636f43172f60ULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],58,0x84c87814a1f0ab72ULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],59,0x8cc702081a6439ecULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],60,0x90befffa23631e28ULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],61,0xa4506cebde82bde9ULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],62,0xbef9a3f7b2c67915ULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],63,0xc67178f2e372532bULL); - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],64,0xca273eceea26619cULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],65,0xd186b8c721c0c207ULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],66,0xeada7dd6cde0eb1eULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],67,0xf57d4f7fee6ed178ULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],68,0x06f067aa72176fbaULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],69,0x0a637dc5a2c898a6ULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],70,0x113f9804bef90daeULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],71,0x1b710b35131c471bULL); - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],72,0x28db77f523047d84ULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],73,0x32caab7b40c72493ULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],74,0x3c9ebe0a15c9bebcULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],75,0x431d67c49c100d4cULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],76,0x4cc5d4becb3e42b6ULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],77,0x597f299cfc657e2aULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],78,0x5fcb6fab3ad6faecULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],79,0x6c44198c4a475817ULL); - -#undef RND - - /* feedback */ - for (i = 0; i < 8; i++) { - sha_info->digest[i] = sha_info->digest[i] + S[i]; - } - -} - - - -/* initialize the SHA digest */ - -static void -sha512_init(SHAobject *sha_info) -{ - sha_info->digest[0] = Py_ULL(0x6a09e667f3bcc908); - sha_info->digest[1] = Py_ULL(0xbb67ae8584caa73b); - sha_info->digest[2] = Py_ULL(0x3c6ef372fe94f82b); - sha_info->digest[3] = Py_ULL(0xa54ff53a5f1d36f1); - sha_info->digest[4] = Py_ULL(0x510e527fade682d1); - sha_info->digest[5] = Py_ULL(0x9b05688c2b3e6c1f); - sha_info->digest[6] = Py_ULL(0x1f83d9abfb41bd6b); - sha_info->digest[7] = Py_ULL(0x5be0cd19137e2179); - sha_info->count_lo = 0L; - sha_info->count_hi = 0L; - sha_info->local = 0; - sha_info->digestsize = 64; -} - -static void -sha384_init(SHAobject *sha_info) -{ - sha_info->digest[0] = Py_ULL(0xcbbb9d5dc1059ed8); - sha_info->digest[1] = Py_ULL(0x629a292a367cd507); - sha_info->digest[2] = Py_ULL(0x9159015a3070dd17); - sha_info->digest[3] = Py_ULL(0x152fecd8f70e5939); - sha_info->digest[4] = Py_ULL(0x67332667ffc00b31); - sha_info->digest[5] = Py_ULL(0x8eb44a8768581511); - sha_info->digest[6] = Py_ULL(0xdb0c2e0d64f98fa7); - sha_info->digest[7] = Py_ULL(0x47b5481dbefa4fa4); - sha_info->count_lo = 0L; - sha_info->count_hi = 0L; - sha_info->local = 0; - sha_info->digestsize = 48; -} - - -/* update the SHA digest */ - -static void -sha512_update(SHAobject *sha_info, SHA_BYTE *buffer, Py_ssize_t count) -{ - Py_ssize_t i; - SHA_INT32 clo; - - clo = sha_info->count_lo + ((SHA_INT32) count << 3); - if (clo < sha_info->count_lo) { - ++sha_info->count_hi; - } - sha_info->count_lo = clo; - sha_info->count_hi += (SHA_INT32) count >> 29; - if (sha_info->local) { - i = SHA_BLOCKSIZE - sha_info->local; - if (i > count) { - i = count; - } - memcpy(((SHA_BYTE *) sha_info->data) + sha_info->local, buffer, i); - count -= i; - buffer += i; - sha_info->local += (int)i; - if (sha_info->local == SHA_BLOCKSIZE) { - sha512_transform(sha_info); - } - else { - return; - } - } - while (count >= SHA_BLOCKSIZE) { - memcpy(sha_info->data, buffer, SHA_BLOCKSIZE); - buffer += SHA_BLOCKSIZE; - count -= SHA_BLOCKSIZE; - sha512_transform(sha_info); - } - memcpy(sha_info->data, buffer, count); - sha_info->local = (int)count; + dest->state = Hacl_Streaming_SHA2_copy_512(src->state); } -/* finish computing the SHA digest */ - -static void -sha512_final(unsigned char digest[SHA_DIGESTSIZE], SHAobject *sha_info) -{ - int count; - SHA_INT32 lo_bit_count, hi_bit_count; - - lo_bit_count = sha_info->count_lo; - hi_bit_count = sha_info->count_hi; - count = (int) ((lo_bit_count >> 3) & 0x7f); - ((SHA_BYTE *) sha_info->data)[count++] = 0x80; - if (count > SHA_BLOCKSIZE - 16) { - memset(((SHA_BYTE *) sha_info->data) + count, 0, - SHA_BLOCKSIZE - count); - sha512_transform(sha_info); - memset((SHA_BYTE *) sha_info->data, 0, SHA_BLOCKSIZE - 16); - } - else { - memset(((SHA_BYTE *) sha_info->data) + count, 0, - SHA_BLOCKSIZE - 16 - count); - } - - /* GJS: note that we add the hi/lo in big-endian. sha512_transform will - swap these values into host-order. */ - sha_info->data[112] = 0; - sha_info->data[113] = 0; - sha_info->data[114] = 0; - sha_info->data[115] = 0; - sha_info->data[116] = 0; - sha_info->data[117] = 0; - sha_info->data[118] = 0; - sha_info->data[119] = 0; - sha_info->data[120] = (hi_bit_count >> 24) & 0xff; - sha_info->data[121] = (hi_bit_count >> 16) & 0xff; - sha_info->data[122] = (hi_bit_count >> 8) & 0xff; - sha_info->data[123] = (hi_bit_count >> 0) & 0xff; - sha_info->data[124] = (lo_bit_count >> 24) & 0xff; - sha_info->data[125] = (lo_bit_count >> 16) & 0xff; - sha_info->data[126] = (lo_bit_count >> 8) & 0xff; - sha_info->data[127] = (lo_bit_count >> 0) & 0xff; - sha512_transform(sha_info); - digest[ 0] = (unsigned char) ((sha_info->digest[0] >> 56) & 0xff); - digest[ 1] = (unsigned char) ((sha_info->digest[0] >> 48) & 0xff); - digest[ 2] = (unsigned char) ((sha_info->digest[0] >> 40) & 0xff); - digest[ 3] = (unsigned char) ((sha_info->digest[0] >> 32) & 0xff); - digest[ 4] = (unsigned char) ((sha_info->digest[0] >> 24) & 0xff); - digest[ 5] = (unsigned char) ((sha_info->digest[0] >> 16) & 0xff); - digest[ 6] = (unsigned char) ((sha_info->digest[0] >> 8) & 0xff); - digest[ 7] = (unsigned char) ((sha_info->digest[0] ) & 0xff); - digest[ 8] = (unsigned char) ((sha_info->digest[1] >> 56) & 0xff); - digest[ 9] = (unsigned char) ((sha_info->digest[1] >> 48) & 0xff); - digest[10] = (unsigned char) ((sha_info->digest[1] >> 40) & 0xff); - digest[11] = (unsigned char) ((sha_info->digest[1] >> 32) & 0xff); - digest[12] = (unsigned char) ((sha_info->digest[1] >> 24) & 0xff); - digest[13] = (unsigned char) ((sha_info->digest[1] >> 16) & 0xff); - digest[14] = (unsigned char) ((sha_info->digest[1] >> 8) & 0xff); - digest[15] = (unsigned char) ((sha_info->digest[1] ) & 0xff); - digest[16] = (unsigned char) ((sha_info->digest[2] >> 56) & 0xff); - digest[17] = (unsigned char) ((sha_info->digest[2] >> 48) & 0xff); - digest[18] = (unsigned char) ((sha_info->digest[2] >> 40) & 0xff); - digest[19] = (unsigned char) ((sha_info->digest[2] >> 32) & 0xff); - digest[20] = (unsigned char) ((sha_info->digest[2] >> 24) & 0xff); - digest[21] = (unsigned char) ((sha_info->digest[2] >> 16) & 0xff); - digest[22] = (unsigned char) ((sha_info->digest[2] >> 8) & 0xff); - digest[23] = (unsigned char) ((sha_info->digest[2] ) & 0xff); - digest[24] = (unsigned char) ((sha_info->digest[3] >> 56) & 0xff); - digest[25] = (unsigned char) ((sha_info->digest[3] >> 48) & 0xff); - digest[26] = (unsigned char) ((sha_info->digest[3] >> 40) & 0xff); - digest[27] = (unsigned char) ((sha_info->digest[3] >> 32) & 0xff); - digest[28] = (unsigned char) ((sha_info->digest[3] >> 24) & 0xff); - digest[29] = (unsigned char) ((sha_info->digest[3] >> 16) & 0xff); - digest[30] = (unsigned char) ((sha_info->digest[3] >> 8) & 0xff); - digest[31] = (unsigned char) ((sha_info->digest[3] ) & 0xff); - digest[32] = (unsigned char) ((sha_info->digest[4] >> 56) & 0xff); - digest[33] = (unsigned char) ((sha_info->digest[4] >> 48) & 0xff); - digest[34] = (unsigned char) ((sha_info->digest[4] >> 40) & 0xff); - digest[35] = (unsigned char) ((sha_info->digest[4] >> 32) & 0xff); - digest[36] = (unsigned char) ((sha_info->digest[4] >> 24) & 0xff); - digest[37] = (unsigned char) ((sha_info->digest[4] >> 16) & 0xff); - digest[38] = (unsigned char) ((sha_info->digest[4] >> 8) & 0xff); - digest[39] = (unsigned char) ((sha_info->digest[4] ) & 0xff); - digest[40] = (unsigned char) ((sha_info->digest[5] >> 56) & 0xff); - digest[41] = (unsigned char) ((sha_info->digest[5] >> 48) & 0xff); - digest[42] = (unsigned char) ((sha_info->digest[5] >> 40) & 0xff); - digest[43] = (unsigned char) ((sha_info->digest[5] >> 32) & 0xff); - digest[44] = (unsigned char) ((sha_info->digest[5] >> 24) & 0xff); - digest[45] = (unsigned char) ((sha_info->digest[5] >> 16) & 0xff); - digest[46] = (unsigned char) ((sha_info->digest[5] >> 8) & 0xff); - digest[47] = (unsigned char) ((sha_info->digest[5] ) & 0xff); - digest[48] = (unsigned char) ((sha_info->digest[6] >> 56) & 0xff); - digest[49] = (unsigned char) ((sha_info->digest[6] >> 48) & 0xff); - digest[50] = (unsigned char) ((sha_info->digest[6] >> 40) & 0xff); - digest[51] = (unsigned char) ((sha_info->digest[6] >> 32) & 0xff); - digest[52] = (unsigned char) ((sha_info->digest[6] >> 24) & 0xff); - digest[53] = (unsigned char) ((sha_info->digest[6] >> 16) & 0xff); - digest[54] = (unsigned char) ((sha_info->digest[6] >> 8) & 0xff); - digest[55] = (unsigned char) ((sha_info->digest[6] ) & 0xff); - digest[56] = (unsigned char) ((sha_info->digest[7] >> 56) & 0xff); - digest[57] = (unsigned char) ((sha_info->digest[7] >> 48) & 0xff); - digest[58] = (unsigned char) ((sha_info->digest[7] >> 40) & 0xff); - digest[59] = (unsigned char) ((sha_info->digest[7] >> 32) & 0xff); - digest[60] = (unsigned char) ((sha_info->digest[7] >> 24) & 0xff); - digest[61] = (unsigned char) ((sha_info->digest[7] >> 16) & 0xff); - digest[62] = (unsigned char) ((sha_info->digest[7] >> 8) & 0xff); - digest[63] = (unsigned char) ((sha_info->digest[7] ) & 0xff); -} - -/* - * End of copied SHA code. - * - * ------------------------------------------------------------------------ - */ - typedef struct { PyTypeObject* sha384_type; PyTypeObject* sha512_type; @@ -463,14 +96,31 @@ SHA_traverse(PyObject *ptr, visitproc visit, void *arg) } static void -SHA512_dealloc(PyObject *ptr) +SHA512_dealloc(SHAobject *ptr) { + Hacl_Streaming_SHA2_free_512(ptr->state); PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); Py_DECREF(tp); } +/* HACL* takes a uint32_t for the length of its parameter, but Py_ssize_t can be + * 64 bits. */ +static void update_512(Hacl_Streaming_SHA2_state_sha2_512 *state, uint8_t *buf, Py_ssize_t len) { + /* Note: we explicitly ignore the error code on the basis that it would take > + * 1 billion years to overflow the maximum admissible length for this API + * (namely, 2^64-1 bytes). */ + while (len > UINT32_MAX) { + Hacl_Streaming_SHA2_update_512(state, buf, UINT32_MAX); + len -= UINT32_MAX; + buf += UINT32_MAX; + } + /* Cast to uint32_t is safe: upon exiting the loop, len <= UINT32_MAX, and + * therefore fits in a uint32_t */ + Hacl_Streaming_SHA2_update_512(state, buf, (uint32_t) len); +} + /* External methods for a hash object */ @@ -514,11 +164,10 @@ static PyObject * SHA512Type_digest_impl(SHAobject *self) /*[clinic end generated code: output=1080bbeeef7dde1b input=f6470dd359071f4b]*/ { - unsigned char digest[SHA_DIGESTSIZE]; - SHAobject temp; - - SHAcopy(self, &temp); - sha512_final(digest, &temp); + uint8_t digest[SHA_DIGESTSIZE]; + // HACL performs copies under the hood so that self->state remains valid + // after this call. + Hacl_Streaming_SHA2_finish_512(self->state, digest); return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); } @@ -532,13 +181,8 @@ static PyObject * SHA512Type_hexdigest_impl(SHAobject *self) /*[clinic end generated code: output=7373305b8601e18b input=498b877b25cbe0a2]*/ { - unsigned char digest[SHA_DIGESTSIZE]; - SHAobject temp; - - /* Get the raw (binary) digest value */ - SHAcopy(self, &temp); - sha512_final(digest, &temp); - + uint8_t digest[SHA_DIGESTSIZE]; + Hacl_Streaming_SHA2_finish_512(self->state, digest); return _Py_strhex((const char *)digest, self->digestsize); } @@ -559,7 +203,7 @@ SHA512Type_update(SHAobject *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - sha512_update(self, buf.buf, buf.len); + update_512(self->state, buf.buf, buf.len); PyBuffer_Release(&buf); Py_RETURN_NONE; @@ -622,15 +266,6 @@ static PyType_Spec sha512_sha384_type_spec = { .slots = sha512_sha384_type_slots }; -static PyType_Slot sha512_sha512_type_slots[] = { - {Py_tp_dealloc, SHA512_dealloc}, - {Py_tp_methods, SHA_methods}, - {Py_tp_members, SHA_members}, - {Py_tp_getset, SHA_getseters}, - {Py_tp_traverse, SHA_traverse}, - {0,0} -}; - // Using PyType_GetModuleState() on this type is safe since // it cannot be subclassed: it does not have the Py_TPFLAGS_BASETYPE flag. static PyType_Spec sha512_sha512_type_spec = { @@ -638,7 +273,7 @@ static PyType_Spec sha512_sha512_type_spec = { .basicsize = sizeof(SHAobject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), - .slots = sha512_sha512_type_slots + .slots = sha512_sha384_type_slots }; /* The single module-level function: new() */ @@ -671,7 +306,8 @@ _sha512_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - sha512_init(new); + new->state = Hacl_Streaming_SHA2_create_in_512(); + new->digestsize = 64; if (PyErr_Occurred()) { Py_DECREF(new); @@ -680,7 +316,7 @@ _sha512_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - sha512_update(new, buf.buf, buf.len); + update_512(new->state, buf.buf, buf.len); PyBuffer_Release(&buf); } @@ -715,7 +351,8 @@ _sha512_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - sha384_init(new); + new->state = Hacl_Streaming_SHA2_create_in_384(); + new->digestsize = 48; if (PyErr_Occurred()) { Py_DECREF(new); @@ -724,7 +361,7 @@ _sha512_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - sha512_update(new, buf.buf, buf.len); + update_512(new->state, buf.buf, buf.len); PyBuffer_Release(&buf); } diff --git a/configure b/configure index 35088f9e5cafd6..c00a1e1d2ec986 100755 --- a/configure +++ b/configure @@ -26950,7 +26950,7 @@ fi as_fn_append MODULE_BLOCK "MODULE__SHA512_STATE=$py_cv_module__sha512$as_nl" if test "x$py_cv_module__sha512" = xyes; then : - + as_fn_append MODULE_BLOCK "MODULE__SHA512_CFLAGS=-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE$as_nl" fi diff --git a/configure.ac b/configure.ac index 1ab48e0d1c160a..92a05c011026f2 100644 --- a/configure.ac +++ b/configure.ac @@ -7200,7 +7200,9 @@ PY_STDLIB_MOD([_sha1], [test "$with_builtin_sha1" = yes]) PY_STDLIB_MOD([_sha256], [test "$with_builtin_sha256" = yes], [], [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE]) -PY_STDLIB_MOD([_sha512], [test "$with_builtin_sha512" = yes]) +PY_STDLIB_MOD([_sha512], + [test "$with_builtin_sha512" = yes], [], + [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE]) PY_STDLIB_MOD([_sha3], [test "$with_builtin_sha3" = yes]) PY_STDLIB_MOD([_blake2], [test "$with_builtin_blake2" = yes], [], From 3690688149dca11589af59b7704541336613199a Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Tue, 14 Feb 2023 10:20:11 +0000 Subject: [PATCH 05/76] GH-101898: Fix missing term references for hashable definition (#101899) Fix missing term references for hashable definition --- Doc/c-api/dict.rst | 2 +- Doc/c-api/object.rst | 2 +- Doc/faq/programming.rst | 2 +- Doc/library/abc.rst | 2 +- Doc/library/collections.abc.rst | 4 ++-- Doc/library/collections.rst | 4 ++-- Doc/library/datetime.rst | 2 +- Doc/library/fractions.rst | 2 +- Doc/library/functools.rst | 2 +- Doc/library/graphlib.rst | 4 ++-- Doc/library/inspect.rst | 4 ++-- Doc/library/operator.rst | 2 +- Doc/library/pathlib.rst | 2 +- Doc/library/stdtypes.rst | 6 +++--- Doc/library/typing.rst | 2 +- Doc/reference/datamodel.rst | 2 +- 16 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index e5f28b59a701e0..34106ee6b1f30d 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -80,7 +80,7 @@ Dictionary Objects .. c:function:: int PyDict_DelItem(PyObject *p, PyObject *key) - Remove the entry in dictionary *p* with key *key*. *key* must be hashable; + Remove the entry in dictionary *p* with key *key*. *key* must be :term:`hashable`; if it isn't, :exc:`TypeError` is raised. If *key* is not in the dictionary, :exc:`KeyError` is raised. Return ``0`` on success or ``-1`` on failure. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 5a25a2b6c9d3db..84c72e7e108b64 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -281,7 +281,7 @@ Object Protocol .. c:function:: Py_hash_t PyObject_HashNotImplemented(PyObject *o) - Set a :exc:`TypeError` indicating that ``type(o)`` is not hashable and return ``-1``. + Set a :exc:`TypeError` indicating that ``type(o)`` is not :term:`hashable` and return ``-1``. This function receives special treatment when stored in a ``tp_hash`` slot, allowing a type to explicitly indicate to the interpreter that it is not hashable. diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index ba42289f3466c2..38f9b171618b26 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1979,7 +1979,7 @@ method result will be released right away. The disadvantage is that if instances accumulate, so too will the accumulated method results. They can grow without bound. -The *lru_cache* approach works with methods that have hashable +The *lru_cache* approach works with methods that have :term:`hashable` arguments. It creates a reference to the instance unless special efforts are made to pass in weak references. diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 3b74622e7ff46c..274b2d69f0ab5c 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -21,7 +21,7 @@ The :mod:`collections` module has some concrete classes that derive from ABCs; these can, of course, be further derived. In addition, the :mod:`collections.abc` submodule has some ABCs that can be used to test whether a class or instance provides a particular interface, for example, if it is -hashable or if it is a mapping. +:term:`hashable` or if it is a mapping. This module provides the metaclass :class:`ABCMeta` for defining ABCs and diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 132b0ce7192ac1..1ada0d352a0cc6 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -22,7 +22,7 @@ This module provides :term:`abstract base classes ` that can be used to test whether a class provides a particular interface; for -example, whether it is hashable or whether it is a mapping. +example, whether it is :term:`hashable` or whether it is a mapping. An :func:`issubclass` or :func:`isinstance` test for an interface works in one of three ways. @@ -406,7 +406,7 @@ Notes on using :class:`Set` and :class:`MutableSet` as a mixin: (3) The :class:`Set` mixin provides a :meth:`_hash` method to compute a hash value for the set; however, :meth:`__hash__` is not defined because not all sets - are hashable or immutable. To add set hashability using mixins, + are :term:`hashable` or immutable. To add set hashability using mixins, inherit from both :meth:`Set` and :meth:`Hashable`, then define ``__hash__ = Set._hash``. diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 2cffc2300a2298..bb46782c06e1c8 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -25,7 +25,7 @@ Python's general purpose built-in containers, :class:`dict`, :class:`list`, :func:`namedtuple` factory function for creating tuple subclasses with named fields :class:`deque` list-like container with fast appends and pops on either end :class:`ChainMap` dict-like class for creating a single view of multiple mappings -:class:`Counter` dict subclass for counting hashable objects +:class:`Counter` dict subclass for counting :term:`hashable` objects :class:`OrderedDict` dict subclass that remembers the order entries were added :class:`defaultdict` dict subclass that calls a factory function to supply missing values :class:`UserDict` wrapper around dictionary objects for easier dict subclassing @@ -242,7 +242,7 @@ For example:: .. class:: Counter([iterable-or-mapping]) - A :class:`Counter` is a :class:`dict` subclass for counting hashable objects. + A :class:`Counter` is a :class:`dict` subclass for counting :term:`hashable` objects. It is a collection where elements are stored as dictionary keys and their counts are stored as dictionary values. Counts are allowed to be any integer value including zero or negative counts. The :class:`Counter` diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 2f1ab7c3dd4b51..50827b27ebea04 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -160,7 +160,7 @@ The :class:`date`, :class:`.datetime`, :class:`.time`, and :class:`timezone` typ share these common features: - Objects of these types are immutable. -- Objects of these types are hashable, meaning that they can be used as +- Objects of these types are :term:`hashable`, meaning that they can be used as dictionary keys. - Objects of these types support efficient pickling via the :mod:`pickle` module. diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index 06b0e038c89bd0..c61bbac892e0e4 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -77,7 +77,7 @@ another rational number, or from a string. The :class:`Fraction` class inherits from the abstract base class :class:`numbers.Rational`, and implements all of the methods and - operations from that class. :class:`Fraction` instances are hashable, + operations from that class. :class:`Fraction` instances are :term:`hashable`, and should be treated as immutable. In addition, :class:`Fraction` has the following properties and methods: diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 2f0a9bd8be8815..80a405e87d8d56 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -147,7 +147,7 @@ The :mod:`functools` module defines the following functions: threads. Since a dictionary is used to cache results, the positional and keyword - arguments to the function must be hashable. + arguments to the function must be :term:`hashable`. Distinct argument patterns may be considered to be distinct calls with separate cache entries. For example, ``f(a=1, b=2)`` and ``f(b=2, a=1)`` diff --git a/Doc/library/graphlib.rst b/Doc/library/graphlib.rst index 2bc80da4ead2a2..fe7932e7a61cb5 100644 --- a/Doc/library/graphlib.rst +++ b/Doc/library/graphlib.rst @@ -17,7 +17,7 @@ .. class:: TopologicalSorter(graph=None) - Provides functionality to topologically sort a graph of hashable nodes. + Provides functionality to topologically sort a graph of :term:`hashable` nodes. A topological order is a linear ordering of the vertices in a graph such that for every directed edge u -> v from vertex u to vertex v, vertex u comes @@ -85,7 +85,7 @@ .. method:: add(node, *predecessors) Add a new node and its predecessors to the graph. Both the *node* and all - elements in *predecessors* must be hashable. + elements in *predecessors* must be :term:`hashable`. If called multiple times with the same node argument, the set of dependencies will be the union of all dependencies passed in. diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 58b84a35a890e3..9c3be5a250a67e 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -689,7 +689,7 @@ function. modified copy. .. versionchanged:: 3.5 - Signature objects are picklable and hashable. + Signature objects are picklable and :term:`hashable`. .. attribute:: Signature.empty @@ -768,7 +768,7 @@ function. you can use :meth:`Parameter.replace` to create a modified copy. .. versionchanged:: 3.5 - Parameter objects are picklable and hashable. + Parameter objects are picklable and :term:`hashable`. .. attribute:: Parameter.empty diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 35e9b49ea8bcca..6d90473f2367b5 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -327,7 +327,7 @@ expect a function argument. return g The items can be any type accepted by the operand's :meth:`__getitem__` - method. Dictionaries accept any hashable value. Lists, tuples, and + method. Dictionaries accept any :term:`hashable` value. Lists, tuples, and strings accept an index or a slice: >>> itemgetter(1)('ABCDEFG') diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index f222745a2c56bc..c8a734ecad8e7b 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -186,7 +186,7 @@ these classes, since they don't provide any operation that does system calls. General properties ^^^^^^^^^^^^^^^^^^ -Paths are immutable and hashable. Paths of a same flavour are comparable +Paths are immutable and :term:`hashable`. Paths of a same flavour are comparable and orderable. These properties respect the flavour's case-folding semantics:: diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 0ef03035a572e5..98bda6ded30f79 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -3775,7 +3775,7 @@ copying. >>> data bytearray(b'z1spam') - One-dimensional memoryviews of hashable (read-only) types with formats + One-dimensional memoryviews of :term:`hashable` (read-only) types with formats 'B', 'b' or 'c' are also hashable. The hash is defined as ``hash(m) == hash(m.tobytes())``:: @@ -3789,7 +3789,7 @@ copying. .. versionchanged:: 3.3 One-dimensional memoryviews can now be sliced. - One-dimensional memoryviews with formats 'B', 'b' or 'c' are now hashable. + One-dimensional memoryviews with formats 'B', 'b' or 'c' are now :term:`hashable`. .. versionchanged:: 3.4 memoryview is now registered automatically with @@ -4710,7 +4710,7 @@ support membership tests: .. versionadded:: 3.10 -Keys views are set-like since their entries are unique and hashable. If all +Keys views are set-like since their entries are unique and :term:`hashable`. If all values are hashable, so that ``(key, value)`` pairs are unique and hashable, then the items view is also set-like. (Values views are not treated as set-like since the entries are generally not unique.) For set-like views, all of the diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 356f919a1897b2..169f7196a74ec6 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -439,7 +439,7 @@ are intended primarily for static type checking. A user-defined generic class can have ABCs as base classes without a metaclass conflict. Generic metaclasses are not supported. The outcome of parameterizing -generics is cached, and most types in the typing module are hashable and +generics is cached, and most types in the typing module are :term:`hashable` and comparable for equality. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 1d2ddf3507aee1..e4e471e50079ae 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1525,7 +1525,7 @@ Basic customization :meth:`__hash__`, its instances will not be usable as items in hashable collections. If a class defines mutable objects and implements an :meth:`__eq__` method, it should not implement :meth:`__hash__`, since the - implementation of hashable collections requires that a key's hash value is + implementation of :term:`hashable` collections requires that a key's hash value is immutable (if the object's hash value changes, it will be in the wrong hash bucket). From 81e3aa835c32363f4547b6566edf1125386f1f6d Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 14 Feb 2023 11:54:13 +0000 Subject: [PATCH 06/76] gh-101799: implement PREP_RERAISE_STAR as an intrinsic function (#101800) --- Doc/library/dis.rst | 26 +++++++++------- Include/internal/pycore_intrinsics.h | 13 ++++++++ Include/internal/pycore_opcode.h | 18 +++++------ Include/opcode.h | 20 ++++++------- Lib/importlib/_bootstrap_external.py | 3 +- Lib/opcode.py | 2 +- ...-02-13-18-21-14.gh-issue-101799.wpHbCn.rst | 2 ++ Python/bytecodes.c | 20 ++++++------- Python/compile.c | 4 +-- Python/generated_cases.c.h | 30 +++++++++---------- Python/intrinsics.c | 18 +++++++++++ Python/opcode_metadata.h | 10 +++---- Python/opcode_targets.h | 14 ++++----- 13 files changed, 107 insertions(+), 73 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index b1e61d7e77b2f5..a5bc5e7fb6ea71 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -768,16 +768,6 @@ iterations of the loop. .. versionadded:: 3.11 -.. opcode:: PREP_RERAISE_STAR - - Combines the raised and reraised exceptions list from ``STACK[-1]``, into an - exception group to propagate from a try-except* block. Uses the original exception - group from ``STACK[-2]`` to reconstruct the structure of reraised exceptions. Pops - two items from the stack and pushes the exception to reraise or ``None`` - if there isn't one. - - .. versionadded:: 3.11 - .. opcode:: WITH_EXCEPT_START Calls the function in position 4 on the stack with arguments (type, val, tb) @@ -1515,7 +1505,8 @@ iterations of the loop. .. opcode:: CALL_INTRINSIC_1 Calls an intrinsic function with one argument. Passes ``STACK[-1]`` as the - argument and sets ``STACK[-1]`` to the result. Used to implement functionality that is necessary but not performance critical. + argument and sets ``STACK[-1]`` to the result. Used to implement + functionality that is necessary but not performance critical. The operand determines which intrinsic function is called: @@ -1529,6 +1520,19 @@ iterations of the loop. .. versionadded:: 3.12 +.. opcode:: CALL_INTRINSIC_2 + + Calls an intrinsic function with two arguments. Passes ``STACK[-2]``, ``STACK[-1]`` as the + arguments and sets ``STACK[-1]`` to the result. Used to implement functionality that is + necessary but not performance critical. + + The operand determines which intrinsic function is called: + + * ``0`` Not valid + * ``1`` Calculates the :exc:`ExceptionGroup` to raise from a ``try-except*``. + + .. versionadded:: 3.12 + **Pseudo-instructions** diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 1da618f2b4a548..deac145fff7627 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -1,4 +1,6 @@ +/* Unary Functions: */ + #define INTRINSIC_PRINT 1 #define INTRINSIC_IMPORT_STAR 2 #define INTRINSIC_STOPITERATION_ERROR 3 @@ -8,6 +10,17 @@ #define MAX_INTRINSIC_1 6 + +/* Binary Functions: */ + +#define INTRINSIC_PREP_RERAISE_STAR 1 + +#define MAX_INTRINSIC_2 1 + + typedef PyObject *(*instrinsic_func1)(PyThreadState* tstate, PyObject *value); +typedef PyObject *(*instrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2); extern instrinsic_func1 _PyIntrinsics_UnaryFunctions[]; +extern instrinsic_func2 _PyIntrinsics_BinaryFunctions[]; + diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 5e65adee9e00a5..f9ab95ca4bb9d3 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -87,6 +87,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, [CALL_INTRINSIC_1] = CALL_INTRINSIC_1, + [CALL_INTRINSIC_2] = CALL_INTRINSIC_2, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, [CALL_NO_KW_BUILTIN_FAST] = CALL, [CALL_NO_KW_BUILTIN_O] = CALL, @@ -187,7 +188,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, [POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE, [POP_TOP] = POP_TOP, - [PREP_RERAISE_STAR] = PREP_RERAISE_STAR, [PUSH_EXC_INFO] = PUSH_EXC_INFO, [PUSH_NULL] = PUSH_NULL, [RAISE_VARARGS] = RAISE_VARARGS, @@ -319,7 +319,7 @@ static const char *const _PyOpcode_OpName[263] = { [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", - [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [POP_EXCEPT] = "POP_EXCEPT", [STORE_NAME] = "STORE_NAME", [DELETE_NAME] = "DELETE_NAME", @@ -344,7 +344,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", + [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -374,7 +374,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_BACKWARD] = "JUMP_BACKWARD", [COMPARE_AND_BRANCH] = "COMPARE_AND_BRANCH", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", + [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -384,20 +384,20 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", - [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", + [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", - [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", + [SEND_GEN] = "SEND_GEN", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", - [SEND_GEN] = "SEND_GEN", + [166] = "<166>", [167] = "<167>", [168] = "<168>", [169] = "<169>", @@ -405,7 +405,7 @@ static const char *const _PyOpcode_OpName[263] = { [CALL] = "CALL", [KW_NAMES] = "KW_NAMES", [CALL_INTRINSIC_1] = "CALL_INTRINSIC_1", - [174] = "<174>", + [CALL_INTRINSIC_2] = "CALL_INTRINSIC_2", [175] = "<175>", [176] = "<176>", [177] = "<177>", @@ -498,11 +498,11 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ + case 166: \ case 167: \ case 168: \ case 169: \ case 170: \ - case 174: \ case 175: \ case 176: \ case 177: \ diff --git a/Include/opcode.h b/Include/opcode.h index d643741c3c3aa0..760ff945f31f9e 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -43,7 +43,6 @@ extern "C" { #define RETURN_GENERATOR 75 #define RETURN_VALUE 83 #define SETUP_ANNOTATIONS 85 -#define PREP_RERAISE_STAR 88 #define POP_EXCEPT 89 #define HAVE_ARGUMENT 90 #define STORE_NAME 90 @@ -117,6 +116,7 @@ extern "C" { #define CALL 171 #define KW_NAMES 172 #define CALL_INTRINSIC_1 173 +#define CALL_INTRINSIC_2 174 #define MIN_PSEUDO_OPCODE 256 #define SETUP_FINALLY 256 #define SETUP_CLEANUP 257 @@ -179,15 +179,15 @@ extern "C" { #define LOAD_GLOBAL_MODULE 84 #define STORE_ATTR_INSTANCE_VALUE 86 #define STORE_ATTR_SLOT 87 -#define STORE_ATTR_WITH_HINT 113 -#define STORE_FAST__LOAD_FAST 143 -#define STORE_FAST__STORE_FAST 153 -#define STORE_SUBSCR_DICT 154 -#define STORE_SUBSCR_LIST_INT 158 -#define UNPACK_SEQUENCE_LIST 159 -#define UNPACK_SEQUENCE_TUPLE 160 -#define UNPACK_SEQUENCE_TWO_TUPLE 161 -#define SEND_GEN 166 +#define STORE_ATTR_WITH_HINT 88 +#define STORE_FAST__LOAD_FAST 113 +#define STORE_FAST__STORE_FAST 143 +#define STORE_SUBSCR_DICT 153 +#define STORE_SUBSCR_LIST_INT 154 +#define UNPACK_SEQUENCE_LIST 158 +#define UNPACK_SEQUENCE_TUPLE 159 +#define UNPACK_SEQUENCE_TWO_TUPLE 160 +#define SEND_GEN 161 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 38d4a384c2cc95..954401cfa85ed3 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -433,6 +433,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth) # Python 3.12a5 3518 (Add RETURN_CONST instruction) # Python 3.12a5 3519 (Modify SEND instruction) +# Python 3.12a5 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) # Python 3.13 will start with 3550 @@ -445,7 +446,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3519).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3520).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index b69cd1bbdd61ca..809d24e51676bd 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -127,7 +127,6 @@ def pseudo_op(name, op, real_ops): def_op('SETUP_ANNOTATIONS', 85) -def_op('PREP_RERAISE_STAR', 88) def_op('POP_EXCEPT', 89) HAVE_ARGUMENT = 90 # real opcodes from here have an argument: @@ -224,6 +223,7 @@ def pseudo_op(name, op, real_ops): def_op('KW_NAMES', 172) hasconst.append(172) def_op('CALL_INTRINSIC_1', 173) +def_op('CALL_INTRINSIC_2', 174) hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst new file mode 100644 index 00000000000000..3233a573be7acd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst @@ -0,0 +1,2 @@ +Add :opcode:`CALL_INTRINSIC_2` and use it instead of +:opcode:`PREP_RERAISE_STAR`. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 429cd7fdafa168..be54e5f6f589eb 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -501,7 +501,14 @@ dummy_func( inst(CALL_INTRINSIC_1, (value -- res)) { assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value); - Py_DECREF(value); + DECREF_INPUTS(); + ERROR_IF(res == NULL, error); + } + + inst(CALL_INTRINSIC_2, (value2, value1 -- res)) { + assert(oparg <= MAX_INTRINSIC_2); + res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } @@ -788,15 +795,6 @@ dummy_func( goto exception_unwind; } - inst(PREP_RERAISE_STAR, (orig, excs -- val)) { - assert(PyList_Check(excs)); - - val = _PyExc_PrepReraiseStar(orig, excs); - DECREF_INPUTS(); - - ERROR_IF(val == NULL, error); - } - inst(END_ASYNC_FOR, (awaitable, exc -- )) { assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { @@ -2383,7 +2381,7 @@ dummy_func( } // Cache layout: counter/1, func_version/2, min_args/1 - // Neither CALL_INTRINSIC_1 nor CALL_FUNCTION_EX are members! + // Neither CALL_INTRINSIC_1/2 nor CALL_FUNCTION_EX are members! family(call, INLINE_CACHE_ENTRIES_CALL) = { CALL, CALL_BOUND_METHOD_EXACT_ARGS, diff --git a/Python/compile.c b/Python/compile.c index b49eda314eeef1..0534b536e3d12e 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -3431,7 +3431,7 @@ compiler_try_except(struct compiler *c, stmt_ty s) [orig, res, rest] Ln+1: LIST_APPEND 1 ) add unhandled exc to res (could be None) - [orig, res] PREP_RERAISE_STAR + [orig, res] CALL_INTRINSIC_2 PREP_RERAISE_STAR [exc] COPY 1 [exc, exc] POP_JUMP_IF_NOT_NONE RER [exc] POP_TOP @@ -3580,7 +3580,7 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) NEW_JUMP_TARGET_LABEL(c, reraise); USE_LABEL(c, reraise_star); - ADDOP(c, NO_LOCATION, PREP_RERAISE_STAR); + ADDOP_I(c, NO_LOCATION, CALL_INTRINSIC_2, INTRINSIC_PREP_RERAISE_STAR); ADDOP_I(c, NO_LOCATION, COPY, 1); ADDOP_JUMP(c, NO_LOCATION, POP_JUMP_IF_NOT_NONE, reraise); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 093ebff026b509..beb797cbd233d7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -689,6 +689,20 @@ DISPATCH(); } + TARGET(CALL_INTRINSIC_2) { + PyObject *value1 = PEEK(1); + PyObject *value2 = PEEK(2); + PyObject *res; + assert(oparg <= MAX_INTRINSIC_2); + res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); + Py_DECREF(value2); + Py_DECREF(value1); + if (res == NULL) goto pop_2_error; + STACK_SHRINK(1); + POKE(1, res); + DISPATCH(); + } + TARGET(RAISE_VARARGS) { PyObject **args = &PEEK(oparg); PyObject *cause = NULL, *exc = NULL; @@ -999,22 +1013,6 @@ goto exception_unwind; } - TARGET(PREP_RERAISE_STAR) { - PyObject *excs = PEEK(1); - PyObject *orig = PEEK(2); - PyObject *val; - assert(PyList_Check(excs)); - - val = _PyExc_PrepReraiseStar(orig, excs); - Py_DECREF(orig); - Py_DECREF(excs); - - if (val == NULL) goto pop_2_error; - STACK_SHRINK(1); - POKE(1, val); - DISPATCH(); - } - TARGET(END_ASYNC_FOR) { PyObject *exc = PEEK(1); PyObject *awaitable = PEEK(2); diff --git a/Python/intrinsics.c b/Python/intrinsics.c index ae1775862d945d..9e90ef32130f1d 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -9,6 +9,7 @@ #include "pycore_pyerrors.h" +/******** Unary functions ********/ static PyObject * no_intrinsic(PyThreadState* tstate, PyObject *unused) @@ -208,3 +209,20 @@ _PyIntrinsics_UnaryFunctions[] = { [INTRINSIC_UNARY_POSITIVE] = unary_pos, [INTRINSIC_LIST_TO_TUPLE] = list_to_tuple, }; + + +/******** Binary functions ********/ + + +static PyObject * +prep_reraise_star(PyThreadState* unused, PyObject *orig, PyObject *excs) +{ + assert(PyList_Check(excs)); + return _PyExc_PrepReraiseStar(orig, excs); +} + +instrinsic_func2 +_PyIntrinsics_BinaryFunctions[] = { + [INTRINSIC_PREP_RERAISE_STAR] = prep_reraise_star, +}; + diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index d622eb12c8cb2d..f27906a3e349eb 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -88,6 +88,8 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 2; case CALL_INTRINSIC_1: return 1; + case CALL_INTRINSIC_2: + return 2; case RAISE_VARARGS: return oparg; case INTERPRETER_EXIT: @@ -112,8 +114,6 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case RERAISE: return oparg + 1; - case PREP_RERAISE_STAR: - return 2; case END_ASYNC_FOR: return 2; case CLEANUP_THROW: @@ -440,6 +440,8 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case CALL_INTRINSIC_1: return 1; + case CALL_INTRINSIC_2: + return 1; case RAISE_VARARGS: return 0; case INTERPRETER_EXIT: @@ -464,8 +466,6 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case RERAISE: return oparg; - case PREP_RERAISE_STAR: - return 1; case END_ASYNC_FOR: return 0; case CLEANUP_THROW: @@ -760,6 +760,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [STORE_SUBSCR_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC }, [DELETE_SUBSCR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [CALL_INTRINSIC_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_INTRINSIC_2] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [RAISE_VARARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [INTERPRETER_EXIT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [RETURN_VALUE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, @@ -772,7 +773,6 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [YIELD_VALUE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [POP_EXCEPT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [RERAISE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [PREP_RERAISE_STAR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [END_ASYNC_FOR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [CLEANUP_THROW] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [LOAD_ASSERTION_ERROR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 301ec6e005dad6..bc64bd582fd572 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -87,7 +87,7 @@ static void *opcode_targets[256] = { &&TARGET_SETUP_ANNOTATIONS, &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, - &&TARGET_PREP_RERAISE_STAR, + &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_POP_EXCEPT, &&TARGET_STORE_NAME, &&TARGET_DELETE_NAME, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -142,7 +142,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_BACKWARD, &&TARGET_COMPARE_AND_BRANCH, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,20 +152,20 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_DICT, + &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, - &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, + &&TARGET_SEND_GEN, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, - &&TARGET_SEND_GEN, + &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -173,7 +173,7 @@ static void *opcode_targets[256] = { &&TARGET_CALL, &&TARGET_KW_NAMES, &&TARGET_CALL_INTRINSIC_1, - &&_unknown_opcode, + &&TARGET_CALL_INTRINSIC_2, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, From 096d0097a09e439a4564531b297a998e5d74c9b5 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Feb 2023 14:26:03 -0700 Subject: [PATCH 07/76] gh-101758: Add a Test For Single-Phase Init Module Variants (gh-101891) The new test exercises the most important variants for single-phase init extension modules. We also add some explanation about those variants to import.c. https://github.com/python/cpython/issues/101758 --- Lib/test/test_imp.py | 199 +++++++++++++++++++ Modules/_testsinglephase.c | 394 ++++++++++++++++++++++++++++++++++--- Python/import.c | 105 +++++++++- 3 files changed, 660 insertions(+), 38 deletions(-) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 80abc720c3251a..31dce21587e2ca 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -251,6 +251,205 @@ def test_issue16421_multiple_modules_in_one_dll(self): with self.assertRaises(ImportError): imp.load_dynamic('nonexistent', pathname) + @requires_load_dynamic + def test_singlephase_variants(self): + '''Exercise the most meaningful variants described in Python/import.c.''' + self.maxDiff = None + + basename = '_testsinglephase' + fileobj, pathname, _ = imp.find_module(basename) + fileobj.close() + + modules = {} + def load(name): + assert name not in modules + module = imp.load_dynamic(name, pathname) + self.assertNotIn(module, modules.values()) + modules[name] = module + return module + + def re_load(name, module): + assert sys.modules[name] is module + before = type(module)(module.__name__) + before.__dict__.update(vars(module)) + + reloaded = imp.load_dynamic(name, pathname) + + return before, reloaded + + def check_common(name, module): + summed = module.sum(1, 2) + lookedup = module.look_up_self() + initialized = module.initialized() + cached = sys.modules[name] + + # module.__name__ might not match, but the spec will. + self.assertEqual(module.__spec__.name, name) + if initialized is not None: + self.assertIsInstance(initialized, float) + self.assertGreater(initialized, 0) + self.assertEqual(summed, 3) + self.assertTrue(issubclass(module.error, Exception)) + self.assertEqual(module.int_const, 1969) + self.assertEqual(module.str_const, 'something different') + self.assertIs(cached, module) + + return lookedup, initialized, cached + + def check_direct(name, module, lookedup): + # The module has its own PyModuleDef, with a matching name. + self.assertEqual(module.__name__, name) + self.assertIs(lookedup, module) + + def check_indirect(name, module, lookedup, orig): + # The module re-uses another's PyModuleDef, with a different name. + assert orig is not module + assert orig.__name__ != name + self.assertNotEqual(module.__name__, name) + self.assertIs(lookedup, module) + + def check_basic(module, initialized): + init_count = module.initialized_count() + + self.assertIsNot(initialized, None) + self.assertIsInstance(init_count, int) + self.assertGreater(init_count, 0) + + return init_count + + def check_common_reloaded(name, module, cached, before, reloaded): + recached = sys.modules[name] + + self.assertEqual(reloaded.__spec__.name, name) + self.assertEqual(reloaded.__name__, before.__name__) + self.assertEqual(before.__dict__, module.__dict__) + self.assertIs(recached, reloaded) + + def check_basic_reloaded(module, lookedup, initialized, init_count, + before, reloaded): + relookedup = reloaded.look_up_self() + reinitialized = reloaded.initialized() + reinit_count = reloaded.initialized_count() + + self.assertIs(reloaded, module) + self.assertIs(reloaded.__dict__, module.__dict__) + # It only happens to be the same but that's good enough here. + # We really just want to verify that the re-loaded attrs + # didn't change. + self.assertIs(relookedup, lookedup) + self.assertEqual(reinitialized, initialized) + self.assertEqual(reinit_count, init_count) + + def check_with_reinit_reloaded(module, lookedup, initialized, + before, reloaded): + relookedup = reloaded.look_up_self() + reinitialized = reloaded.initialized() + + self.assertIsNot(reloaded, module) + self.assertIsNot(reloaded, module) + self.assertNotEqual(reloaded.__dict__, module.__dict__) + self.assertIs(relookedup, reloaded) + if initialized is None: + self.assertIs(reinitialized, None) + else: + self.assertGreater(reinitialized, initialized) + + # Check the "basic" module. + + name = basename + expected_init_count = 1 + with self.subTest(name): + mod = load(name) + lookedup, initialized, cached = check_common(name, mod) + check_direct(name, mod, lookedup) + init_count = check_basic(mod, initialized) + self.assertEqual(init_count, expected_init_count) + + before, reloaded = re_load(name, mod) + check_common_reloaded(name, mod, cached, before, reloaded) + check_basic_reloaded(mod, lookedup, initialized, init_count, + before, reloaded) + basic = mod + + # Check its indirect variants. + + name = f'{basename}_basic_wrapper' + expected_init_count += 1 + with self.subTest(name): + mod = load(name) + lookedup, initialized, cached = check_common(name, mod) + check_indirect(name, mod, lookedup, basic) + init_count = check_basic(mod, initialized) + self.assertEqual(init_count, expected_init_count) + + before, reloaded = re_load(name, mod) + check_common_reloaded(name, mod, cached, before, reloaded) + check_basic_reloaded(mod, lookedup, initialized, init_count, + before, reloaded) + + # Currently _PyState_AddModule() always replaces the cached module. + self.assertIs(basic.look_up_self(), mod) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # The cached module shouldn't be changed after this point. + basic_lookedup = mod + + # Check its direct variant. + + name = f'{basename}_basic_copy' + expected_init_count += 1 + with self.subTest(name): + mod = load(name) + lookedup, initialized, cached = check_common(name, mod) + check_direct(name, mod, lookedup) + init_count = check_basic(mod, initialized) + self.assertEqual(init_count, expected_init_count) + + before, reloaded = re_load(name, mod) + check_common_reloaded(name, mod, cached, before, reloaded) + check_basic_reloaded(mod, lookedup, initialized, init_count, + before, reloaded) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # Check the non-basic variant that has no state. + + name = f'{basename}_with_reinit' + with self.subTest(name): + mod = load(name) + lookedup, initialized, cached = check_common(name, mod) + self.assertIs(initialized, None) + check_direct(name, mod, lookedup) + + before, reloaded = re_load(name, mod) + check_common_reloaded(name, mod, cached, before, reloaded) + check_with_reinit_reloaded(mod, lookedup, initialized, + before, reloaded) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # Check the basic variant that has state. + + name = f'{basename}_with_state' + with self.subTest(name): + mod = load(name) + lookedup, initialized, cached = check_common(name, mod) + self.assertIsNot(initialized, None) + check_direct(name, mod, lookedup) + + before, reloaded = re_load(name, mod) + check_common_reloaded(name, mod, cached, before, reloaded) + check_with_reinit_reloaded(mod, lookedup, initialized, + before, reloaded) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + @requires_load_dynamic def test_load_dynamic_ImportError_path(self): # Issue #1559549 added `name` and `path` attributes to ImportError diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index 3bfe159e54fe49..9e8dd647ee761a 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -5,74 +5,412 @@ # define Py_BUILD_CORE_MODULE 1 #endif +//#include #include "Python.h" #include "pycore_namespace.h" // _PyNamespace_New() +typedef struct { + _PyTime_t initialized; + PyObject *error; + PyObject *int_const; + PyObject *str_const; +} module_state; + +/* Process-global state is only used by _testsinglephase + since it's the only one that does not support re-init. */ +static struct { + int initialized_count; + module_state module; +} global_state = { .initialized_count = -1 }; + +static inline module_state * +get_module_state(PyObject *module) +{ + PyModuleDef *def = PyModule_GetDef(module); + if (def->m_size == -1) { + return &global_state.module; + } + else if (def->m_size == 0) { + return NULL; + } + else { + module_state *state = (module_state*)PyModule_GetState(module); + assert(state != NULL); + return state; + } +} + +static void +clear_state(module_state *state) +{ + state->initialized = 0; + Py_CLEAR(state->error); + Py_CLEAR(state->int_const); + Py_CLEAR(state->str_const); +} + +static int +_set_initialized(_PyTime_t *initialized) +{ + /* We go strictly monotonic to ensure each time is unique. */ + _PyTime_t prev; + if (_PyTime_GetMonotonicClockWithInfo(&prev, NULL) != 0) { + return -1; + } + /* We do a busy sleep since the interval should be super short. */ + _PyTime_t t; + do { + if (_PyTime_GetMonotonicClockWithInfo(&t, NULL) != 0) { + return -1; + } + } while (t == prev); + + *initialized = t; + return 0; +} + +static int +init_state(module_state *state) +{ + assert(state->initialized == 0 && + state->error == NULL && + state->int_const == NULL && + state->str_const == NULL); + + if (_set_initialized(&state->initialized) != 0) { + goto error; + } + assert(state->initialized > 0); + + /* Add an exception type */ + state->error = PyErr_NewException("_testsinglephase.error", NULL, NULL); + if (state->error == NULL) { + goto error; + } + + state->int_const = PyLong_FromLong(1969); + if (state->int_const == NULL) { + goto error; + } + + state->str_const = PyUnicode_FromString("something different"); + if (state->str_const == NULL) { + goto error; + } + + return 0; + +error: + clear_state(state); + return -1; +} + +static int +init_module(PyObject *module, module_state *state) +{ + if (PyModule_AddObjectRef(module, "error", state->error) != 0) { + return -1; + } + if (PyModule_AddObjectRef(module, "int_const", state->int_const) != 0) { + return -1; + } + if (PyModule_AddObjectRef(module, "str_const", state->str_const) != 0) { + return -1; + } + return 0; +} + + +PyDoc_STRVAR(common_initialized_doc, +"initialized()\n\ +\n\ +Return the seconds-since-epoch when the module was initialized."); + +static PyObject * +common_initialized(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + module_state *state = get_module_state(self); + if (state == NULL) { + Py_RETURN_NONE; + } + double d = _PyTime_AsSecondsDouble(state->initialized); + return PyFloat_FromDouble(d); +} + +#define INITIALIZED_METHODDEF \ + {"initialized", common_initialized, METH_NOARGS, \ + common_initialized_doc} + + +PyDoc_STRVAR(common_look_up_self_doc, +"look_up_self()\n\ +\n\ +Return the module associated with this module's def.m_base.m_index."); + +static PyObject * +common_look_up_self(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyModuleDef *def = PyModule_GetDef(self); + if (def == NULL) { + return NULL; + } + return Py_NewRef( + PyState_FindModule(def)); +} + +#define LOOK_UP_SELF_METHODDEF \ + {"look_up_self", common_look_up_self, METH_NOARGS, common_look_up_self_doc} + + /* Function of two integers returning integer */ -PyDoc_STRVAR(testexport_foo_doc, -"foo(i,j)\n\ +PyDoc_STRVAR(common_sum_doc, +"sum(i,j)\n\ \n\ Return the sum of i and j."); static PyObject * -testexport_foo(PyObject *self, PyObject *args) +common_sum(PyObject *self, PyObject *args) { long i, j; long res; - if (!PyArg_ParseTuple(args, "ll:foo", &i, &j)) + if (!PyArg_ParseTuple(args, "ll:sum", &i, &j)) return NULL; res = i + j; return PyLong_FromLong(res); } +#define SUM_METHODDEF \ + {"sum", common_sum, METH_VARARGS, common_sum_doc} -static PyMethodDef TestMethods[] = { - {"foo", testexport_foo, METH_VARARGS, - testexport_foo_doc}, - {NULL, NULL} /* sentinel */ -}; +PyDoc_STRVAR(basic_initialized_count_doc, +"initialized_count()\n\ +\n\ +Return how many times the module has been initialized."); + +static PyObject * +basic_initialized_count(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + assert(PyModule_GetDef(self)->m_size == -1); + return PyLong_FromLong(global_state.initialized_count); +} + +#define INITIALIZED_COUNT_METHODDEF \ + {"initialized_count", basic_initialized_count, METH_VARARGS, \ + basic_initialized_count_doc} + + +/*********************************************/ +/* the _testsinglephase module (and aliases) */ +/*********************************************/ + +/* This ia more typical of legacy extensions in the wild: + - single-phase init + - no module state + - does not support repeated initialization + (so m_copy is used) + - the module def is cached in _PyRuntime.extensions + (by name/filename) + + Also note that, because the module has single-phase init, + it is cached in interp->module_by_index (using mod->md_def->m_base.m_index). + */ -static struct PyModuleDef _testsinglephase = { +static PyMethodDef TestMethods_Basic[] = { + LOOK_UP_SELF_METHODDEF, + SUM_METHODDEF, + INITIALIZED_METHODDEF, + INITIALIZED_COUNT_METHODDEF, + {NULL, NULL} /* sentinel */ +}; + +static struct PyModuleDef _testsinglephase_basic = { PyModuleDef_HEAD_INIT, .m_name = "_testsinglephase", - .m_doc = PyDoc_STR("Test module _testsinglephase (main)"), + .m_doc = PyDoc_STR("Test module _testsinglephase"), .m_size = -1, // no module state - .m_methods = TestMethods, + .m_methods = TestMethods_Basic, }; +static PyObject * +init__testsinglephase_basic(PyModuleDef *def) +{ + if (global_state.initialized_count == -1) { + global_state.initialized_count = 0; + } + + PyObject *module = PyModule_Create(def); + if (module == NULL) { + return NULL; + } + + module_state *state = &global_state.module; + // It may have been set by a previous run or under a different name. + clear_state(state); + if (init_state(state) < 0) { + Py_CLEAR(module); + return NULL; + } + + if (init_module(module, state) < 0) { + Py_CLEAR(module); + goto finally; + } + + global_state.initialized_count++; + +finally: + return module; +} PyMODINIT_FUNC PyInit__testsinglephase(void) { - PyObject *module = PyModule_Create(&_testsinglephase); + return init__testsinglephase_basic(&_testsinglephase_basic); +} + + +PyMODINIT_FUNC +PyInit__testsinglephase_basic_wrapper(void) +{ + return PyInit__testsinglephase(); +} + + +PyMODINIT_FUNC +PyInit__testsinglephase_basic_copy(void) +{ + static struct PyModuleDef def = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_basic_copy", + .m_doc = PyDoc_STR("Test module _testsinglephase_basic_copy"), + .m_size = -1, // no module state + .m_methods = TestMethods_Basic, + }; + return init__testsinglephase_basic(&def); +} + + +/*******************************************/ +/* the _testsinglephase_with_reinit module */ +/*******************************************/ + +/* This ia less typical of legacy extensions in the wild: + - single-phase init (same as _testsinglephase above) + - no module state + - supports repeated initialization + (so m_copy is not used) + - the module def is not cached in _PyRuntime.extensions + + At this point most modules would reach for multi-phase init (PEP 489). + However, module state has been around a while (PEP 3121), + and most extensions predate multi-phase init. + + (This module is basically the same as _testsinglephase, + but supports repeated initialization.) + */ + +static PyMethodDef TestMethods_Reinit[] = { + LOOK_UP_SELF_METHODDEF, + SUM_METHODDEF, + INITIALIZED_METHODDEF, + {NULL, NULL} /* sentinel */ +}; + +static struct PyModuleDef _testsinglephase_with_reinit = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_with_reinit", + .m_doc = PyDoc_STR("Test module _testsinglephase_with_reinit"), + .m_size = 0, + .m_methods = TestMethods_Reinit, +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_with_reinit(void) +{ + /* We purposefully do not try PyState_FindModule() first here + since we want to check the behavior of re-loading the module. */ + PyObject *module = PyModule_Create(&_testsinglephase_with_reinit); if (module == NULL) { return NULL; } - /* Add an exception type */ - PyObject *temp = PyErr_NewException("_testsinglephase.error", NULL, NULL); - if (temp == NULL) { - goto error; + assert(get_module_state(module) == NULL); + + module_state state = {0}; + if (init_state(&state) < 0) { + Py_CLEAR(module); + return NULL; } - if (PyModule_AddObject(module, "error", temp) != 0) { - Py_DECREF(temp); - goto error; + + if (init_module(module, &state) < 0) { + Py_CLEAR(module); + goto finally; } - if (PyModule_AddIntConstant(module, "int_const", 1969) != 0) { - goto error; +finally: + /* We only needed the module state for setting the module attrs. */ + clear_state(&state); + return module; +} + + +/******************************************/ +/* the _testsinglephase_with_state module */ +/******************************************/ + +/* This ia less typical of legacy extensions in the wild: + - single-phase init (same as _testsinglephase above) + - has some module state + - supports repeated initialization + (so m_copy is not used) + - the module def is not cached in _PyRuntime.extensions + + At this point most modules would reach for multi-phase init (PEP 489). + However, module state has been around a while (PEP 3121), + and most extensions predate multi-phase init. + */ + +static PyMethodDef TestMethods_WithState[] = { + LOOK_UP_SELF_METHODDEF, + SUM_METHODDEF, + INITIALIZED_METHODDEF, + {NULL, NULL} /* sentinel */ +}; + +static struct PyModuleDef _testsinglephase_with_state = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_with_state", + .m_doc = PyDoc_STR("Test module _testsinglephase_with_state"), + .m_size = sizeof(module_state), + .m_methods = TestMethods_WithState, +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_with_state(void) +{ + /* We purposefully do not try PyState_FindModule() first here + since we want to check the behavior of re-loading the module. */ + PyObject *module = PyModule_Create(&_testsinglephase_with_state); + if (module == NULL) { + return NULL; } - if (PyModule_AddStringConstant(module, "str_const", "something different") != 0) { - goto error; + module_state *state = get_module_state(module); + assert(state != NULL); + if (init_state(state) < 0) { + Py_CLEAR(module); + return NULL; } - return module; + if (init_module(module, state) < 0) { + clear_state(state); + Py_CLEAR(module); + goto finally; + } -error: - Py_DECREF(module); - return NULL; +finally: + return module; } diff --git a/Python/import.c b/Python/import.c index 302255d76edcd5..63ed2443657b29 100644 --- a/Python/import.c +++ b/Python/import.c @@ -428,6 +428,71 @@ PyImport_GetMagicTag(void) } +/* +We support a number of kinds of single-phase init builtin/extension modules: + +* "basic" + * no module state (PyModuleDef.m_size == -1) + * does not support repeated init (we use PyModuleDef.m_base.m_copy) + * may have process-global state + * the module's def is cached in _PyRuntime.imports.extensions, + by (name, filename) +* "reinit" + * no module state (PyModuleDef.m_size == 0) + * supports repeated init (m_copy is never used) + * should not have any process-global state + * its def is never cached in _PyRuntime.imports.extensions + (except, currently, under the main interpreter, for some reason) +* "with state" (almost the same as reinit) + * has module state (PyModuleDef.m_size > 0) + * supports repeated init (m_copy is never used) + * should not have any process-global state + * its def is never cached in _PyRuntime.imports.extensions + (except, currently, under the main interpreter, for some reason) + +There are also variants within those classes: + +* two or more modules share a PyModuleDef + * a module's init func uses another module's PyModuleDef + * a module's init func calls another's module's init func + * a module's init "func" is actually a variable statically initialized + to another module's init func +* two or modules share "methods" + * a module's init func copies another module's PyModuleDef + (with a different name) +* (basic-only) two or modules share process-global state + +In the first case, where modules share a PyModuleDef, the following +notable weirdness happens: + +* the module's __name__ matches the def, not the requested name +* the last module (with the same def) to be imported for the first time wins + * returned by PyState_Find_Module() (via interp->modules_by_index) + * (non-basic-only) its init func is used when re-loading any of them + (via the def's m_init) + * (basic-only) the copy of its __dict__ is used when re-loading any of them + (via the def's m_copy) + +However, the following happens as expected: + +* a new module object (with its own __dict__) is created for each request +* the module's __spec__ has the requested name +* the loaded module is cached in sys.modules under the requested name +* the m_index field of the shared def is not changed, + so at least PyState_FindModule() will always look in the same place + +For "basic" modules there are other quirks: + +* (whether sharing a def or not) when loaded the first time, + m_copy is set before _init_module_attrs() is called + in importlib._bootstrap.module_from_spec(), + so when the module is re-loaded, the previous value + for __wpec__ (and others) is reset, possibly unexpectedly. + +Generally, when multiple interpreters are involved, some of the above +gets even messier. +*/ + /* Magic for extension modules (built-in as well as dynamically loaded). To prevent initializing an extension module more than once, we keep a static dictionary 'extensions' keyed by the tuple @@ -489,9 +554,8 @@ _extensions_cache_clear(void) Py_CLEAR(_PyRuntime.imports.extensions); } -int -_PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, - PyObject *filename, PyObject *modules) +static int +fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) { if (mod == NULL || !PyModule_Check(mod)) { PyErr_BadInternalCall(); @@ -505,16 +569,13 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, } PyThreadState *tstate = _PyThreadState_GET(); - if (PyObject_SetItem(modules, name, mod) < 0) { - return -1; - } if (_PyState_AddModule(tstate, mod, def) < 0) { - PyMapping_DelItem(modules, name); return -1; } // bpo-44050: Extensions and def->m_base.m_copy can be updated // when the extension module doesn't support sub-interpreters. + // XXX Why special-case the main interpreter? if (_Py_IsMainInterpreter(tstate->interp) || def->m_size == -1) { if (def->m_size == -1) { if (def->m_base.m_copy) { @@ -541,15 +602,39 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, return 0; } +int +_PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, + PyObject *filename, PyObject *modules) +{ + if (PyObject_SetItem(modules, name, mod) < 0) { + return -1; + } + if (fix_up_extension(mod, name, filename) < 0) { + PyMapping_DelItem(modules, name); + return -1; + } + return 0; +} + int _PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules) { - int res; + int res = -1; PyObject *nameobj; nameobj = PyUnicode_InternFromString(name); - if (nameobj == NULL) + if (nameobj == NULL) { return -1; - res = _PyImport_FixupExtensionObject(mod, nameobj, nameobj, modules); + } + if (PyObject_SetItem(modules, nameobj, mod) < 0) { + goto finally; + } + if (fix_up_extension(mod, nameobj, nameobj) < 0) { + PyMapping_DelItem(modules, nameobj); + goto finally; + } + res = 0; + +finally: Py_DECREF(nameobj); return res; } From d777790bab878b8d1a218a1a60894b2823485cca Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Tue, 14 Feb 2023 15:57:01 -0800 Subject: [PATCH 08/76] gh-99108: Build the hashlib HACL* code as a static library. (#101917) This builds HACL* as a library in one place. A followup to #101707 which broke some WASM builds. This fixes 2/4 of them, but the enscripten toolchain in the others don't deduplicate linker arguments and error out. A follow-on PR will address those. --- Makefile.pre.in | 33 +++++++++++++++++-- Modules/Setup.stdlib.in | 4 +-- .../{include => }/python_hacl_namespaces.h | 1 + 3 files changed, 34 insertions(+), 4 deletions(-) rename Modules/_hacl/{include => }/python_hacl_namespaces.h (97%) diff --git a/Makefile.pre.in b/Makefile.pre.in index d42d4d8a3c1c9f..ce3fed3d648536 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -207,6 +207,7 @@ ENSUREPIP= @ENSUREPIP@ # Internal static libraries LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a LIBEXPAT_A= Modules/expat/libexpat.a +LIBHACL_A= Modules/_hacl/libHacl_Streaming_SHA2.a # Module state, compiler flags and linker flags # Empty CFLAGS and LDFLAGS are omitted. @@ -571,6 +572,23 @@ LIBEXPAT_HEADERS= \ Modules/expat/xmltok.h \ Modules/expat/xmltok_impl.h +########################################################################## +# hashlib's HACL* library + +LIBHACL_OBJS= \ + Modules/_hacl/Hacl_Streaming_SHA2.o + +LIBHACL_HEADERS= \ + Modules/_hacl/Hacl_Streaming_SHA2.h \ + Modules/_hacl/include/krml/FStar_UInt128_Verified.h \ + Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h \ + Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h \ + Modules/_hacl/include/krml/internal/target.h \ + Modules/_hacl/include/krml/lowstar_endianness.h \ + Modules/_hacl/include/krml/types.h \ + Modules/_hacl/internal/Hacl_SHA2_Generic.h \ + Modules/_hacl/python_hacl_namespaces.h + ######################################################################### # Rules @@ -890,6 +908,17 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS) -rm -f $@ $(AR) $(ARFLAGS) $@ $(LIBEXPAT_OBJS) +########################################################################## +# Build HACL* static libraries for hashlib: libHacl_Streaming_SHA2.a +LIBHACL_CFLAGS=-I$(srcdir)/Modules/_hacl/include -D_BSD_SOURCE -D_DEFAULT_SOURCE $(PY_STDMODULE_CFLAGS) $(CCSHARED) + +Modules/_hacl/Hacl_Streaming_SHA2.o: $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.c $(LIBHACL_HEADERS) + $(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.c + +$(LIBHACL_A): $(LIBHACL_OBJS) + -rm -f $@ + $(AR) $(ARFLAGS) $@ $(LIBHACL_OBJS) + # create relative links from build/lib.platform/egg.so to Modules/egg.so # pybuilddir.txt is created too late. We cannot use it in Makefile # targets. ln --relative is not portable. @@ -2606,9 +2635,9 @@ MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h -MODULE__SHA256_DEPS=$(srcdir)/Modules/hashlib.h $(srcdir)/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h $(srcdir)/Modules/_hacl/include/krml/lowstar_endianness.h $(srcdir)/Modules/_hacl/include/krml/internal/target.h $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.h +MODULE__SHA256_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) $(LIBHACL_A) MODULE__SHA3_DEPS=$(srcdir)/Modules/_sha3/sha3.c $(srcdir)/Modules/_sha3/sha3.h $(srcdir)/Modules/hashlib.h -MODULE__SHA512_DEPS=$(srcdir)/Modules/hashlib.h $(srcdir)/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h $(srcdir)/Modules/_hacl/include/krml/lowstar_endianness.h $(srcdir)/Modules/_hacl/include/krml/internal/target.h $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.h +MODULE__SHA512_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) $(LIBHACL_A) MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data.h $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index b6d13e04d3fa87..22bcb423db233f 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -79,8 +79,8 @@ # hashing builtins, can be disabled with --without-builtin-hashlib-hashes @MODULE__MD5_TRUE@_md5 md5module.c @MODULE__SHA1_TRUE@_sha1 sha1module.c -@MODULE__SHA256_TRUE@_sha256 sha256module.c _hacl/Hacl_Streaming_SHA2.c -@MODULE__SHA512_TRUE@_sha512 sha512module.c _hacl/Hacl_Streaming_SHA2.c +@MODULE__SHA256_TRUE@_sha256 sha256module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a +@MODULE__SHA512_TRUE@_sha512 sha512module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a @MODULE__SHA3_TRUE@_sha3 _sha3/sha3module.c @MODULE__BLAKE2_TRUE@_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c diff --git a/Modules/_hacl/include/python_hacl_namespaces.h b/Modules/_hacl/python_hacl_namespaces.h similarity index 97% rename from Modules/_hacl/include/python_hacl_namespaces.h rename to Modules/_hacl/python_hacl_namespaces.h index 65608d1fd283c4..ac12f386257b19 100644 --- a/Modules/_hacl/include/python_hacl_namespaces.h +++ b/Modules/_hacl/python_hacl_namespaces.h @@ -25,6 +25,7 @@ #define Hacl_Streaming_SHA2_init_224 python_hashlib_Hacl_Streaming_SHA2_init_224 #define Hacl_Streaming_SHA2_init_512 python_hashlib_Hacl_Streaming_SHA2_init_512 #define Hacl_Streaming_SHA2_init_384 python_hashlib_Hacl_Streaming_SHA2_init_384 +#define Hacl_SHA2_Scalar32_sha512_init python_hashlib_Hacl_SHA2_Scalar32_sha512_init #define Hacl_Streaming_SHA2_update_256 python_hashlib_Hacl_Streaming_SHA2_update_256 #define Hacl_Streaming_SHA2_update_224 python_hashlib_Hacl_Streaming_SHA2_update_224 #define Hacl_Streaming_SHA2_update_512 python_hashlib_Hacl_Streaming_SHA2_update_512 From 8a2b7ee64d1bde762438b458ea7fe88f054a3a88 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 15 Feb 2023 06:27:16 +0100 Subject: [PATCH 09/76] gh-101693: In sqlite3, deprecate using named placeholders with parameters supplied as a sequence (#101698) --- Doc/library/sqlite3.rst | 19 ++++++++++++++++++- Doc/whatsnew/3.12.rst | 7 +++++++ Lib/test/test_sqlite3/test_dbapi.py | 15 +++++++++++++++ ...-02-08-18-20-58.gh-issue-101693.4_LPXj.rst | 6 ++++++ Modules/_sqlite/cursor.c | 13 +++++++++++++ 5 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 8ffc0aad91995c..18d0a5e630f6a9 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1442,6 +1442,14 @@ Cursor objects and there is no open transaction, a transaction is implicitly opened before executing *sql*. + .. deprecated-removed:: 3.12 3.14 + + :exc:`DeprecationWarning` is emitted if + :ref:`named placeholders ` are used + and *parameters* is a sequence instead of a :class:`dict`. + Starting with Python 3.14, :exc:`ProgrammingError` will + be raised instead. + Use :meth:`executescript` to execute multiple SQL statements. .. method:: executemany(sql, parameters, /) @@ -1476,6 +1484,15 @@ Cursor objects # cur is an sqlite3.Cursor object cur.executemany("INSERT INTO data VALUES(?)", rows) + .. deprecated-removed:: 3.12 3.14 + + :exc:`DeprecationWarning` is emitted if + :ref:`named placeholders ` are used + and the items in *parameters* are sequences + instead of :class:`dict`\s. + Starting with Python 3.14, :exc:`ProgrammingError` will + be raised instead. + .. method:: executescript(sql_script, /) Execute the SQL statements in *sql_script*. @@ -1971,7 +1988,7 @@ question marks (qmark style) or named placeholders (named style). For the qmark style, *parameters* must be a :term:`sequence` whose length must match the number of placeholders, or a :exc:`ProgrammingError` is raised. -For the named style, *parameters* should be +For the named style, *parameters* must be an instance of a :class:`dict` (or a subclass), which must contain keys for all named parameters; any extra items are ignored. diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 45a5e5062d55b6..c62f462a19a2df 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -415,6 +415,13 @@ Deprecated and tailor them to your needs. (Contributed by Erlend E. Aasland in :gh:`90016`.) +* In :meth:`~sqlite3.Cursor.execute`, :exc:`DeprecationWarning` is now emitted + when :ref:`named placeholders ` are used together with + parameters supplied as a :term:`sequence` instead of as a :class:`dict`. + Starting from Python 3.14, using named placeholders with parameters supplied + as a sequence will raise a :exc:`~sqlite3.ProgrammingError`. + (Contributed by Erlend E. Aasland in :gh:`101698`.) + * The 3-arg signatures (type, value, traceback) of :meth:`~coroutine.throw`, :meth:`~generator.throw` and :meth:`~agen.athrow` are deprecated and may be removed in a future version of Python. Use the single-arg versions diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 363a308f3e5fec..695e213cdc7b75 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -861,6 +861,21 @@ def __getitem__(slf, x): with self.assertRaises(ZeroDivisionError): self.cu.execute("select name from test where name=?", L()) + def test_execute_named_param_and_sequence(self): + dataset = ( + ("select :a", (1,)), + ("select :a, ?, ?", (1, 2, 3)), + ("select ?, :b, ?", (1, 2, 3)), + ("select ?, ?, :c", (1, 2, 3)), + ("select :a, :b, ?", (1, 2, 3)), + ) + msg = "Binding.*is a named parameter" + for query, params in dataset: + with self.subTest(query=query, params=params): + with self.assertWarnsRegex(DeprecationWarning, msg) as cm: + self.cu.execute(query, params) + self.assertEqual(cm.filename, __file__) + def test_execute_too_many_params(self): category = sqlite.SQLITE_LIMIT_VARIABLE_NUMBER msg = "too many SQL variables" diff --git a/Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst b/Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst new file mode 100644 index 00000000000000..e436054b15b657 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst @@ -0,0 +1,6 @@ +In :meth:`sqlite3.Cursor.execute`, :exc:`DeprecationWarning` is now emitted +when :ref:`named placeholders ` are used together with +parameters supplied as a :term:`sequence` instead of as a :class:`dict`. +Starting from Python 3.14, using named placeholders with parameters supplied +as a sequence will raise a :exc:`~sqlite3.ProgrammingError`. +Patch by Erlend E. Aasland. diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index a4e22bb4a2b58d..6f7970cf8197a2 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -662,6 +662,19 @@ bind_parameters(pysqlite_state *state, pysqlite_Statement *self, return; } for (i = 0; i < num_params; i++) { + const char *name = sqlite3_bind_parameter_name(self->st, i+1); + if (name != NULL) { + int ret = PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "Binding %d ('%s') is a named parameter, but you " + "supplied a sequence which requires nameless (qmark) " + "placeholders. Starting with Python 3.14 an " + "sqlite3.ProgrammingError will be raised.", + i+1, name); + if (ret < 0) { + return; + } + } + if (PyTuple_CheckExact(parameters)) { PyObject *item = PyTuple_GET_ITEM(parameters, i); current_param = Py_NewRef(item); From e8b6aaad2faf11fe315410138a5c5943d610d8d8 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 15 Feb 2023 11:18:27 +0100 Subject: [PATCH 10/76] gh-101819: Remove _testcapi dependencies on specific _io symbols (#101918) --- Modules/_io/_iomodule.c | 12 ++---------- Modules/_testcapimodule.c | 8 ++++++-- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 175fa97479d27d..811b1d221a0122 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -720,16 +720,8 @@ PyInit__io(void) // Add types for (size_t i=0; i < Py_ARRAY_LENGTH(static_types); i++) { PyTypeObject *type = static_types[i]; - // Private type not exposed in the _io module - if (type == &_PyBytesIOBuffer_Type) { - if (PyType_Ready(type) < 0) { - goto fail; - } - } - else { - if (PyModule_AddType(m, type) < 0) { - goto fail; - } + if (PyModule_AddType(m, type) < 0) { + goto fail; } } diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 3c411fa0d76358..5610a7689136f6 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1448,12 +1448,10 @@ test_from_contiguous(PyObject* self, PyObject *Py_UNUSED(ignored)) } #if (defined(__linux__) || defined(__FreeBSD__)) && defined(__GNUC__) -extern PyTypeObject _PyBytesIOBuffer_Type; static PyObject * test_pep3118_obsolete_write_locks(PyObject* self, PyObject *Py_UNUSED(ignored)) { - PyTypeObject *type = &_PyBytesIOBuffer_Type; PyObject *b; char *dummy[1]; int ret, match; @@ -1466,7 +1464,13 @@ test_pep3118_obsolete_write_locks(PyObject* self, PyObject *Py_UNUSED(ignored)) goto error; /* bytesiobuf_getbuffer() */ + PyTypeObject *type = (PyTypeObject *)_PyImport_GetModuleAttrString( + "_io", "_BytesIOBuffer"); + if (type == NULL) { + return NULL; + } b = type->tp_alloc(type, 0); + Py_DECREF(type); if (b == NULL) { return NULL; } From c7766245c14fa03b8afd3aff9be30b13d0069f95 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 15 Feb 2023 12:21:40 +0000 Subject: [PATCH 11/76] GH-87849: Fix refleak in SEND instruction. (GH-101908) Fix refleak in SEND instruction. --- Python/bytecodes.c | 1 + Python/generated_cases.c.h | 1 + 2 files changed, 2 insertions(+) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index be54e5f6f589eb..d5d5034cbfbf74 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -727,6 +727,7 @@ dummy_func( else { assert(retval != NULL); } + Py_DECREF(v); } inst(SEND_GEN, (unused/1, receiver, v -- receiver)) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index beb797cbd233d7..8b8a7161ad898e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -934,6 +934,7 @@ else { assert(retval != NULL); } + Py_DECREF(v); POKE(1, retval); JUMPBY(1); DISPATCH(); From eb0c485b6c836abb71932537a5058344d11d7bc8 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 15 Feb 2023 14:07:59 +0100 Subject: [PATCH 12/76] gh-101819: Remove _PyWindowsConsoleIO_Type from the Windows DLL (GH-101904) Automerge-Triggered-By: GH:erlend-aasland --- .../pycore_global_objects_fini_generated.h | 2 ++ Include/internal/pycore_global_strings.h | 2 ++ Include/internal/pycore_runtime_init_generated.h | 2 ++ Include/internal/pycore_unicodeobject_generated.h | 4 ++++ Modules/_io/_iomodule.h | 4 ---- Modules/_io/winconsoleio.c | 4 +--- PC/_testconsole.c | 11 +++++++++-- Python/pylifecycle.c | 14 +++++++++----- 8 files changed, 29 insertions(+), 14 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 8c210111b5899f..dc5cd58d853534 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -577,6 +577,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(True)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(WarningMessage)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_WindowsConsoleIO)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__IOBase_closed)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__abc_tpflags__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__abs__)); @@ -752,6 +753,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_get_sourcefile)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_handle_fromlist)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_initializing)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_io)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_is_text_encoding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_length_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_limbo)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 6b1c8424424d96..8b23aa15479301 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -63,6 +63,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(True) STRUCT_FOR_ID(WarningMessage) STRUCT_FOR_ID(_) + STRUCT_FOR_ID(_WindowsConsoleIO) STRUCT_FOR_ID(__IOBase_closed) STRUCT_FOR_ID(__abc_tpflags__) STRUCT_FOR_ID(__abs__) @@ -238,6 +239,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_get_sourcefile) STRUCT_FOR_ID(_handle_fromlist) STRUCT_FOR_ID(_initializing) + STRUCT_FOR_ID(_io) STRUCT_FOR_ID(_is_text_encoding) STRUCT_FOR_ID(_length_) STRUCT_FOR_ID(_limbo) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index fcb613083ffe99..471efadb13bb4f 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -569,6 +569,7 @@ extern "C" { INIT_ID(True), \ INIT_ID(WarningMessage), \ INIT_ID(_), \ + INIT_ID(_WindowsConsoleIO), \ INIT_ID(__IOBase_closed), \ INIT_ID(__abc_tpflags__), \ INIT_ID(__abs__), \ @@ -744,6 +745,7 @@ extern "C" { INIT_ID(_get_sourcefile), \ INIT_ID(_handle_fromlist), \ INIT_ID(_initializing), \ + INIT_ID(_io), \ INIT_ID(_is_text_encoding), \ INIT_ID(_length_), \ INIT_ID(_limbo), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 301aee5210e799..b47d240e492ff9 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -32,6 +32,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(_); PyUnicode_InternInPlace(&string); + string = &_Py_ID(_WindowsConsoleIO); + PyUnicode_InternInPlace(&string); string = &_Py_ID(__IOBase_closed); PyUnicode_InternInPlace(&string); string = &_Py_ID(__abc_tpflags__); @@ -382,6 +384,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(_initializing); PyUnicode_InternInPlace(&string); + string = &_Py_ID(_io); + PyUnicode_InternInPlace(&string); string = &_Py_ID(_is_text_encoding); PyUnicode_InternInPlace(&string); string = &_Py_ID(_length_); diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index c260080f0e348b..7617cb8fb70e43 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -21,13 +21,9 @@ extern PyTypeObject PyBufferedRandom_Type; extern PyTypeObject PyTextIOWrapper_Type; extern PyTypeObject PyIncrementalNewlineDecoder_Type; -#ifndef Py_LIMITED_API #ifdef MS_WINDOWS extern PyTypeObject PyWindowsConsoleIO_Type; -PyAPI_DATA(PyObject *) _PyWindowsConsoleIO_Type; -#define PyWindowsConsoleIO_Check(op) (PyObject_TypeCheck((op), (PyTypeObject*)_PyWindowsConsoleIO_Type)) #endif /* MS_WINDOWS */ -#endif /* Py_LIMITED_API */ /* These functions are used as METH_NOARGS methods, are normally called * with args=NULL, and return a new reference. diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 4f41ab965e2e67..e913d831874617 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -260,7 +260,7 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, int fd_is_own = 0; HANDLE handle = NULL; - assert(PyWindowsConsoleIO_Check(self)); + assert(PyObject_TypeCheck(self, (PyTypeObject *)&PyWindowsConsoleIO_Type)); if (self->fd >= 0) { if (self->closefd) { /* Have to close the existing file first. */ @@ -1174,6 +1174,4 @@ PyTypeObject PyWindowsConsoleIO_Type = { 0, /* tp_finalize */ }; -PyObject * _PyWindowsConsoleIO_Type = (PyObject*)&PyWindowsConsoleIO_Type; - #endif /* MS_WINDOWS */ diff --git a/PC/_testconsole.c b/PC/_testconsole.c index a8308835d8f85d..f14a2d45b1be26 100644 --- a/PC/_testconsole.c +++ b/PC/_testconsole.c @@ -10,7 +10,7 @@ #ifdef MS_WINDOWS #include "pycore_fileutils.h" // _Py_get_osfhandle() -#include "..\modules\_io\_iomodule.h" +#include "pycore_runtime.h" // _Py_ID() #define WIN32_LEAN_AND_MEAN #include @@ -51,7 +51,14 @@ _testconsole_write_input_impl(PyObject *module, PyObject *file, { INPUT_RECORD *rec = NULL; - if (!PyWindowsConsoleIO_Check(file)) { + PyTypeObject *winconsoleio_type = (PyTypeObject *)_PyImport_GetModuleAttr( + &_Py_ID(_io), &_Py_ID(_WindowsConsoleIO)); + if (winconsoleio_type == NULL) { + return NULL; + } + int is_subclass = PyObject_TypeCheck(file, winconsoleio_type); + Py_DECREF(winconsoleio_type); + if (!is_subclass) { PyErr_SetString(PyExc_TypeError, "expected raw console object"); return NULL; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index a8a8e7f3d84f21..045a2996e8988b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -54,10 +54,6 @@ extern void _PyIO_Fini(void); #ifdef MS_WINDOWS # undef BYTE - - extern PyTypeObject PyWindowsConsoleIO_Type; -# define PyWindowsConsoleIO_Check(op) \ - (PyObject_TypeCheck((op), &PyWindowsConsoleIO_Type)) #endif #define PUTS(fd, str) _Py_write_noraise(fd, str, (int)strlen(str)) @@ -2358,8 +2354,16 @@ create_stdio(const PyConfig *config, PyObject* io, #ifdef MS_WINDOWS /* Windows console IO is always UTF-8 encoded */ - if (PyWindowsConsoleIO_Check(raw)) + PyTypeObject *winconsoleio_type = (PyTypeObject *)_PyImport_GetModuleAttr( + &_Py_ID(_io), &_Py_ID(_WindowsConsoleIO)); + if (winconsoleio_type == NULL) { + goto error; + } + int is_subclass = PyObject_TypeCheck(raw, winconsoleio_type); + Py_DECREF(winconsoleio_type); + if (is_subclass) { encoding = L"utf-8"; + } #endif text = PyUnicode_FromString(name); From c1ce0d178fe57b50f37578b285a343d77485ac02 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 15 Feb 2023 22:58:48 +0100 Subject: [PATCH 13/76] gh-99138: Isolate _zoneinfo (#99218) * Convert zone info type to heap type and add it to module state * Add global variables to module state --- Doc/library/zoneinfo.rst | 2 +- Lib/test/test_zoneinfo/test_zoneinfo.py | 4 +- ...3-01-02-22-41-44.gh-issue-99138.17hp9U.rst | 1 + Modules/_zoneinfo.c | 519 +++++++++++------- Modules/clinic/_zoneinfo.c.h | 215 +++++++- Tools/c-analyzer/cpython/globals-to-fix.tsv | 11 - 6 files changed, 539 insertions(+), 213 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst diff --git a/Doc/library/zoneinfo.rst b/Doc/library/zoneinfo.rst index d2e5619e7e47c2..f8624da6e51dbb 100644 --- a/Doc/library/zoneinfo.rst +++ b/Doc/library/zoneinfo.rst @@ -241,7 +241,7 @@ The following class methods are also available: .. warning:: Invoking this function may change the semantics of datetimes using - ``ZoneInfo`` in surprising ways; this modifies process-wide global state + ``ZoneInfo`` in surprising ways; this modifies module state and thus may have wide-ranging effects. Only use it if you know that you need to. diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index fd0e3bc032ec0c..82041a2b488334 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -1797,12 +1797,10 @@ def test_cache_location(self): self.assertTrue(hasattr(py_zoneinfo.ZoneInfo, "_weak_cache")) def test_gc_tracked(self): - # The pure Python version is tracked by the GC but (for now) the C - # version is not. import gc self.assertTrue(gc.is_tracked(py_zoneinfo.ZoneInfo)) - self.assertFalse(gc.is_tracked(c_zoneinfo.ZoneInfo)) + self.assertTrue(gc.is_tracked(c_zoneinfo.ZoneInfo)) @dataclasses.dataclass(frozen=True) diff --git a/Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst b/Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst new file mode 100644 index 00000000000000..3dd4646f40e1e5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst @@ -0,0 +1 @@ +Apply :pep:`687` to :mod:`zoneinfo`. Patch by Erlend E. Aasland. diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index 9d38589ea3d1b0..9f423559f51a43 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -19,10 +19,6 @@ class zoneinfo.ZoneInfo "PyObject *" "PyTypeObject *" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=d12c73c0eef36df8]*/ -// Imports -static PyObject *io_open = NULL; -static PyObject *_tzpath_find_tzfile = NULL; -static PyObject *_common_mod = NULL; typedef struct TransitionRuleType TransitionRuleType; typedef struct StrongCacheNode StrongCacheNode; @@ -90,15 +86,21 @@ struct StrongCacheNode { PyObject *zone; }; -static PyTypeObject PyZoneInfo_ZoneInfoType; +typedef struct { + PyTypeObject *ZoneInfoType; + + // Imports + PyObject *io_open; + PyObject *_tzpath_find_tzfile; + PyObject *_common_mod; -// Globals -static PyObject *TIMEDELTA_CACHE = NULL; -static PyObject *ZONEINFO_WEAK_CACHE = NULL; -static StrongCacheNode *ZONEINFO_STRONG_CACHE = NULL; -static size_t ZONEINFO_STRONG_CACHE_MAX_SIZE = 8; + // Caches + PyObject *TIMEDELTA_CACHE; + PyObject *ZONEINFO_WEAK_CACHE; + StrongCacheNode *ZONEINFO_STRONG_CACHE; -static _ttinfo NO_TTINFO = {NULL, NULL, NULL, 0}; + _ttinfo NO_TTINFO; +} zoneinfo_state; // Constants static const int EPOCHORDINAL = 719163; @@ -114,9 +116,12 @@ static const int SOURCE_NOCACHE = 0; static const int SOURCE_CACHE = 1; static const int SOURCE_FILE = 2; +static const size_t ZONEINFO_STRONG_CACHE_MAX_SIZE = 8; + // Forward declarations static int -load_data(PyZoneInfo_ZoneInfo *self, PyObject *file_obj); +load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, + PyObject *file_obj); static void utcoff_to_dstoff(size_t *trans_idx, long *utcoffs, long *dstoffs, unsigned char *isdsts, size_t num_transitions, @@ -127,7 +132,7 @@ ts_to_local(size_t *trans_idx, int64_t *trans_utc, long *utcoff, size_t num_transitions); static int -parse_tz_str(PyObject *tz_str_obj, _tzrule *out); +parse_tz_str(zoneinfo_state *state, PyObject *tz_str_obj, _tzrule *out); static Py_ssize_t parse_abbr(const char *const p, PyObject **abbr); @@ -146,26 +151,27 @@ find_tzrule_ttinfo_fromutc(_tzrule *rule, int64_t ts, int year, unsigned char *fold); static int -build_ttinfo(long utcoffset, long dstoffset, PyObject *tzname, _ttinfo *out); +build_ttinfo(zoneinfo_state *state, long utcoffset, long dstoffset, + PyObject *tzname, _ttinfo *out); static void xdecref_ttinfo(_ttinfo *ttinfo); static int ttinfo_eq(const _ttinfo *const tti0, const _ttinfo *const tti1); static int -build_tzrule(PyObject *std_abbr, PyObject *dst_abbr, long std_offset, - long dst_offset, TransitionRuleType *start, +build_tzrule(zoneinfo_state *state, PyObject *std_abbr, PyObject *dst_abbr, + long std_offset, long dst_offset, TransitionRuleType *start, TransitionRuleType *end, _tzrule *out); static void free_tzrule(_tzrule *tzrule); static PyObject * -load_timedelta(long seconds); +load_timedelta(zoneinfo_state *state, long seconds); static int get_local_timestamp(PyObject *dt, int64_t *local_ts); static _ttinfo * -find_ttinfo(PyZoneInfo_ZoneInfo *self, PyObject *dt); +find_ttinfo(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *dt); static int ymd_to_ord(int y, int m, int d); @@ -176,27 +182,57 @@ static size_t _bisect(const int64_t value, const int64_t *arr, size_t size); static int -eject_from_strong_cache(const PyTypeObject *const type, PyObject *key); +eject_from_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, + PyObject *key); static void -clear_strong_cache(const PyTypeObject *const type); +clear_strong_cache(zoneinfo_state *state, const PyTypeObject *const type); static void -update_strong_cache(const PyTypeObject *const type, PyObject *key, - PyObject *zone); +update_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, + PyObject *key, PyObject *zone); static PyObject * -zone_from_strong_cache(const PyTypeObject *const type, PyObject *const key); +zone_from_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, + PyObject *const key); + +static inline zoneinfo_state * +zoneinfo_get_state(PyObject *mod) +{ + zoneinfo_state *state = (zoneinfo_state *)PyModule_GetState(mod); + assert(state != NULL); + return state; +} + +static inline zoneinfo_state * +zoneinfo_get_state_by_cls(PyTypeObject *cls) +{ + zoneinfo_state *state = (zoneinfo_state *)PyType_GetModuleState(cls); + assert(state != NULL); + return state; +} + +static struct PyModuleDef zoneinfomodule; + +static inline zoneinfo_state * +zoneinfo_get_state_by_self(PyTypeObject *self) +{ + PyObject *mod = PyType_GetModuleByDef(self, &zoneinfomodule); + assert(mod != NULL); + return zoneinfo_get_state(mod); +} static PyObject * -zoneinfo_new_instance(PyTypeObject *type, PyObject *key) +zoneinfo_new_instance(zoneinfo_state *state, PyTypeObject *type, PyObject *key) { PyObject *file_obj = NULL; PyObject *file_path = NULL; - file_path = PyObject_CallFunctionObjArgs(_tzpath_find_tzfile, key, NULL); + file_path = PyObject_CallFunctionObjArgs(state->_tzpath_find_tzfile, + key, NULL); if (file_path == NULL) { return NULL; } else if (file_path == Py_None) { - file_obj = PyObject_CallMethod(_common_mod, "load_tzdata", "O", key); + PyObject *meth = state->_common_mod; + file_obj = PyObject_CallMethod(meth, "load_tzdata", "O", key); if (file_obj == NULL) { Py_DECREF(file_path); return NULL; @@ -209,13 +245,14 @@ zoneinfo_new_instance(PyTypeObject *type, PyObject *key) } if (file_obj == NULL) { - file_obj = PyObject_CallFunction(io_open, "Os", file_path, "rb"); + PyObject *func = state->io_open; + file_obj = PyObject_CallFunction(func, "Os", file_path, "rb"); if (file_obj == NULL) { goto error; } } - if (load_data((PyZoneInfo_ZoneInfo *)self, file_obj)) { + if (load_data(state, (PyZoneInfo_ZoneInfo *)self, file_obj)) { goto error; } @@ -248,10 +285,10 @@ zoneinfo_new_instance(PyTypeObject *type, PyObject *key) } static PyObject * -get_weak_cache(PyTypeObject *type) +get_weak_cache(zoneinfo_state *state, PyTypeObject *type) { - if (type == &PyZoneInfo_ZoneInfoType) { - return ZONEINFO_WEAK_CACHE; + if (type == state->ZoneInfoType) { + return state->ZONEINFO_WEAK_CACHE; } else { PyObject *cache = @@ -273,12 +310,13 @@ zoneinfo_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; } - PyObject *instance = zone_from_strong_cache(type, key); + zoneinfo_state *state = zoneinfo_get_state_by_self(type); + PyObject *instance = zone_from_strong_cache(state, type, key); if (instance != NULL || PyErr_Occurred()) { return instance; } - PyObject *weak_cache = get_weak_cache(type); + PyObject *weak_cache = get_weak_cache(state, type); instance = PyObject_CallMethod(weak_cache, "get", "O", key, Py_None); if (instance == NULL) { return NULL; @@ -286,7 +324,7 @@ zoneinfo_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (instance == Py_None) { Py_DECREF(instance); - PyObject *tmp = zoneinfo_new_instance(type, key); + PyObject *tmp = zoneinfo_new_instance(state, type, key); if (tmp == NULL) { return NULL; } @@ -300,14 +338,32 @@ zoneinfo_new(PyTypeObject *type, PyObject *args, PyObject *kw) ((PyZoneInfo_ZoneInfo *)instance)->source = SOURCE_CACHE; } - update_strong_cache(type, key, instance); + update_strong_cache(state, type, key, instance); return instance; } +static int +zoneinfo_traverse(PyZoneInfo_ZoneInfo *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + Py_VISIT(self->key); + return 0; +} + +static int +zoneinfo_clear(PyZoneInfo_ZoneInfo *self) +{ + Py_CLEAR(self->key); + Py_CLEAR(self->file_repr); + return 0; +} + static void zoneinfo_dealloc(PyObject *obj_self) { PyZoneInfo_ZoneInfo *self = (PyZoneInfo_ZoneInfo *)obj_self; + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); if (self->weakreflist != NULL) { PyObject_ClearWeakRefs(obj_self); @@ -336,16 +392,16 @@ zoneinfo_dealloc(PyObject *obj_self) free_tzrule(&(self->tzrule_after)); - Py_XDECREF(self->key); - Py_XDECREF(self->file_repr); - - Py_TYPE(self)->tp_free((PyObject *)self); + zoneinfo_clear(self); + tp->tp_free(obj_self); + Py_DECREF(tp); } /*[clinic input] @classmethod zoneinfo.ZoneInfo.from_file + cls: defining_class file_obj: object / key: object = None @@ -354,9 +410,9 @@ Create a ZoneInfo file from a file object. [clinic start generated code]*/ static PyObject * -zoneinfo_ZoneInfo_from_file_impl(PyTypeObject *type, PyObject *file_obj, - PyObject *key) -/*[clinic end generated code: output=68ed2022404ae5be input=ccfe73708133d2e4]*/ +zoneinfo_ZoneInfo_from_file_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *file_obj, PyObject *key) +/*[clinic end generated code: output=77887d1d56a48324 input=d26111f29eed6863]*/ { PyObject *file_repr = NULL; PyZoneInfo_ZoneInfo *self = NULL; @@ -372,7 +428,8 @@ zoneinfo_ZoneInfo_from_file_impl(PyTypeObject *type, PyObject *file_obj, goto error; } - if (load_data(self, file_obj)) { + zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); + if (load_data(state, self, file_obj)) { goto error; } @@ -391,16 +448,20 @@ zoneinfo_ZoneInfo_from_file_impl(PyTypeObject *type, PyObject *file_obj, @classmethod zoneinfo.ZoneInfo.no_cache + cls: defining_class + / key: object Get a new instance of ZoneInfo, bypassing the cache. [clinic start generated code]*/ static PyObject * -zoneinfo_ZoneInfo_no_cache_impl(PyTypeObject *type, PyObject *key) -/*[clinic end generated code: output=751c6894ad66f91b input=bb24afd84a80ba46]*/ +zoneinfo_ZoneInfo_no_cache_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *key) +/*[clinic end generated code: output=b0b09b3344c171b7 input=0238f3d56b1ea3f1]*/ { - PyObject *out = zoneinfo_new_instance(type, key); + zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); + PyObject *out = zoneinfo_new_instance(state, type, key); if (out != NULL) { ((PyZoneInfo_ZoneInfo *)out)->source = SOURCE_NOCACHE; } @@ -412,6 +473,8 @@ zoneinfo_ZoneInfo_no_cache_impl(PyTypeObject *type, PyObject *key) @classmethod zoneinfo.ZoneInfo.clear_cache + cls: defining_class + / * only_keys: object = None @@ -419,10 +482,12 @@ Clear the ZoneInfo cache. [clinic start generated code]*/ static PyObject * -zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyObject *only_keys) -/*[clinic end generated code: output=eec0a3276f07bd90 input=8cff0182a95f295b]*/ +zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *only_keys) +/*[clinic end generated code: output=114d9b7c8a22e660 input=e32ca3bb396788ba]*/ { - PyObject *weak_cache = get_weak_cache(type); + zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); + PyObject *weak_cache = get_weak_cache(state, type); if (only_keys == NULL || only_keys == Py_None) { PyObject *rv = PyObject_CallMethod(weak_cache, "clear", NULL); @@ -430,7 +495,7 @@ zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyObject *only_keys) Py_DECREF(rv); } - clear_strong_cache(type); + clear_strong_cache(state, type); } else { PyObject *item = NULL; @@ -447,7 +512,7 @@ zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyObject *only_keys) while ((item = PyIter_Next(iter))) { // Remove from strong cache - if (eject_from_strong_cache(type, item) < 0) { + if (eject_from_strong_cache(state, type, item) < 0) { Py_DECREF(item); break; } @@ -473,30 +538,68 @@ zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyObject *only_keys) Py_RETURN_NONE; } +/*[clinic input] +zoneinfo.ZoneInfo.utcoffset + + cls: defining_class + dt: object + / + +Retrieve a timedelta representing the UTC offset in a zone at the given datetime. +[clinic start generated code]*/ + static PyObject * -zoneinfo_utcoffset(PyObject *self, PyObject *dt) +zoneinfo_ZoneInfo_utcoffset_impl(PyObject *self, PyTypeObject *cls, + PyObject *dt) +/*[clinic end generated code: output=b71016c319ba1f91 input=2bb6c5364938f19c]*/ { - _ttinfo *tti = find_ttinfo((PyZoneInfo_ZoneInfo *)self, dt); + zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); + _ttinfo *tti = find_ttinfo(state, (PyZoneInfo_ZoneInfo *)self, dt); if (tti == NULL) { return NULL; } return Py_NewRef(tti->utcoff); } +/*[clinic input] +zoneinfo.ZoneInfo.dst + + cls: defining_class + dt: object + / + +Retrieve a timedelta representing the amount of DST applied in a zone at the given datetime. +[clinic start generated code]*/ + static PyObject * -zoneinfo_dst(PyObject *self, PyObject *dt) +zoneinfo_ZoneInfo_dst_impl(PyObject *self, PyTypeObject *cls, PyObject *dt) +/*[clinic end generated code: output=cb6168d7723a6ae6 input=2167fb80cf8645c6]*/ { - _ttinfo *tti = find_ttinfo((PyZoneInfo_ZoneInfo *)self, dt); + zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); + _ttinfo *tti = find_ttinfo(state, (PyZoneInfo_ZoneInfo *)self, dt); if (tti == NULL) { return NULL; } return Py_NewRef(tti->dstoff); } +/*[clinic input] +zoneinfo.ZoneInfo.tzname + + cls: defining_class + dt: object + / + +Retrieve a string containing the abbreviation for the time zone that applies in a zone at a given datetime. +[clinic start generated code]*/ + static PyObject * -zoneinfo_tzname(PyObject *self, PyObject *dt) +zoneinfo_ZoneInfo_tzname_impl(PyObject *self, PyTypeObject *cls, + PyObject *dt) +/*[clinic end generated code: output=3b6ae6c3053ea75a input=15a59a4f92ed1f1f]*/ { - _ttinfo *tti = find_ttinfo((PyZoneInfo_ZoneInfo *)self, dt); + zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); + _ttinfo *tti = find_ttinfo(state, (PyZoneInfo_ZoneInfo *)self, dt); if (tti == NULL) { return NULL; } @@ -693,28 +796,37 @@ zoneinfo_reduce(PyObject *obj_self, PyObject *unused) return rv; } +/*[clinic input] +@classmethod +zoneinfo.ZoneInfo._unpickle + + cls: defining_class + key: object + from_cache: unsigned_char(bitwise=True) + / + +Private method used in unpickling. +[clinic start generated code]*/ + static PyObject * -zoneinfo__unpickle(PyTypeObject *cls, PyObject *args) +zoneinfo_ZoneInfo__unpickle_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *key, unsigned char from_cache) +/*[clinic end generated code: output=556712fc709deecb input=6ac8c73eed3de316]*/ { - PyObject *key; - unsigned char from_cache; - if (!PyArg_ParseTuple(args, "OB", &key, &from_cache)) { - return NULL; - } - if (from_cache) { PyObject *val_args = Py_BuildValue("(O)", key); if (val_args == NULL) { return NULL; } - PyObject *rv = zoneinfo_new(cls, val_args, NULL); + PyObject *rv = zoneinfo_new(type, val_args, NULL); Py_DECREF(val_args); return rv; } else { - return zoneinfo_new_instance(cls, key); + zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); + return zoneinfo_new_instance(state, type, key); } } @@ -732,14 +844,14 @@ zoneinfo__unpickle(PyTypeObject *cls, PyObject *args) * This returns a new reference to the timedelta. */ static PyObject * -load_timedelta(long seconds) +load_timedelta(zoneinfo_state *state, long seconds) { PyObject *rv; PyObject *pyoffset = PyLong_FromLong(seconds); if (pyoffset == NULL) { return NULL; } - rv = PyDict_GetItemWithError(TIMEDELTA_CACHE, pyoffset); + rv = PyDict_GetItemWithError(state->TIMEDELTA_CACHE, pyoffset); if (rv == NULL) { if (PyErr_Occurred()) { goto error; @@ -751,7 +863,7 @@ load_timedelta(long seconds) goto error; } - rv = PyDict_SetDefault(TIMEDELTA_CACHE, pyoffset, tmp); + rv = PyDict_SetDefault(state->TIMEDELTA_CACHE, pyoffset, tmp); Py_DECREF(tmp); } @@ -768,19 +880,20 @@ load_timedelta(long seconds) * initialized _ttinfo objects. */ static int -build_ttinfo(long utcoffset, long dstoffset, PyObject *tzname, _ttinfo *out) +build_ttinfo(zoneinfo_state *state, long utcoffset, long dstoffset, + PyObject *tzname, _ttinfo *out) { out->utcoff = NULL; out->dstoff = NULL; out->tzname = NULL; out->utcoff_seconds = utcoffset; - out->utcoff = load_timedelta(utcoffset); + out->utcoff = load_timedelta(state, utcoffset); if (out->utcoff == NULL) { return -1; } - out->dstoff = load_timedelta(dstoffset); + out->dstoff = load_timedelta(state, dstoffset); if (out->dstoff == NULL) { return -1; } @@ -836,7 +949,7 @@ ttinfo_eq(const _ttinfo *const tti0, const _ttinfo *const tti1) * the object only needs to be freed / deallocated if this succeeds. */ static int -load_data(PyZoneInfo_ZoneInfo *self, PyObject *file_obj) +load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) { PyObject *data_tuple = NULL; @@ -854,7 +967,8 @@ load_data(PyZoneInfo_ZoneInfo *self, PyObject *file_obj) size_t ttinfos_allocated = 0; - data_tuple = PyObject_CallMethod(_common_mod, "load_data", "O", file_obj); + data_tuple = PyObject_CallMethod(state->_common_mod, "load_data", "O", + file_obj); if (data_tuple == NULL) { goto error; @@ -1012,7 +1126,9 @@ load_data(PyZoneInfo_ZoneInfo *self, PyObject *file_obj) } ttinfos_allocated++; - if (build_ttinfo(utcoff[i], dstoff[i], tzname, &(self->_ttinfos[i]))) { + int rc = build_ttinfo(state, utcoff[i], dstoff[i], tzname, + &(self->_ttinfos[i])); + if (rc) { goto error; } } @@ -1044,7 +1160,7 @@ load_data(PyZoneInfo_ZoneInfo *self, PyObject *file_obj) } if (tz_str != Py_None && PyObject_IsTrue(tz_str)) { - if (parse_tz_str(tz_str, &(self->tzrule_after))) { + if (parse_tz_str(state, tz_str, &(self->tzrule_after))) { goto error; } } @@ -1063,8 +1179,8 @@ load_data(PyZoneInfo_ZoneInfo *self, PyObject *file_obj) } _ttinfo *tti = &(self->_ttinfos[idx]); - build_tzrule(tti->tzname, NULL, tti->utcoff_seconds, 0, NULL, NULL, - &(self->tzrule_after)); + build_tzrule(state, tti->tzname, NULL, tti->utcoff_seconds, 0, NULL, + NULL, &(self->tzrule_after)); // We've abused the build_tzrule constructor to construct an STD-only // rule mimicking whatever ttinfo we've picked up, but it's possible @@ -1463,7 +1579,7 @@ find_tzrule_ttinfo_fromutc(_tzrule *rule, int64_t ts, int year, * https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html */ static int -parse_tz_str(PyObject *tz_str_obj, _tzrule *out) +parse_tz_str(zoneinfo_state *state, PyObject *tz_str_obj, _tzrule *out) { PyObject *std_abbr = NULL; PyObject *dst_abbr = NULL; @@ -1555,7 +1671,8 @@ parse_tz_str(PyObject *tz_str_obj, _tzrule *out) } complete: - build_tzrule(std_abbr, dst_abbr, std_offset, dst_offset, start, end, out); + build_tzrule(state, std_abbr, dst_abbr, std_offset, dst_offset, + start, end, out); Py_DECREF(std_abbr); Py_XDECREF(dst_abbr); @@ -1913,8 +2030,8 @@ parse_transition_time(const char *const p, int8_t *hour, int8_t *minute, * Returns 0 on success. */ static int -build_tzrule(PyObject *std_abbr, PyObject *dst_abbr, long std_offset, - long dst_offset, TransitionRuleType *start, +build_tzrule(zoneinfo_state *state, PyObject *std_abbr, PyObject *dst_abbr, + long std_offset, long dst_offset, TransitionRuleType *start, TransitionRuleType *end, _tzrule *out) { _tzrule rv = {{0}}; @@ -1922,13 +2039,13 @@ build_tzrule(PyObject *std_abbr, PyObject *dst_abbr, long std_offset, rv.start = start; rv.end = end; - if (build_ttinfo(std_offset, 0, std_abbr, &rv.std)) { + if (build_ttinfo(state, std_offset, 0, std_abbr, &rv.std)) { goto error; } if (dst_abbr != NULL) { rv.dst_diff = dst_offset - std_offset; - if (build_ttinfo(dst_offset, rv.dst_diff, dst_abbr, &rv.dst)) { + if (build_ttinfo(state, dst_offset, rv.dst_diff, dst_abbr, &rv.dst)) { goto error; } } @@ -2132,7 +2249,7 @@ _bisect(const int64_t value, const int64_t *arr, size_t size) /* Find the ttinfo rules that apply at a given local datetime. */ static _ttinfo * -find_ttinfo(PyZoneInfo_ZoneInfo *self, PyObject *dt) +find_ttinfo(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *dt) { // datetime.time has a .tzinfo attribute that passes None as the dt // argument; it only really has meaning for fixed-offset zones. @@ -2141,7 +2258,7 @@ find_ttinfo(PyZoneInfo_ZoneInfo *self, PyObject *dt) return &(self->tzrule_after.std); } else { - return &NO_TTINFO; + return &(state->NO_TTINFO); } } @@ -2317,10 +2434,10 @@ strong_cache_free(StrongCacheNode *root) * the front of the cache. */ static void -remove_from_strong_cache(StrongCacheNode *node) +remove_from_strong_cache(zoneinfo_state *state, StrongCacheNode *node) { - if (ZONEINFO_STRONG_CACHE == node) { - ZONEINFO_STRONG_CACHE = node->next; + if (state->ZONEINFO_STRONG_CACHE == node) { + state->ZONEINFO_STRONG_CACHE = node->next; } if (node->prev != NULL) { @@ -2366,15 +2483,17 @@ find_in_strong_cache(const StrongCacheNode *const root, PyObject *const key) * This function is used to enable the per-key functionality in clear_cache. */ static int -eject_from_strong_cache(const PyTypeObject *const type, PyObject *key) +eject_from_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, + PyObject *key) { - if (type != &PyZoneInfo_ZoneInfoType) { + if (type != state->ZoneInfoType) { return 0; } - StrongCacheNode *node = find_in_strong_cache(ZONEINFO_STRONG_CACHE, key); + StrongCacheNode *cache = state->ZONEINFO_STRONG_CACHE; + StrongCacheNode *node = find_in_strong_cache(cache, key); if (node != NULL) { - remove_from_strong_cache(node); + remove_from_strong_cache(state, node); strong_cache_node_free(node); } @@ -2390,14 +2509,15 @@ eject_from_strong_cache(const PyTypeObject *const type, PyObject *key) * it is not at the front of the cache, it needs to be moved there. */ static void -move_strong_cache_node_to_front(StrongCacheNode **root, StrongCacheNode *node) +move_strong_cache_node_to_front(zoneinfo_state *state, StrongCacheNode **root, + StrongCacheNode *node) { StrongCacheNode *root_p = *root; if (root_p == node) { return; } - remove_from_strong_cache(node); + remove_from_strong_cache(state, node); node->prev = NULL; node->next = root_p; @@ -2419,16 +2539,19 @@ move_strong_cache_node_to_front(StrongCacheNode **root, StrongCacheNode *node) * always returns a cache miss for subclasses. */ static PyObject * -zone_from_strong_cache(const PyTypeObject *const type, PyObject *const key) +zone_from_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, + PyObject *const key) { - if (type != &PyZoneInfo_ZoneInfoType) { + if (type != state->ZoneInfoType) { return NULL; // Strong cache currently only implemented for base class } - StrongCacheNode *node = find_in_strong_cache(ZONEINFO_STRONG_CACHE, key); + StrongCacheNode *cache = state->ZONEINFO_STRONG_CACHE; + StrongCacheNode *node = find_in_strong_cache(cache, key); if (node != NULL) { - move_strong_cache_node_to_front(&ZONEINFO_STRONG_CACHE, node); + StrongCacheNode **root = &(state->ZONEINFO_STRONG_CACHE); + move_strong_cache_node_to_front(state, root, node); return Py_NewRef(node->zone); } @@ -2442,16 +2565,16 @@ zone_from_strong_cache(const PyTypeObject *const type, PyObject *const key) * the cache to at most ZONEINFO_STRONG_CACHE_MAX_SIZE). */ static void -update_strong_cache(const PyTypeObject *const type, PyObject *key, - PyObject *zone) +update_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, + PyObject *key, PyObject *zone) { - if (type != &PyZoneInfo_ZoneInfoType) { + if (type != state->ZoneInfoType) { return; } StrongCacheNode *new_node = strong_cache_node_new(key, zone); - - move_strong_cache_node_to_front(&ZONEINFO_STRONG_CACHE, new_node); + StrongCacheNode **root = &(state->ZONEINFO_STRONG_CACHE); + move_strong_cache_node_to_front(state, root, new_node); StrongCacheNode *node = new_node->next; for (size_t i = 1; i < ZONEINFO_STRONG_CACHE_MAX_SIZE; ++i) { @@ -2476,14 +2599,14 @@ update_strong_cache(const PyTypeObject *const type, PyObject *key, * for everything except the base class. */ void -clear_strong_cache(const PyTypeObject *const type) +clear_strong_cache(zoneinfo_state *state, const PyTypeObject *const type) { - if (type != &PyZoneInfo_ZoneInfoType) { + if (type != state->ZoneInfoType) { return; } - strong_cache_free(ZONEINFO_STRONG_CACHE); - ZONEINFO_STRONG_CACHE = NULL; + strong_cache_free(state->ZONEINFO_STRONG_CACHE); + state->ZONEINFO_STRONG_CACHE = NULL; } static PyObject * @@ -2499,29 +2622,17 @@ new_weak_cache(void) return weak_cache; } +// This function is not idempotent and must be called on a new module object. static int -initialize_caches(void) +initialize_caches(zoneinfo_state *state) { - // TODO: Move to a PyModule_GetState / PEP 573 based caching system. - if (TIMEDELTA_CACHE == NULL) { - TIMEDELTA_CACHE = PyDict_New(); - } - else { - Py_INCREF(TIMEDELTA_CACHE); - } - - if (TIMEDELTA_CACHE == NULL) { + state->TIMEDELTA_CACHE = PyDict_New(); + if (state->TIMEDELTA_CACHE == NULL) { return -1; } - if (ZONEINFO_WEAK_CACHE == NULL) { - ZONEINFO_WEAK_CACHE = new_weak_cache(); - } - else { - Py_INCREF(ZONEINFO_WEAK_CACHE); - } - - if (ZONEINFO_WEAK_CACHE == NULL) { + state->ZONEINFO_WEAK_CACHE = new_weak_cache(); + if (state->ZONEINFO_WEAK_CACHE == NULL) { return -1; } @@ -2551,22 +2662,15 @@ static PyMethodDef zoneinfo_methods[] = { ZONEINFO_ZONEINFO_CLEAR_CACHE_METHODDEF ZONEINFO_ZONEINFO_NO_CACHE_METHODDEF ZONEINFO_ZONEINFO_FROM_FILE_METHODDEF - {"utcoffset", (PyCFunction)zoneinfo_utcoffset, METH_O, - PyDoc_STR("Retrieve a timedelta representing the UTC offset in a zone at " - "the given datetime.")}, - {"dst", (PyCFunction)zoneinfo_dst, METH_O, - PyDoc_STR("Retrieve a timedelta representing the amount of DST applied " - "in a zone at the given datetime.")}, - {"tzname", (PyCFunction)zoneinfo_tzname, METH_O, - PyDoc_STR("Retrieve a string containing the abbreviation for the time " - "zone that applies in a zone at a given datetime.")}, + ZONEINFO_ZONEINFO_UTCOFFSET_METHODDEF + ZONEINFO_ZONEINFO_DST_METHODDEF + ZONEINFO_ZONEINFO_TZNAME_METHODDEF {"fromutc", (PyCFunction)zoneinfo_fromutc, METH_O, PyDoc_STR("Given a datetime with local time in UTC, retrieve an adjusted " "datetime in local time.")}, {"__reduce__", (PyCFunction)zoneinfo_reduce, METH_NOARGS, PyDoc_STR("Function for serialization with the pickle protocol.")}, - {"_unpickle", (PyCFunction)zoneinfo__unpickle, METH_VARARGS | METH_CLASS, - PyDoc_STR("Private method used in unpickling.")}, + ZONEINFO_ZONEINFO__UNPICKLE_METHODDEF {"__init_subclass__", (PyCFunction)(void (*)(void))zoneinfo_init_subclass, METH_VARARGS | METH_KEYWORDS | METH_CLASS, PyDoc_STR("Function to initialize subclasses.")}, @@ -2579,50 +2683,88 @@ static PyMemberDef zoneinfo_members[] = { .type = T_OBJECT_EX, .flags = READONLY, .doc = NULL}, + {.name = "__weaklistoffset__", + .offset = offsetof(PyZoneInfo_ZoneInfo, weakreflist), + .type = T_PYSSIZET, + .flags = READONLY}, {NULL}, /* Sentinel */ }; -static PyTypeObject PyZoneInfo_ZoneInfoType = { - PyVarObject_HEAD_INIT(NULL, 0) // - .tp_name = "zoneinfo.ZoneInfo", - .tp_basicsize = sizeof(PyZoneInfo_ZoneInfo), - .tp_weaklistoffset = offsetof(PyZoneInfo_ZoneInfo, weakreflist), - .tp_repr = (reprfunc)zoneinfo_repr, - .tp_str = (reprfunc)zoneinfo_str, - .tp_getattro = PyObject_GenericGetAttr, - .tp_flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE), - /* .tp_doc = zoneinfo_doc, */ - .tp_methods = zoneinfo_methods, - .tp_members = zoneinfo_members, - .tp_new = zoneinfo_new, - .tp_dealloc = zoneinfo_dealloc, +static PyType_Slot zoneinfo_slots[] = { + {Py_tp_repr, zoneinfo_repr}, + {Py_tp_str, zoneinfo_str}, + {Py_tp_getattro, PyObject_GenericGetAttr}, + {Py_tp_methods, zoneinfo_methods}, + {Py_tp_members, zoneinfo_members}, + {Py_tp_new, zoneinfo_new}, + {Py_tp_dealloc, zoneinfo_dealloc}, + {Py_tp_traverse, zoneinfo_traverse}, + {Py_tp_clear, zoneinfo_clear}, + {0, NULL}, +}; + +static PyType_Spec zoneinfo_spec = { + .name = "zoneinfo.ZoneInfo", + .basicsize = sizeof(PyZoneInfo_ZoneInfo), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE), + .slots = zoneinfo_slots, }; ///// // Specify the _zoneinfo module static PyMethodDef module_methods[] = {{NULL, NULL}}; -static void -module_free(void *m) + +static int +module_traverse(PyObject *mod, visitproc visit, void *arg) { - Py_CLEAR(_tzpath_find_tzfile); - Py_CLEAR(_common_mod); - Py_CLEAR(io_open); + zoneinfo_state *state = zoneinfo_get_state(mod); - xdecref_ttinfo(&NO_TTINFO); + Py_VISIT(state->ZoneInfoType); + Py_VISIT(state->io_open); + Py_VISIT(state->_tzpath_find_tzfile); + Py_VISIT(state->_common_mod); + Py_VISIT(state->TIMEDELTA_CACHE); + Py_VISIT(state->ZONEINFO_WEAK_CACHE); - if (TIMEDELTA_CACHE != NULL && Py_REFCNT(TIMEDELTA_CACHE) > 1) { - Py_DECREF(TIMEDELTA_CACHE); - } else { - Py_CLEAR(TIMEDELTA_CACHE); + StrongCacheNode *node = state->ZONEINFO_STRONG_CACHE; + while (node != NULL) { + StrongCacheNode *next = node->next; + Py_VISIT(node->key); + Py_VISIT(node->zone); + node = next; } - if (ZONEINFO_WEAK_CACHE != NULL && Py_REFCNT(ZONEINFO_WEAK_CACHE) > 1) { - Py_DECREF(ZONEINFO_WEAK_CACHE); - } else { - Py_CLEAR(ZONEINFO_WEAK_CACHE); - } + Py_VISIT(state->NO_TTINFO.utcoff); + Py_VISIT(state->NO_TTINFO.dstoff); + Py_VISIT(state->NO_TTINFO.tzname); - clear_strong_cache(&PyZoneInfo_ZoneInfoType); + return 0; +} + +static int +module_clear(PyObject *mod) +{ + zoneinfo_state *state = zoneinfo_get_state(mod); + + Py_CLEAR(state->ZoneInfoType); + Py_CLEAR(state->io_open); + Py_CLEAR(state->_tzpath_find_tzfile); + Py_CLEAR(state->_common_mod); + Py_CLEAR(state->TIMEDELTA_CACHE); + Py_CLEAR(state->ZONEINFO_WEAK_CACHE); + clear_strong_cache(state, state->ZoneInfoType); + Py_CLEAR(state->NO_TTINFO.utcoff); + Py_CLEAR(state->NO_TTINFO.dstoff); + Py_CLEAR(state->NO_TTINFO.tzname); + + return 0; +} + +static void +module_free(void *mod) +{ + (void)module_clear((PyObject *)mod); } static int @@ -2632,39 +2774,45 @@ zoneinfomodule_exec(PyObject *m) if (PyDateTimeAPI == NULL) { goto error; } - PyZoneInfo_ZoneInfoType.tp_base = PyDateTimeAPI->TZInfoType; - if (PyType_Ready(&PyZoneInfo_ZoneInfoType) < 0) { + + zoneinfo_state *state = zoneinfo_get_state(m); + PyObject *base = (PyObject *)PyDateTimeAPI->TZInfoType; + state->ZoneInfoType = (PyTypeObject *)PyType_FromModuleAndSpec(m, + &zoneinfo_spec, base); + if (state->ZoneInfoType == NULL) { goto error; } - if (PyModule_AddObjectRef(m, "ZoneInfo", (PyObject *)&PyZoneInfo_ZoneInfoType) < 0) { + int rc = PyModule_AddObjectRef(m, "ZoneInfo", + (PyObject *)state->ZoneInfoType); + if (rc < 0) { goto error; } /* Populate imports */ - _tzpath_find_tzfile = + state->_tzpath_find_tzfile = _PyImport_GetModuleAttrString("zoneinfo._tzpath", "find_tzfile"); - if (_tzpath_find_tzfile == NULL) { + if (state->_tzpath_find_tzfile == NULL) { goto error; } - io_open = _PyImport_GetModuleAttrString("io", "open"); - if (io_open == NULL) { + state->io_open = _PyImport_GetModuleAttrString("io", "open"); + if (state->io_open == NULL) { goto error; } - _common_mod = PyImport_ImportModule("zoneinfo._common"); - if (_common_mod == NULL) { + state->_common_mod = PyImport_ImportModule("zoneinfo._common"); + if (state->_common_mod == NULL) { goto error; } - if (NO_TTINFO.utcoff == NULL) { - NO_TTINFO.utcoff = Py_NewRef(Py_None); - NO_TTINFO.dstoff = Py_NewRef(Py_None); - NO_TTINFO.tzname = Py_NewRef(Py_None); + if (state->NO_TTINFO.utcoff == NULL) { + state->NO_TTINFO.utcoff = Py_NewRef(Py_None); + state->NO_TTINFO.dstoff = Py_NewRef(Py_None); + state->NO_TTINFO.tzname = Py_NewRef(Py_None); } - if (initialize_caches()) { + if (initialize_caches(state)) { goto error; } @@ -2678,13 +2826,16 @@ static PyModuleDef_Slot zoneinfomodule_slots[] = { {Py_mod_exec, zoneinfomodule_exec}, {0, NULL}}; static struct PyModuleDef zoneinfomodule = { - PyModuleDef_HEAD_INIT, + .m_base = PyModuleDef_HEAD_INIT, .m_name = "_zoneinfo", .m_doc = "C implementation of the zoneinfo module", - .m_size = 0, + .m_size = sizeof(zoneinfo_state), .m_methods = module_methods, .m_slots = zoneinfomodule_slots, - .m_free = (freefunc)module_free}; + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = module_free, +}; PyMODINIT_FUNC PyInit__zoneinfo(void) diff --git a/Modules/clinic/_zoneinfo.c.h b/Modules/clinic/_zoneinfo.c.h index 78fcbfa9411bb8..ae62865e0f67df 100644 --- a/Modules/clinic/_zoneinfo.c.h +++ b/Modules/clinic/_zoneinfo.c.h @@ -15,14 +15,14 @@ PyDoc_STRVAR(zoneinfo_ZoneInfo_from_file__doc__, "Create a ZoneInfo file from a file object."); #define ZONEINFO_ZONEINFO_FROM_FILE_METHODDEF \ - {"from_file", _PyCFunction_CAST(zoneinfo_ZoneInfo_from_file), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, zoneinfo_ZoneInfo_from_file__doc__}, + {"from_file", _PyCFunction_CAST(zoneinfo_ZoneInfo_from_file), METH_METHOD|METH_FASTCALL|METH_KEYWORDS|METH_CLASS, zoneinfo_ZoneInfo_from_file__doc__}, static PyObject * -zoneinfo_ZoneInfo_from_file_impl(PyTypeObject *type, PyObject *file_obj, - PyObject *key); +zoneinfo_ZoneInfo_from_file_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *file_obj, PyObject *key); static PyObject * -zoneinfo_ZoneInfo_from_file(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +zoneinfo_ZoneInfo_from_file(PyTypeObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -65,7 +65,7 @@ zoneinfo_ZoneInfo_from_file(PyTypeObject *type, PyObject *const *args, Py_ssize_ } key = args[1]; skip_optional_pos: - return_value = zoneinfo_ZoneInfo_from_file_impl(type, file_obj, key); + return_value = zoneinfo_ZoneInfo_from_file_impl(type, cls, file_obj, key); exit: return return_value; @@ -78,13 +78,14 @@ PyDoc_STRVAR(zoneinfo_ZoneInfo_no_cache__doc__, "Get a new instance of ZoneInfo, bypassing the cache."); #define ZONEINFO_ZONEINFO_NO_CACHE_METHODDEF \ - {"no_cache", _PyCFunction_CAST(zoneinfo_ZoneInfo_no_cache), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, zoneinfo_ZoneInfo_no_cache__doc__}, + {"no_cache", _PyCFunction_CAST(zoneinfo_ZoneInfo_no_cache), METH_METHOD|METH_FASTCALL|METH_KEYWORDS|METH_CLASS, zoneinfo_ZoneInfo_no_cache__doc__}, static PyObject * -zoneinfo_ZoneInfo_no_cache_impl(PyTypeObject *type, PyObject *key); +zoneinfo_ZoneInfo_no_cache_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *key); static PyObject * -zoneinfo_ZoneInfo_no_cache(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +zoneinfo_ZoneInfo_no_cache(PyTypeObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -120,7 +121,7 @@ zoneinfo_ZoneInfo_no_cache(PyTypeObject *type, PyObject *const *args, Py_ssize_t goto exit; } key = args[0]; - return_value = zoneinfo_ZoneInfo_no_cache_impl(type, key); + return_value = zoneinfo_ZoneInfo_no_cache_impl(type, cls, key); exit: return return_value; @@ -133,13 +134,14 @@ PyDoc_STRVAR(zoneinfo_ZoneInfo_clear_cache__doc__, "Clear the ZoneInfo cache."); #define ZONEINFO_ZONEINFO_CLEAR_CACHE_METHODDEF \ - {"clear_cache", _PyCFunction_CAST(zoneinfo_ZoneInfo_clear_cache), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, zoneinfo_ZoneInfo_clear_cache__doc__}, + {"clear_cache", _PyCFunction_CAST(zoneinfo_ZoneInfo_clear_cache), METH_METHOD|METH_FASTCALL|METH_KEYWORDS|METH_CLASS, zoneinfo_ZoneInfo_clear_cache__doc__}, static PyObject * -zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyObject *only_keys); +zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *only_keys); static PyObject * -zoneinfo_ZoneInfo_clear_cache(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +zoneinfo_ZoneInfo_clear_cache(PyTypeObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -180,9 +182,194 @@ zoneinfo_ZoneInfo_clear_cache(PyTypeObject *type, PyObject *const *args, Py_ssiz } only_keys = args[0]; skip_optional_kwonly: - return_value = zoneinfo_ZoneInfo_clear_cache_impl(type, only_keys); + return_value = zoneinfo_ZoneInfo_clear_cache_impl(type, cls, only_keys); exit: return return_value; } -/*[clinic end generated code: output=d2da73ef66146b83 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(zoneinfo_ZoneInfo_utcoffset__doc__, +"utcoffset($self, dt, /)\n" +"--\n" +"\n" +"Retrieve a timedelta representing the UTC offset in a zone at the given datetime."); + +#define ZONEINFO_ZONEINFO_UTCOFFSET_METHODDEF \ + {"utcoffset", _PyCFunction_CAST(zoneinfo_ZoneInfo_utcoffset), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, zoneinfo_ZoneInfo_utcoffset__doc__}, + +static PyObject * +zoneinfo_ZoneInfo_utcoffset_impl(PyObject *self, PyTypeObject *cls, + PyObject *dt); + +static PyObject * +zoneinfo_ZoneInfo_utcoffset(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "utcoffset", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *dt; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + dt = args[0]; + return_value = zoneinfo_ZoneInfo_utcoffset_impl(self, cls, dt); + +exit: + return return_value; +} + +PyDoc_STRVAR(zoneinfo_ZoneInfo_dst__doc__, +"dst($self, dt, /)\n" +"--\n" +"\n" +"Retrieve a timedelta representing the amount of DST applied in a zone at the given datetime."); + +#define ZONEINFO_ZONEINFO_DST_METHODDEF \ + {"dst", _PyCFunction_CAST(zoneinfo_ZoneInfo_dst), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, zoneinfo_ZoneInfo_dst__doc__}, + +static PyObject * +zoneinfo_ZoneInfo_dst_impl(PyObject *self, PyTypeObject *cls, PyObject *dt); + +static PyObject * +zoneinfo_ZoneInfo_dst(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "dst", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *dt; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + dt = args[0]; + return_value = zoneinfo_ZoneInfo_dst_impl(self, cls, dt); + +exit: + return return_value; +} + +PyDoc_STRVAR(zoneinfo_ZoneInfo_tzname__doc__, +"tzname($self, dt, /)\n" +"--\n" +"\n" +"Retrieve a string containing the abbreviation for the time zone that applies in a zone at a given datetime."); + +#define ZONEINFO_ZONEINFO_TZNAME_METHODDEF \ + {"tzname", _PyCFunction_CAST(zoneinfo_ZoneInfo_tzname), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, zoneinfo_ZoneInfo_tzname__doc__}, + +static PyObject * +zoneinfo_ZoneInfo_tzname_impl(PyObject *self, PyTypeObject *cls, + PyObject *dt); + +static PyObject * +zoneinfo_ZoneInfo_tzname(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "tzname", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *dt; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + dt = args[0]; + return_value = zoneinfo_ZoneInfo_tzname_impl(self, cls, dt); + +exit: + return return_value; +} + +PyDoc_STRVAR(zoneinfo_ZoneInfo__unpickle__doc__, +"_unpickle($type, key, from_cache, /)\n" +"--\n" +"\n" +"Private method used in unpickling."); + +#define ZONEINFO_ZONEINFO__UNPICKLE_METHODDEF \ + {"_unpickle", _PyCFunction_CAST(zoneinfo_ZoneInfo__unpickle), METH_METHOD|METH_FASTCALL|METH_KEYWORDS|METH_CLASS, zoneinfo_ZoneInfo__unpickle__doc__}, + +static PyObject * +zoneinfo_ZoneInfo__unpickle_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *key, unsigned char from_cache); + +static PyObject * +zoneinfo_ZoneInfo__unpickle(PyTypeObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_unpickle", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *key; + unsigned char from_cache; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + key = args[0]; + { + unsigned long ival = PyLong_AsUnsignedLongMask(args[1]); + if (ival == (unsigned long)-1 && PyErr_Occurred()) { + goto exit; + } + else { + from_cache = (unsigned char) ival; + } + } + return_value = zoneinfo_ZoneInfo__unpickle_impl(type, cls, key, from_cache); + +exit: + return return_value; +} +/*[clinic end generated code: output=54051388dfc408af input=a9049054013a1b77]*/ diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 52ea0b4901d4bb..6011b1604508af 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -403,7 +403,6 @@ Modules/_pickle.c - PicklerMemoProxyType - Modules/_pickle.c - Pickler_Type - Modules/_pickle.c - UnpicklerMemoProxyType - Modules/_pickle.c - Unpickler_Type - -Modules/_zoneinfo.c - PyZoneInfo_ZoneInfoType - Modules/ossaudiodev.c - OSSAudioType - Modules/ossaudiodev.c - OSSMixerType - Modules/socketmodule.c - sock_type - @@ -442,11 +441,6 @@ Modules/xxmodule.c - ErrorObject - Modules/_ctypes/callproc.c _ctypes_get_errobj error_object_name - Modules/_ctypes/_ctypes.c CreateSwappedType suffix - -## other - during module init -Modules/_zoneinfo.c - io_open - -Modules/_zoneinfo.c - _tzpath_find_tzfile - -Modules/_zoneinfo.c - _common_mod - - ##----------------------- ## other @@ -481,8 +475,6 @@ Modules/_tkinter.c - tcl_lock - Modules/_tkinter.c - excInCmd - Modules/_tkinter.c - valInCmd - Modules/_tkinter.c - trbInCmd - -Modules/_zoneinfo.c - TIMEDELTA_CACHE - -Modules/_zoneinfo.c - ZONEINFO_WEAK_CACHE - ################################## @@ -556,9 +548,6 @@ Modules/_tkinter.c - HeadFHCD - Modules/_tkinter.c - stdin_ready - Modules/_tkinter.c - event_tstate - Modules/_xxsubinterpretersmodule.c - _globals - -Modules/_zoneinfo.c - ZONEINFO_STRONG_CACHE - -Modules/_zoneinfo.c - ZONEINFO_STRONG_CACHE_MAX_SIZE - -Modules/_zoneinfo.c - NO_TTINFO - Modules/readline.c - completer_word_break_characters - Modules/readline.c - _history_length - Modules/readline.c - should_auto_add_history - From b2fc5492789623d656953d458f3eeaac03c1ef56 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 15 Feb 2023 15:32:31 -0700 Subject: [PATCH 14/76] gh-101758: Clean Up Uses of Import State (gh-101919) This change is almost entirely moving code around and hiding import state behind internal API. We introduce no changes to behavior, nor to non-internal API. (Since there was already going to be a lot of churn, I took this as an opportunity to re-organize import.c into topically-grouped sections of code.) The motivation is to simplify a number of upcoming changes. Specific changes: * move existing import-related code to import.c, wherever possible * add internal API for interacting with import state (both global and per-interpreter) * use only API outside of import.c (to limit churn there when changing the location, etc.) * consolidate the import-related state of PyInterpreterState into a single struct field (this changes layout slightly) * add macros for import state in import.c (to simplify changing the location) * group code in import.c into sections *remove _PyState_AddModule() https://github.com/python/cpython/issues/101758 --- Include/internal/pycore_import.h | 103 +- Include/internal/pycore_interp.h | 35 +- Include/internal/pycore_pylifecycle.h | 4 - Include/internal/pycore_pystate.h | 6 - Include/internal/pycore_runtime_init.h | 15 +- Include/internal/pycore_sysmodule.h | 3 + Lib/test/test_imp.py | 2 +- Lib/test/test_stable_abi_ctypes.py | 1 - Misc/stable_abi.toml | 3 - Objects/moduleobject.c | 25 +- PC/python3dll.c | 1 - Python/_warnings.c | 11 +- Python/ceval.c | 2 +- Python/dynload_shlib.c | 2 +- Python/import.c | 2210 +++++++++++++++--------- Python/importdl.c | 17 +- Python/pylifecycle.c | 127 +- Python/pystate.c | 151 +- Python/pythonrun.c | 10 +- Python/sysmodule.c | 40 +- 20 files changed, 1586 insertions(+), 1182 deletions(-) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 9036dff6725330..da766253ef6b9c 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -36,11 +36,112 @@ struct _import_runtime_state { const char * pkgcontext; }; +struct _import_state { + /* cached sys.modules dictionary */ + PyObject *modules; + /* This is the list of module objects for all legacy (single-phase init) + extension modules ever loaded in this process (i.e. imported + in this interpreter or in any other). Py_None stands in for + modules that haven't actually been imported in this interpreter. + + A module's index (PyModuleDef.m_base.m_index) is used to look up + the corresponding module object for this interpreter, if any. + (See PyState_FindModule().) When any extension module + is initialized during import, its moduledef gets initialized by + PyModuleDef_Init(), and the first time that happens for each + PyModuleDef, its index gets set to the current value of + a global counter (see _PyRuntimeState.imports.last_module_index). + The entry for that index in this interpreter remains unset until + the module is actually imported here. (Py_None is used as + a placeholder.) Note that multi-phase init modules always get + an index for which there will never be a module set. + + This is initialized lazily in PyState_AddModule(), which is also + where modules get added. */ + PyObject *modules_by_index; + /* importlib module._bootstrap */ + PyObject *importlib; + /* override for config->use_frozen_modules (for tests) + (-1: "off", 1: "on", 0: no override) */ + int override_frozen_modules; +#ifdef HAVE_DLOPEN + int dlopenflags; +#endif + PyObject *import_func; +}; + +#ifdef HAVE_DLOPEN +# include +# if HAVE_DECL_RTLD_NOW +# define _Py_DLOPEN_FLAGS RTLD_NOW +# else +# define _Py_DLOPEN_FLAGS RTLD_LAZY +# endif +# define DLOPENFLAGS_INIT .dlopenflags = _Py_DLOPEN_FLAGS, +#else +# define _Py_DLOPEN_FLAGS 0 +# define DLOPENFLAGS_INIT +#endif + +#define IMPORTS_INIT \ + { \ + .override_frozen_modules = 0, \ + DLOPENFLAGS_INIT \ + } + +extern void _PyImport_ClearCore(PyInterpreterState *interp); + +extern Py_ssize_t _PyImport_GetNextModuleIndex(void); +extern const char * _PyImport_ResolveNameWithPackageContext(const char *name); +extern const char * _PyImport_SwapPackageContext(const char *newcontext); + +extern int _PyImport_GetDLOpenFlags(PyInterpreterState *interp); +extern void _PyImport_SetDLOpenFlags(PyInterpreterState *interp, int new_val); + +extern PyObject * _PyImport_InitModules(PyInterpreterState *interp); +extern PyObject * _PyImport_GetModules(PyInterpreterState *interp); +extern void _PyImport_ClearModules(PyInterpreterState *interp); + +extern void _PyImport_ClearModulesByIndex(PyInterpreterState *interp); + +extern int _PyImport_InitDefaultImportFunc(PyInterpreterState *interp); +extern int _PyImport_IsDefaultImportFunc( + PyInterpreterState *interp, + PyObject *func); + +extern PyObject * _PyImport_GetImportlibLoader( + PyInterpreterState *interp, + const char *loader_name); +extern PyObject * _PyImport_GetImportlibExternalLoader( + PyInterpreterState *interp, + const char *loader_name); +extern PyObject * _PyImport_BlessMyLoader( + PyInterpreterState *interp, + PyObject *module_globals); +extern PyObject * _PyImport_ImportlibModuleRepr( + PyInterpreterState *interp, + PyObject *module); + + +extern PyStatus _PyImport_Init(void); +extern void _PyImport_Fini(void); +extern void _PyImport_Fini2(void); + +extern PyStatus _PyImport_InitCore( + PyThreadState *tstate, + PyObject *sysmod, + int importlib); +extern PyStatus _PyImport_InitExternal(PyThreadState *tstate); +extern void _PyImport_FiniCore(PyInterpreterState *interp); +extern void _PyImport_FiniExternal(PyInterpreterState *interp); + #ifdef HAVE_FORK extern PyStatus _PyImport_ReInitLock(void); #endif -extern PyObject* _PyImport_BootstrapImp(PyThreadState *tstate); + + +extern PyObject* _PyImport_GetBuiltinModuleNames(void); struct _module_alias { const char *name; /* ASCII encoded string */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 0e3d46852f2e6d..60de31b336f613 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -21,6 +21,7 @@ extern "C" { #include "pycore_function.h" // FUNC_MAX_WATCHERS #include "pycore_genobject.h" // struct _Py_async_gen_state #include "pycore_gc.h" // struct _gc_runtime_state +#include "pycore_import.h" // struct _import_state #include "pycore_list.h" // struct _Py_list_state #include "pycore_global_objects.h" // struct _Py_interp_static_objects #include "pycore_tuple.h" // struct _Py_tuple_state @@ -92,37 +93,12 @@ struct _is { struct _ceval_state ceval; struct _gc_runtime_state gc; - // sys.modules dictionary - PyObject *modules; - /* This is the list of module objects for all legacy (single-phase init) - extension modules ever loaded in this process (i.e. imported - in this interpreter or in any other). Py_None stands in for - modules that haven't actually been imported in this interpreter. - - A module's index (PyModuleDef.m_base.m_index) is used to look up - the corresponding module object for this interpreter, if any. - (See PyState_FindModule().) When any extension module - is initialized during import, its moduledef gets initialized by - PyModuleDef_Init(), and the first time that happens for each - PyModuleDef, its index gets set to the current value of - a global counter (see _PyRuntimeState.imports.last_module_index). - The entry for that index in this interpreter remains unset until - the module is actually imported here. (Py_None is used as - a placeholder.) Note that multi-phase init modules always get - an index for which there will never be a module set. - - This is initialized lazily in _PyState_AddModule(), which is also - where modules get added. */ - PyObject *modules_by_index; + struct _import_state imports; + // Dictionary of the sys module PyObject *sysdict; // Dictionary of the builtins module PyObject *builtins; - // importlib module - PyObject *importlib; - // override for config->use_frozen_modules (for tests) - // (-1: "off", 1: "on", 0: no override) - int override_frozen_modules; PyObject *codec_search_path; PyObject *codec_search_cache; @@ -130,15 +106,11 @@ struct _is { int codecs_initialized; PyConfig config; -#ifdef HAVE_DLOPEN - int dlopenflags; -#endif unsigned long feature_flags; PyObject *dict; /* Stores per-interpreter state */ PyObject *builtins_copy; - PyObject *import_func; // Initialized to _PyEval_EvalFrameDefault(). _PyFrameEvalFunction eval_frame; @@ -205,7 +177,6 @@ struct _is { /* other API */ -extern void _PyInterpreterState_ClearModules(PyInterpreterState *interp); extern void _PyInterpreterState_Clear(PyThreadState *tstate); diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 2d431befd74f99..e7a31807205254 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -30,7 +30,6 @@ PyAPI_FUNC(int) _Py_IsLocaleCoercionTarget(const char *ctype_loc); /* Various one-time initializers */ extern void _Py_InitVersion(void); -extern PyStatus _PyImport_Init(void); extern PyStatus _PyFaulthandler_Init(int enable); extern int _PyTraceMalloc_Init(int enable); extern PyObject * _PyBuiltin_Init(PyInterpreterState *interp); @@ -45,7 +44,6 @@ extern int _PyBuiltins_AddExceptions(PyObject * bltinmod); extern PyStatus _Py_HashRandomization_Init(const PyConfig *); extern PyStatus _PyTime_Init(void); -extern PyStatus _PyImportZip_Init(PyThreadState *tstate); extern PyStatus _PyGC_Init(PyInterpreterState *interp); extern PyStatus _PyAtExit_Init(PyInterpreterState *interp); extern int _Py_Deepfreeze_Init(void); @@ -55,8 +53,6 @@ extern int _Py_Deepfreeze_Init(void); extern int _PySignal_Init(int install_signal_handlers); extern void _PySignal_Fini(void); -extern void _PyImport_Fini(void); -extern void _PyImport_Fini2(void); extern void _PyGC_Fini(PyInterpreterState *interp); extern void _Py_HashRandomization_Fini(void); extern void _PyFaulthandler_Fini(void); diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 7046ec8d9adaaf..638b86253879ea 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -152,12 +152,6 @@ extern void _PySignal_AfterFork(void); #endif -PyAPI_FUNC(int) _PyState_AddModule( - PyThreadState *tstate, - PyObject* module, - PyModuleDef* def); - - PyAPI_FUNC(int) _PyOS_InterruptOccurred(PyThreadState *tstate); #define HEAD_LOCK(runtime) \ diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index c6a27d076eae2d..a8d5953ff98b0b 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -97,23 +97,10 @@ extern "C" { ._main_interpreter = _PyInterpreterState_INIT, \ } -#ifdef HAVE_DLOPEN -# include -# if HAVE_DECL_RTLD_NOW -# define _Py_DLOPEN_FLAGS RTLD_NOW -# else -# define _Py_DLOPEN_FLAGS RTLD_LAZY -# endif -# define DLOPENFLAGS_INIT .dlopenflags = _Py_DLOPEN_FLAGS, -#else -# define _Py_DLOPEN_FLAGS 0 -# define DLOPENFLAGS_INIT -#endif - #define _PyInterpreterState_INIT \ { \ .id_refcount = -1, \ - DLOPENFLAGS_INIT \ + .imports = IMPORTS_INIT, \ .ceval = { \ .recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \ }, \ diff --git a/Include/internal/pycore_sysmodule.h b/Include/internal/pycore_sysmodule.h index 10d092cdc30a2c..b4b1febafa4479 100644 --- a/Include/internal/pycore_sysmodule.h +++ b/Include/internal/pycore_sysmodule.h @@ -20,6 +20,9 @@ extern void _PySys_ClearAuditHooks(PyThreadState *tstate); PyAPI_FUNC(int) _PySys_SetAttr(PyObject *, PyObject *); +extern int _PySys_ClearAttrString(PyInterpreterState *interp, + const char *name, int verbose); + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 31dce21587e2ca..c85ab92307de78 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -387,7 +387,7 @@ def check_with_reinit_reloaded(module, lookedup, initialized, check_basic_reloaded(mod, lookedup, initialized, init_count, before, reloaded) - # Currently _PyState_AddModule() always replaces the cached module. + # Currently PyState_AddModule() always replaces the cached module. self.assertIs(basic.look_up_self(), mod) self.assertEqual(basic.initialized_count(), expected_init_count) diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index e77c1c8409880d..7e50fbda2c07cb 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -864,7 +864,6 @@ def test_windows_feature_macros(self): "_PyObject_GC_Resize", "_PyObject_New", "_PyObject_NewVar", - "_PyState_AddModule", "_PyThreadState_Init", "_PyThreadState_Prealloc", "_PyWeakref_CallableProxyType", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 21ff9616133445..c04a3a228caf56 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -1684,9 +1684,6 @@ [function._PyObject_NewVar] added = '3.2' abi_only = true -[function._PyState_AddModule] - added = '3.2' - abi_only = true [function._PyThreadState_Init] added = '3.2' abi_only = true diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 24190e320ee6d6..a0be19a3ca8ac8 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -42,10 +42,9 @@ PyModuleDef_Init(PyModuleDef* def) { assert(PyModuleDef_Type.tp_flags & Py_TPFLAGS_READY); if (def->m_base.m_index == 0) { - _PyRuntime.imports.last_module_index++; Py_SET_REFCNT(def, 1); Py_SET_TYPE(def, &PyModuleDef_Type); - def->m_base.m_index = _PyRuntime.imports.last_module_index; + def->m_base.m_index = _PyImport_GetNextModuleIndex(); } return (PyObject*)def; } @@ -209,24 +208,7 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) "module %s: PyModule_Create is incompatible with m_slots", name); return NULL; } - /* Make sure name is fully qualified. - - This is a bit of a hack: when the shared library is loaded, - the module name is "package.module", but the module calls - PyModule_Create*() with just "module" for the name. The shared - library loader squirrels away the true name of the module in - _Py_PackageContext, and PyModule_Create*() will substitute this - (if the name actually matches). - */ -#define _Py_PackageContext (_PyRuntime.imports.pkgcontext) - if (_Py_PackageContext != NULL) { - const char *p = strrchr(_Py_PackageContext, '.'); - if (p != NULL && strcmp(module->m_name, p+1) == 0) { - name = _Py_PackageContext; - _Py_PackageContext = NULL; - } - } -#undef _Py_PackageContext + name = _PyImport_ResolveNameWithPackageContext(name); if ((m = (PyModuleObject*)PyModule_New(name)) == NULL) return NULL; @@ -710,8 +692,7 @@ static PyObject * module_repr(PyModuleObject *m) { PyInterpreterState *interp = _PyInterpreterState_GET(); - - return PyObject_CallMethod(interp->importlib, "_module_repr", "O", m); + return _PyImport_ImportlibModuleRepr(interp, (PyObject *)m); } /* Check if the "_initializing" attribute of the module spec is set to true. diff --git a/PC/python3dll.c b/PC/python3dll.c index e300819365756e..79f09037282f54 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -34,7 +34,6 @@ EXPORT_FUNC(_PyObject_GC_NewVar) EXPORT_FUNC(_PyObject_GC_Resize) EXPORT_FUNC(_PyObject_New) EXPORT_FUNC(_PyObject_NewVar) -EXPORT_FUNC(_PyState_AddModule) EXPORT_FUNC(_PyThreadState_Init) EXPORT_FUNC(_PyThreadState_Prealloc) EXPORT_FUNC(Py_AddPendingCall) diff --git a/Python/_warnings.c b/Python/_warnings.c index e78f21644f372b..d510381c365b66 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -214,7 +214,7 @@ get_warnings_attr(PyInterpreterState *interp, PyObject *attr, int try_import) gone, then we can't even use PyImport_GetModule without triggering an interpreter abort. */ - if (!interp->modules) { + if (!_PyImport_GetModules(interp)) { return NULL; } warnings_module = PyImport_GetModule(&_Py_ID(warnings)); @@ -1050,7 +1050,6 @@ warnings_warn_impl(PyObject *module, PyObject *message, PyObject *category, static PyObject * get_source_line(PyInterpreterState *interp, PyObject *module_globals, int lineno) { - PyObject *external; PyObject *loader; PyObject *module_name; PyObject *get_source; @@ -1059,13 +1058,7 @@ get_source_line(PyInterpreterState *interp, PyObject *module_globals, int lineno PyObject *source_line; /* stolen from import.c */ - external = PyObject_GetAttrString(interp->importlib, "_bootstrap_external"); - if (external == NULL) { - return NULL; - } - - loader = PyObject_CallMethod(external, "_bless_my_loader", "O", module_globals, NULL); - Py_DECREF(external); + loader = _PyImport_BlessMyLoader(interp, module_globals); if (loader == NULL) { return NULL; } diff --git a/Python/ceval.c b/Python/ceval.c index 611d62b0eba9af..09fd2f29266c87 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2688,7 +2688,7 @@ import_name(PyThreadState *tstate, _PyInterpreterFrame *frame, } PyObject *locals = frame->f_locals; /* Fast path for not overloaded __import__. */ - if (import_func == tstate->interp->import_func) { + if (_PyImport_IsDefaultImportFunc(tstate->interp, import_func)) { int ilevel = _PyLong_AsInt(level); if (ilevel == -1 && _PyErr_Occurred(tstate)) { return NULL; diff --git a/Python/dynload_shlib.c b/Python/dynload_shlib.c index 3c5fd83df584d5..6761bba457983b 100644 --- a/Python/dynload_shlib.c +++ b/Python/dynload_shlib.c @@ -75,7 +75,7 @@ _PyImport_FindSharedFuncptr(const char *prefix, return NULL; } - dlopenflags = _PyInterpreterState_GET()->dlopenflags; + dlopenflags = _PyImport_GetDLOpenFlags(_PyInterpreterState_GET()); handle = dlopen(pathname, dlopenflags); diff --git a/Python/import.c b/Python/import.c index 63ed2443657b29..ae27aaf56848d6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -4,7 +4,7 @@ #include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_initconfig.h" // _PyStatus_OK() -#include "pycore_interp.h" // _PyInterpreterState_ClearModules() +#include "pycore_interp.h" // struct _import_runtime_state #include "pycore_namespace.h" // _PyNamespace_Type #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_pyhash.h" // _Py_KeyedHash() @@ -24,8 +24,18 @@ extern "C" { #endif -/* Forward references */ -static PyObject *import_add_module(PyThreadState *tstate, PyObject *name); + +/*[clinic input] +module _imp +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=9c332475d8686284]*/ + +#include "clinic/import.c.h" + + +/*******************************/ +/* process-global import state */ +/*******************************/ /* This table is defined in config.c: */ extern struct _inittab _PyImport_Inittab[]; @@ -37,67 +47,51 @@ struct _inittab *PyImport_Inittab = _PyImport_Inittab; // we track the pointer here so we can deallocate it during finalization. static struct _inittab *inittab_copy = NULL; -/*[clinic input] -module _imp -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=9c332475d8686284]*/ - -#include "clinic/import.c.h" - -/* Initialize things */ -PyStatus -_PyImportZip_Init(PyThreadState *tstate) -{ - PyObject *path_hooks; - int err = 0; +/*******************************/ +/* runtime-global import state */ +/*******************************/ - path_hooks = PySys_GetObject("path_hooks"); - if (path_hooks == NULL) { - _PyErr_SetString(tstate, PyExc_RuntimeError, - "unable to get sys.path_hooks"); - goto error; - } +#define INITTAB _PyRuntime.imports.inittab +#define LAST_MODULE_INDEX _PyRuntime.imports.last_module_index +#define EXTENSIONS _PyRuntime.imports.extensions - int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose; - if (verbose) { - PySys_WriteStderr("# installing zipimport hook\n"); - } +#define import_lock _PyRuntime.imports.lock.mutex +#define import_lock_thread _PyRuntime.imports.lock.thread +#define import_lock_level _PyRuntime.imports.lock.level - PyObject *zipimporter = _PyImport_GetModuleAttrString("zipimport", "zipimporter"); - if (zipimporter == NULL) { - _PyErr_Clear(tstate); /* No zipimporter object -- okay */ - if (verbose) { - PySys_WriteStderr("# can't import zipimport.zipimporter\n"); - } - } - else { - /* sys.path_hooks.insert(0, zipimporter) */ - err = PyList_Insert(path_hooks, 0, zipimporter); - Py_DECREF(zipimporter); - if (err < 0) { - goto error; - } - if (verbose) { - PySys_WriteStderr("# installed zipimport hook\n"); - } - } +#define FIND_AND_LOAD _PyRuntime.imports.find_and_load +#define PKGCONTEXT (_PyRuntime.imports.pkgcontext) + + +/*******************************/ +/* interpreter import state */ +/*******************************/ + +#define MODULES(interp) \ + (interp)->imports.modules +#define MODULES_BY_INDEX(interp) \ + (interp)->imports.modules_by_index +#define IMPORTLIB(interp) \ + (interp)->imports.importlib +#define OVERRIDE_FROZEN_MODULES(interp) \ + (interp)->imports.override_frozen_modules +#ifdef HAVE_DLOPEN +# define DLOPENFLAGS(interp) \ + (interp)->imports.dlopenflags +#endif +#define IMPORT_FUNC(interp) \ + (interp)->imports.import_func - return _PyStatus_OK(); - error: - PyErr_Print(); - return _PyStatus_ERR("initializing zipimport failed"); -} +/*******************/ +/* the import lock */ +/*******************/ /* Locking primitives to prevent parallel imports of the same module in different threads to return with a partially loaded module. These calls are serialized by the global interpreter lock. */ -#define import_lock _PyRuntime.imports.lock.mutex -#define import_lock_thread _PyRuntime.imports.lock.thread -#define import_lock_level _PyRuntime.imports.lock.level - void _PyImport_AcquireLock(void) { @@ -170,154 +164,45 @@ _PyImport_ReInitLock(void) } #endif -/*[clinic input] -_imp.lock_held - -Return True if the import lock is currently held, else False. - -On platforms without threads, return False. -[clinic start generated code]*/ - -static PyObject * -_imp_lock_held_impl(PyObject *module) -/*[clinic end generated code: output=8b89384b5e1963fc input=9b088f9b217d9bdf]*/ -{ - return PyBool_FromLong(import_lock_thread != PYTHREAD_INVALID_THREAD_ID); -} - -/*[clinic input] -_imp.acquire_lock - -Acquires the interpreter's import lock for the current thread. - -This lock should be used by import hooks to ensure thread-safety when importing -modules. On platforms without threads, this function does nothing. -[clinic start generated code]*/ - -static PyObject * -_imp_acquire_lock_impl(PyObject *module) -/*[clinic end generated code: output=1aff58cb0ee1b026 input=4a2d4381866d5fdc]*/ -{ - _PyImport_AcquireLock(); - Py_RETURN_NONE; -} - -/*[clinic input] -_imp.release_lock - -Release the interpreter's import lock. -On platforms without threads, this function does nothing. -[clinic start generated code]*/ +/***************/ +/* sys.modules */ +/***************/ -static PyObject * -_imp_release_lock_impl(PyObject *module) -/*[clinic end generated code: output=7faab6d0be178b0a input=934fb11516dd778b]*/ +PyObject * +_PyImport_InitModules(PyInterpreterState *interp) { - if (_PyImport_ReleaseLock() < 0) { - PyErr_SetString(PyExc_RuntimeError, - "not holding the import lock"); + assert(MODULES(interp) == NULL); + MODULES(interp) = PyDict_New(); + if (MODULES(interp) == NULL) { return NULL; } - Py_RETURN_NONE; -} - -PyStatus -_PyImport_Init(void) -{ - if (_PyRuntime.imports.inittab != NULL) { - return _PyStatus_ERR("global import state already initialized"); - } - PyStatus status = _PyStatus_OK(); - - size_t size; - for (size = 0; PyImport_Inittab[size].name != NULL; size++) - ; - size++; - - /* Force default raw memory allocator to get a known allocator to be able - to release the memory in _PyImport_Fini() */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - /* Make the copy. */ - struct _inittab *copied = PyMem_RawMalloc(size * sizeof(struct _inittab)); - if (copied == NULL) { - status = PyStatus_NoMemory(); - goto done; - } - memcpy(copied, PyImport_Inittab, size * sizeof(struct _inittab)); - _PyRuntime.imports.inittab = copied; - -done: - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - return status; + return MODULES(interp); } -static inline void _extensions_cache_clear(void); - -void -_PyImport_Fini(void) +PyObject * +_PyImport_GetModules(PyInterpreterState *interp) { - _extensions_cache_clear(); - if (import_lock != NULL) { - PyThread_free_lock(import_lock); - import_lock = NULL; - } - - /* Use the same memory allocator as _PyImport_Init(). */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - /* Free memory allocated by _PyImport_Init() */ - struct _inittab *inittab = _PyRuntime.imports.inittab; - _PyRuntime.imports.inittab = NULL; - PyMem_RawFree(inittab); - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + return MODULES(interp); } void -_PyImport_Fini2(void) +_PyImport_ClearModules(PyInterpreterState *interp) { - /* Use the same memory allocator than PyImport_ExtendInittab(). */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - // Reset PyImport_Inittab - PyImport_Inittab = _PyImport_Inittab; - - /* Free memory allocated by PyImport_ExtendInittab() */ - PyMem_RawFree(inittab_copy); - inittab_copy = NULL; - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + Py_SETREF(MODULES(interp), NULL); } -/* Helper for sys */ - PyObject * PyImport_GetModuleDict(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->modules == NULL) { + if (MODULES(interp) == NULL) { Py_FatalError("interpreter has no modules dictionary"); } - return interp->modules; -} - -/* In some corner cases it is important to be sure that the import - machinery has been initialized (or not cleaned up yet). For - example, see issue #4236 and PyModule_Create2(). */ - -int -_PyImport_IsInitialized(PyInterpreterState *interp) -{ - if (interp->modules == NULL) - return 0; - return 1; + return MODULES(interp); } +// This is only kept around for extensions that use _Py_IDENTIFIER. PyObject * _PyImport_GetModuleId(_Py_Identifier *nameid) { @@ -332,7 +217,7 @@ int _PyImport_SetModule(PyObject *name, PyObject *m) { PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *modules = interp->modules; + PyObject *modules = MODULES(interp); return PyObject_SetItem(modules, name, m); } @@ -340,14 +225,14 @@ int _PyImport_SetModuleString(const char *name, PyObject *m) { PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *modules = interp->modules; + PyObject *modules = MODULES(interp); return PyMapping_SetItemString(modules, name, m); } static PyObject * import_get_module(PyThreadState *tstate, PyObject *name) { - PyObject *modules = tstate->interp->modules; + PyObject *modules = MODULES(tstate->interp); if (modules == NULL) { _PyErr_SetString(tstate, PyExc_RuntimeError, "unable to get sys.modules"); @@ -370,7 +255,6 @@ import_get_module(PyThreadState *tstate, PyObject *name) return m; } - static int import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *name) { @@ -387,7 +271,7 @@ import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *n if (busy) { /* Wait until module is done importing. */ PyObject *value = _PyObject_CallMethodOneArg( - interp->importlib, &_Py_ID(_lock_unlock_module), name); + IMPORTLIB(interp), &_Py_ID(_lock_unlock_module), name); if (value == NULL) { return -1; } @@ -396,47 +280,381 @@ import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *n return 0; } +static void remove_importlib_frames(PyThreadState *tstate); -/* Helper for pythonrun.c -- return magic number and tag. */ - -long -PyImport_GetMagicNumber(void) +PyObject * +PyImport_GetModule(PyObject *name) { - long res; - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *external, *pyc_magic; + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *mod; - external = PyObject_GetAttrString(interp->importlib, "_bootstrap_external"); - if (external == NULL) - return -1; - pyc_magic = PyObject_GetAttrString(external, "_RAW_MAGIC_NUMBER"); - Py_DECREF(external); - if (pyc_magic == NULL) - return -1; - res = PyLong_AsLong(pyc_magic); - Py_DECREF(pyc_magic); - return res; + mod = import_get_module(tstate, name); + if (mod != NULL && mod != Py_None) { + if (import_ensure_initialized(tstate->interp, mod, name) < 0) { + Py_DECREF(mod); + remove_importlib_frames(tstate); + return NULL; + } + } + return mod; } +/* Get the module object corresponding to a module name. + First check the modules dictionary if there's one there, + if not, create a new one and insert it in the modules dictionary. */ -extern const char * _PySys_ImplCacheTag; - -const char * -PyImport_GetMagicTag(void) +static PyObject * +import_add_module(PyThreadState *tstate, PyObject *name) { - return _PySys_ImplCacheTag; -} + PyObject *modules = MODULES(tstate->interp); + if (modules == NULL) { + _PyErr_SetString(tstate, PyExc_RuntimeError, + "no import module dictionary"); + return NULL; + } + PyObject *m; + if (PyDict_CheckExact(modules)) { + m = Py_XNewRef(PyDict_GetItemWithError(modules, name)); + } + else { + m = PyObject_GetItem(modules, name); + // For backward-compatibility we copy the behavior + // of PyDict_GetItemWithError(). + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + _PyErr_Clear(tstate); + } + } + if (_PyErr_Occurred(tstate)) { + return NULL; + } + if (m != NULL && PyModule_Check(m)) { + return m; + } + Py_XDECREF(m); + m = PyModule_NewObject(name); + if (m == NULL) + return NULL; + if (PyObject_SetItem(modules, name, m) != 0) { + Py_DECREF(m); + return NULL; + } -/* -We support a number of kinds of single-phase init builtin/extension modules: + return m; +} -* "basic" - * no module state (PyModuleDef.m_size == -1) - * does not support repeated init (we use PyModuleDef.m_base.m_copy) - * may have process-global state - * the module's def is cached in _PyRuntime.imports.extensions, - by (name, filename) +PyObject * +PyImport_AddModuleObject(PyObject *name) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *mod = import_add_module(tstate, name); + if (mod) { + PyObject *ref = PyWeakref_NewRef(mod, NULL); + Py_DECREF(mod); + if (ref == NULL) { + return NULL; + } + mod = PyWeakref_GetObject(ref); + Py_DECREF(ref); + } + return mod; /* borrowed reference */ +} + + +PyObject * +PyImport_AddModule(const char *name) +{ + PyObject *nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) { + return NULL; + } + PyObject *module = PyImport_AddModuleObject(nameobj); + Py_DECREF(nameobj); + return module; +} + + +/* Remove name from sys.modules, if it's there. + * Can be called with an exception raised. + * If fail to remove name a new exception will be chained with the old + * exception, otherwise the old exception is preserved. + */ +static void +remove_module(PyThreadState *tstate, PyObject *name) +{ + PyObject *type, *value, *traceback; + _PyErr_Fetch(tstate, &type, &value, &traceback); + + PyObject *modules = MODULES(tstate->interp); + if (PyDict_CheckExact(modules)) { + PyObject *mod = _PyDict_Pop(modules, name, Py_None); + Py_XDECREF(mod); + } + else if (PyMapping_DelItem(modules, name) < 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + _PyErr_Clear(tstate); + } + } + + _PyErr_ChainExceptions(type, value, traceback); +} + + +/************************************/ +/* per-interpreter modules-by-index */ +/************************************/ + +Py_ssize_t +_PyImport_GetNextModuleIndex(void) +{ + LAST_MODULE_INDEX++; + return LAST_MODULE_INDEX; +} + +static const char * +_modules_by_index_check(PyInterpreterState *interp, Py_ssize_t index) +{ + if (index == 0) { + return "invalid module index"; + } + if (MODULES_BY_INDEX(interp) == NULL) { + return "Interpreters module-list not accessible."; + } + if (index > PyList_GET_SIZE(MODULES_BY_INDEX(interp))) { + return "Module index out of bounds."; + } + return NULL; +} + +static PyObject * +_modules_by_index_get(PyInterpreterState *interp, PyModuleDef *def) +{ + Py_ssize_t index = def->m_base.m_index; + if (_modules_by_index_check(interp, index) != NULL) { + return NULL; + } + PyObject *res = PyList_GET_ITEM(MODULES_BY_INDEX(interp), index); + return res==Py_None ? NULL : res; +} + +static int +_modules_by_index_set(PyInterpreterState *interp, + PyModuleDef *def, PyObject *module) +{ + assert(def != NULL); + assert(def->m_slots == NULL); + assert(def->m_base.m_index > 0); + + if (MODULES_BY_INDEX(interp) == NULL) { + MODULES_BY_INDEX(interp) = PyList_New(0); + if (MODULES_BY_INDEX(interp) == NULL) { + return -1; + } + } + + Py_ssize_t index = def->m_base.m_index; + while (PyList_GET_SIZE(MODULES_BY_INDEX(interp)) <= index) { + if (PyList_Append(MODULES_BY_INDEX(interp), Py_None) < 0) { + return -1; + } + } + + return PyList_SetItem(MODULES_BY_INDEX(interp), index, Py_NewRef(module)); +} + +static int +_modules_by_index_clear(PyInterpreterState *interp, PyModuleDef *def) +{ + Py_ssize_t index = def->m_base.m_index; + const char *err = _modules_by_index_check(interp, index); + if (err != NULL) { + Py_FatalError(err); + return -1; + } + return PyList_SetItem(MODULES_BY_INDEX(interp), index, Py_NewRef(Py_None)); +} + + +PyObject* +PyState_FindModule(PyModuleDef* module) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (module->m_slots) { + return NULL; + } + return _modules_by_index_get(interp, module); +} + +int +PyState_AddModule(PyObject* module, PyModuleDef* def) +{ + if (!def) { + Py_FatalError("module definition is NULL"); + return -1; + } + + PyThreadState *tstate = _PyThreadState_GET(); + if (def->m_slots) { + _PyErr_SetString(tstate, + PyExc_SystemError, + "PyState_AddModule called on module with slots"); + return -1; + } + + PyInterpreterState *interp = tstate->interp; + Py_ssize_t index = def->m_base.m_index; + if (MODULES_BY_INDEX(interp) && + index < PyList_GET_SIZE(MODULES_BY_INDEX(interp)) && + module == PyList_GET_ITEM(MODULES_BY_INDEX(interp), index)) + { + _Py_FatalErrorFormat(__func__, "module %p already added", module); + return -1; + } + + return _modules_by_index_set(interp, def, module); +} + +int +PyState_RemoveModule(PyModuleDef* def) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (def->m_slots) { + _PyErr_SetString(tstate, + PyExc_SystemError, + "PyState_RemoveModule called on module with slots"); + return -1; + } + return _modules_by_index_clear(tstate->interp, def); +} + + +// Used by finalize_modules() +void +_PyImport_ClearModulesByIndex(PyInterpreterState *interp) +{ + if (!MODULES_BY_INDEX(interp)) { + return; + } + + Py_ssize_t i; + for (i = 0; i < PyList_GET_SIZE(MODULES_BY_INDEX(interp)); i++) { + PyObject *m = PyList_GET_ITEM(MODULES_BY_INDEX(interp), i); + if (PyModule_Check(m)) { + /* cleanup the saved copy of module dicts */ + PyModuleDef *md = PyModule_GetDef(m); + if (md) { + Py_CLEAR(md->m_base.m_copy); + } + } + } + + /* Setting modules_by_index to NULL could be dangerous, so we + clear the list instead. */ + if (PyList_SetSlice(MODULES_BY_INDEX(interp), + 0, PyList_GET_SIZE(MODULES_BY_INDEX(interp)), + NULL)) { + PyErr_WriteUnraisable(MODULES_BY_INDEX(interp)); + } +} + + +/*********************/ +/* extension modules */ +/*********************/ + +/* Make sure name is fully qualified. + + This is a bit of a hack: when the shared library is loaded, + the module name is "package.module", but the module calls + PyModule_Create*() with just "module" for the name. The shared + library loader squirrels away the true name of the module in + _PyRuntime.imports.pkgcontext, and PyModule_Create*() will + substitute this (if the name actually matches). +*/ +const char * +_PyImport_ResolveNameWithPackageContext(const char *name) +{ + if (PKGCONTEXT != NULL) { + const char *p = strrchr(PKGCONTEXT, '.'); + if (p != NULL && strcmp(name, p+1) == 0) { + name = PKGCONTEXT; + PKGCONTEXT = NULL; + } + } + return name; +} + +const char * +_PyImport_SwapPackageContext(const char *newcontext) +{ + const char *oldcontext = PKGCONTEXT; + PKGCONTEXT = newcontext; + return oldcontext; +} + +#ifdef HAVE_DLOPEN +int +_PyImport_GetDLOpenFlags(PyInterpreterState *interp) +{ + return DLOPENFLAGS(interp); +} + +void +_PyImport_SetDLOpenFlags(PyInterpreterState *interp, int new_val) +{ + DLOPENFLAGS(interp) = new_val; +} +#endif // HAVE_DLOPEN + + +/* Common implementation for _imp.exec_dynamic and _imp.exec_builtin */ +static int +exec_builtin_or_dynamic(PyObject *mod) { + PyModuleDef *def; + void *state; + + if (!PyModule_Check(mod)) { + return 0; + } + + def = PyModule_GetDef(mod); + if (def == NULL) { + return 0; + } + + state = PyModule_GetState(mod); + if (state) { + /* Already initialized; skip reload */ + return 0; + } + + return PyModule_ExecDef(mod, def); +} + + +/*******************/ + +#if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) +#include +EM_JS(PyObject*, _PyImport_InitFunc_TrampolineCall, (PyModInitFunction func), { + return wasmTable.get(func)(); +}); +#endif // __EMSCRIPTEN__ && PY_CALL_TRAMPOLINE + + +/*****************************/ +/* single-phase init modules */ +/*****************************/ + +/* +We support a number of kinds of single-phase init builtin/extension modules: + +* "basic" + * no module state (PyModuleDef.m_size == -1) + * does not support repeated init (we use PyModuleDef.m_base.m_copy) + * may have process-global state + * the module's def is cached in _PyRuntime.imports.extensions, + by (name, filename) * "reinit" * no module state (PyModuleDef.m_size == 0) * supports repeated init (m_copy is never used) @@ -512,7 +730,7 @@ gets even messier. static PyModuleDef * _extensions_cache_get(PyObject *filename, PyObject *name) { - PyObject *extensions = _PyRuntime.imports.extensions; + PyObject *extensions = EXTENSIONS; if (extensions == NULL) { return NULL; } @@ -528,13 +746,13 @@ _extensions_cache_get(PyObject *filename, PyObject *name) static int _extensions_cache_set(PyObject *filename, PyObject *name, PyModuleDef *def) { - PyObject *extensions = _PyRuntime.imports.extensions; + PyObject *extensions = EXTENSIONS; if (extensions == NULL) { extensions = PyDict_New(); if (extensions == NULL) { return -1; } - _PyRuntime.imports.extensions = extensions; + EXTENSIONS = extensions; } PyObject *key = PyTuple_Pack(2, filename, name); if (key == NULL) { @@ -551,7 +769,7 @@ _extensions_cache_set(PyObject *filename, PyObject *name, PyModuleDef *def) static void _extensions_cache_clear(void) { - Py_CLEAR(_PyRuntime.imports.extensions); + Py_CLEAR(EXTENSIONS); } static int @@ -569,7 +787,7 @@ fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) } PyThreadState *tstate = _PyThreadState_GET(); - if (_PyState_AddModule(tstate, mod, def) < 0) { + if (_modules_by_index_set(tstate->interp, def, mod) < 0) { return -1; } @@ -616,40 +834,19 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, return 0; } -int -_PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules) -{ - int res = -1; - PyObject *nameobj; - nameobj = PyUnicode_InternFromString(name); - if (nameobj == NULL) { - return -1; - } - if (PyObject_SetItem(modules, nameobj, mod) < 0) { - goto finally; - } - if (fix_up_extension(mod, nameobj, nameobj) < 0) { - PyMapping_DelItem(modules, nameobj); - goto finally; - } - res = 0; - -finally: - Py_DECREF(nameobj); - return res; -} static PyObject * import_find_extension(PyThreadState *tstate, PyObject *name, PyObject *filename) { + /* Only single-phase init modules will be in the cache. */ PyModuleDef *def = _extensions_cache_get(filename, name); if (def == NULL) { return NULL; } PyObject *mod, *mdict; - PyObject *modules = tstate->interp->modules; + PyObject *modules = MODULES(tstate->interp); if (def->m_size == -1) { /* Module does not support repeated initialization */ @@ -679,7 +876,7 @@ import_find_extension(PyThreadState *tstate, PyObject *name, return NULL; } } - if (_PyState_AddModule(tstate, mod, def) < 0) { + if (_modules_by_index_set(tstate->interp, def, mod) < 0) { PyMapping_DelItem(modules, name); Py_DECREF(mod); return NULL; @@ -694,107 +891,268 @@ import_find_extension(PyThreadState *tstate, PyObject *name, } -/* Get the module object corresponding to a module name. - First check the modules dictionary if there's one there, - if not, create a new one and insert it in the modules dictionary. */ +/*******************/ +/* builtin modules */ +/*******************/ -static PyObject * -import_add_module(PyThreadState *tstate, PyObject *name) +int +_PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules) { - PyObject *modules = tstate->interp->modules; - if (modules == NULL) { - _PyErr_SetString(tstate, PyExc_RuntimeError, - "no import module dictionary"); - return NULL; + int res = -1; + PyObject *nameobj; + nameobj = PyUnicode_InternFromString(name); + if (nameobj == NULL) { + return -1; } - - PyObject *m; - if (PyDict_CheckExact(modules)) { - m = Py_XNewRef(PyDict_GetItemWithError(modules, name)); + if (PyObject_SetItem(modules, nameobj, mod) < 0) { + goto finally; } - else { - m = PyObject_GetItem(modules, name); - // For backward-compatibility we copy the behavior - // of PyDict_GetItemWithError(). - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - _PyErr_Clear(tstate); - } + if (fix_up_extension(mod, nameobj, nameobj) < 0) { + PyMapping_DelItem(modules, nameobj); + goto finally; } - if (_PyErr_Occurred(tstate)) { - return NULL; + res = 0; + +finally: + Py_DECREF(nameobj); + return res; +} + +/* Helper to test for built-in module */ + +static int +is_builtin(PyObject *name) +{ + int i; + struct _inittab *inittab = INITTAB; + for (i = 0; inittab[i].name != NULL; i++) { + if (_PyUnicode_EqualToASCIIString(name, inittab[i].name)) { + if (inittab[i].initfunc == NULL) + return -1; + else + return 1; + } } - if (m != NULL && PyModule_Check(m)) { - return m; + return 0; +} + +static PyObject* +create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) +{ + PyObject *mod = import_find_extension(tstate, name, name); + if (mod || _PyErr_Occurred(tstate)) { + return mod; } - Py_XDECREF(m); - m = PyModule_NewObject(name); - if (m == NULL) - return NULL; - if (PyObject_SetItem(modules, name, m) != 0) { - Py_DECREF(m); - return NULL; + + PyObject *modules = MODULES(tstate->interp); + for (struct _inittab *p = INITTAB; p->name != NULL; p++) { + if (_PyUnicode_EqualToASCIIString(name, p->name)) { + if (p->initfunc == NULL) { + /* Cannot re-init internal module ("sys" or "builtins") */ + mod = PyImport_AddModuleObject(name); + return Py_XNewRef(mod); + } + mod = _PyImport_InitFunc_TrampolineCall(*p->initfunc); + if (mod == NULL) { + return NULL; + } + + if (PyObject_TypeCheck(mod, &PyModuleDef_Type)) { + return PyModule_FromDefAndSpec((PyModuleDef*)mod, spec); + } + else { + /* Remember pointer to module init function. */ + PyModuleDef *def = PyModule_GetDef(mod); + if (def == NULL) { + return NULL; + } + + def->m_base.m_init = p->initfunc; + if (_PyImport_FixupExtensionObject(mod, name, name, + modules) < 0) { + return NULL; + } + return mod; + } + } } - return m; + // not found + Py_RETURN_NONE; } -PyObject * -PyImport_AddModuleObject(PyObject *name) + +/*****************************/ +/* the builtin modules table */ +/*****************************/ + +/* API for embedding applications that want to add their own entries + to the table of built-in modules. This should normally be called + *before* Py_Initialize(). When the table resize fails, -1 is + returned and the existing table is unchanged. + + After a similar function by Just van Rossum. */ + +int +PyImport_ExtendInittab(struct _inittab *newtab) { - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *mod = import_add_module(tstate, name); - if (mod) { - PyObject *ref = PyWeakref_NewRef(mod, NULL); - Py_DECREF(mod); - if (ref == NULL) { - return NULL; - } - mod = PyWeakref_GetObject(ref); - Py_DECREF(ref); + struct _inittab *p; + size_t i, n; + int res = 0; + + if (INITTAB != NULL) { + Py_FatalError("PyImport_ExtendInittab() may not be called after Py_Initialize()"); } - return mod; /* borrowed reference */ + + /* Count the number of entries in both tables */ + for (n = 0; newtab[n].name != NULL; n++) + ; + if (n == 0) + return 0; /* Nothing to do */ + for (i = 0; PyImport_Inittab[i].name != NULL; i++) + ; + + /* Force default raw memory allocator to get a known allocator to be able + to release the memory in _PyImport_Fini2() */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + /* Allocate new memory for the combined table */ + p = NULL; + if (i + n <= SIZE_MAX / sizeof(struct _inittab) - 1) { + size_t size = sizeof(struct _inittab) * (i + n + 1); + p = PyMem_RawRealloc(inittab_copy, size); + } + if (p == NULL) { + res = -1; + goto done; + } + + /* Copy the tables into the new memory at the first call + to PyImport_ExtendInittab(). */ + if (inittab_copy != PyImport_Inittab) { + memcpy(p, PyImport_Inittab, (i+1) * sizeof(struct _inittab)); + } + memcpy(p + i, newtab, (n + 1) * sizeof(struct _inittab)); + PyImport_Inittab = inittab_copy = p; + +done: + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + return res; } +/* Shorthand to add a single entry given a name and a function */ -PyObject * -PyImport_AddModule(const char *name) +int +PyImport_AppendInittab(const char *name, PyObject* (*initfunc)(void)) { - PyObject *nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) { - return NULL; + struct _inittab newtab[2]; + + if (INITTAB != NULL) { + Py_FatalError("PyImport_AppendInittab() may not be called after Py_Initialize()"); } - PyObject *module = PyImport_AddModuleObject(nameobj); - Py_DECREF(nameobj); - return module; + + memset(newtab, '\0', sizeof newtab); + + newtab[0].name = name; + newtab[0].initfunc = initfunc; + + return PyImport_ExtendInittab(newtab); } -/* Remove name from sys.modules, if it's there. - * Can be called with an exception raised. - * If fail to remove name a new exception will be chained with the old - * exception, otherwise the old exception is preserved. - */ +/* the internal table */ + +static int +init_builtin_modules_table(void) +{ + size_t size; + for (size = 0; PyImport_Inittab[size].name != NULL; size++) + ; + size++; + + /* Make the copy. */ + struct _inittab *copied = PyMem_RawMalloc(size * sizeof(struct _inittab)); + if (copied == NULL) { + return -1; + } + memcpy(copied, PyImport_Inittab, size * sizeof(struct _inittab)); + INITTAB = copied; + return 0; +} + static void -remove_module(PyThreadState *tstate, PyObject *name) +fini_builtin_modules_table(void) { - PyObject *type, *value, *traceback; - _PyErr_Fetch(tstate, &type, &value, &traceback); + struct _inittab *inittab = INITTAB; + INITTAB = NULL; + PyMem_RawFree(inittab); +} - PyObject *modules = tstate->interp->modules; - if (PyDict_CheckExact(modules)) { - PyObject *mod = _PyDict_Pop(modules, name, Py_None); - Py_XDECREF(mod); +PyObject * +_PyImport_GetBuiltinModuleNames(void) +{ + PyObject *list = PyList_New(0); + if (list == NULL) { + return NULL; } - else if (PyMapping_DelItem(modules, name) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - _PyErr_Clear(tstate); + struct _inittab *inittab = INITTAB; + for (Py_ssize_t i = 0; inittab[i].name != NULL; i++) { + PyObject *name = PyUnicode_FromString(inittab[i].name); + if (name == NULL) { + Py_DECREF(list); + return NULL; } + if (PyList_Append(list, name) < 0) { + Py_DECREF(name); + Py_DECREF(list); + return NULL; + } + Py_DECREF(name); } + return list; +} - _PyErr_ChainExceptions(type, value, traceback); + +/********************/ +/* the magic number */ +/********************/ + +/* Helper for pythonrun.c -- return magic number and tag. */ + +long +PyImport_GetMagicNumber(void) +{ + long res; + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *external, *pyc_magic; + + external = PyObject_GetAttrString(IMPORTLIB(interp), "_bootstrap_external"); + if (external == NULL) + return -1; + pyc_magic = PyObject_GetAttrString(external, "_RAW_MAGIC_NUMBER"); + Py_DECREF(external); + if (pyc_magic == NULL) + return -1; + res = PyLong_AsLong(pyc_magic); + Py_DECREF(pyc_magic); + return res; +} + + +extern const char * _PySys_ImplCacheTag; + +const char * +PyImport_GetMagicTag(void) +{ + return _PySys_ImplCacheTag; } +/*********************************/ +/* a Python module's code object */ +/*********************************/ + /* Execute a code object in a module and return the module object * WITH INCREMENTED REFERENCE COUNT. If an error occurs, name is * removed from sys.modules, to avoid leaving damaged module objects @@ -851,7 +1209,7 @@ PyImport_ExecCodeModuleWithPathnames(const char *name, PyObject *co, Py_FatalError("no current interpreter"); } - external= PyObject_GetAttrString(interp->importlib, + external= PyObject_GetAttrString(IMPORTLIB(interp), "_bootstrap_external"); if (external != NULL) { pathobj = _PyObject_CallMethodOneArg( @@ -936,7 +1294,7 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, if (pathname == NULL) { pathname = ((PyCodeObject *)co)->co_filename; } - external = PyObject_GetAttrString(tstate->interp->importlib, + external = PyObject_GetAttrString(IMPORTLIB(tstate->interp), "_bootstrap_external"); if (external == NULL) { Py_DECREF(d); @@ -989,204 +1347,10 @@ update_compiled_module(PyCodeObject *co, PyObject *newname) Py_DECREF(oldname); } -/*[clinic input] -_imp._fix_co_filename - - code: object(type="PyCodeObject *", subclass_of="&PyCode_Type") - Code object to change. - - path: unicode - File path to use. - / - -Changes code.co_filename to specify the passed-in file path. -[clinic start generated code]*/ - -static PyObject * -_imp__fix_co_filename_impl(PyObject *module, PyCodeObject *code, - PyObject *path) -/*[clinic end generated code: output=1d002f100235587d input=895ba50e78b82f05]*/ - -{ - update_compiled_module(code, path); - - Py_RETURN_NONE; -} - - -/* Helper to test for built-in module */ - -static int -is_builtin(PyObject *name) -{ - int i; - struct _inittab *inittab = _PyRuntime.imports.inittab; - for (i = 0; inittab[i].name != NULL; i++) { - if (_PyUnicode_EqualToASCIIString(name, inittab[i].name)) { - if (inittab[i].initfunc == NULL) - return -1; - else - return 1; - } - } - return 0; -} - - -/* Return a finder object for a sys.path/pkg.__path__ item 'p', - possibly by fetching it from the path_importer_cache dict. If it - wasn't yet cached, traverse path_hooks until a hook is found - that can handle the path item. Return None if no hook could; - this tells our caller that the path based finder could not find - a finder for this path item. Cache the result in - path_importer_cache. */ - -static PyObject * -get_path_importer(PyThreadState *tstate, PyObject *path_importer_cache, - PyObject *path_hooks, PyObject *p) -{ - PyObject *importer; - Py_ssize_t j, nhooks; - - /* These conditions are the caller's responsibility: */ - assert(PyList_Check(path_hooks)); - assert(PyDict_Check(path_importer_cache)); - - nhooks = PyList_Size(path_hooks); - if (nhooks < 0) - return NULL; /* Shouldn't happen */ - - importer = PyDict_GetItemWithError(path_importer_cache, p); - if (importer != NULL || _PyErr_Occurred(tstate)) { - return Py_XNewRef(importer); - } - - /* set path_importer_cache[p] to None to avoid recursion */ - if (PyDict_SetItem(path_importer_cache, p, Py_None) != 0) - return NULL; - - for (j = 0; j < nhooks; j++) { - PyObject *hook = PyList_GetItem(path_hooks, j); - if (hook == NULL) - return NULL; - importer = PyObject_CallOneArg(hook, p); - if (importer != NULL) - break; - - if (!_PyErr_ExceptionMatches(tstate, PyExc_ImportError)) { - return NULL; - } - _PyErr_Clear(tstate); - } - if (importer == NULL) { - Py_RETURN_NONE; - } - if (PyDict_SetItem(path_importer_cache, p, importer) < 0) { - Py_DECREF(importer); - return NULL; - } - return importer; -} - -PyObject * -PyImport_GetImporter(PyObject *path) -{ - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *path_importer_cache = PySys_GetObject("path_importer_cache"); - PyObject *path_hooks = PySys_GetObject("path_hooks"); - if (path_importer_cache == NULL || path_hooks == NULL) { - return NULL; - } - return get_path_importer(tstate, path_importer_cache, path_hooks, path); -} - -#if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) -#include -EM_JS(PyObject*, _PyImport_InitFunc_TrampolineCall, (PyModInitFunction func), { - return wasmTable.get(func)(); -}); -#endif // __EMSCRIPTEN__ && PY_CALL_TRAMPOLINE - -static PyObject* -create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) -{ - PyObject *mod = import_find_extension(tstate, name, name); - if (mod || _PyErr_Occurred(tstate)) { - return mod; - } - - PyObject *modules = tstate->interp->modules; - for (struct _inittab *p = _PyRuntime.imports.inittab; p->name != NULL; p++) { - if (_PyUnicode_EqualToASCIIString(name, p->name)) { - if (p->initfunc == NULL) { - /* Cannot re-init internal module ("sys" or "builtins") */ - mod = PyImport_AddModuleObject(name); - return Py_XNewRef(mod); - } - mod = _PyImport_InitFunc_TrampolineCall(*p->initfunc); - if (mod == NULL) { - return NULL; - } - - if (PyObject_TypeCheck(mod, &PyModuleDef_Type)) { - return PyModule_FromDefAndSpec((PyModuleDef*)mod, spec); - } - else { - /* Remember pointer to module init function. */ - PyModuleDef *def = PyModule_GetDef(mod); - if (def == NULL) { - return NULL; - } - - def->m_base.m_init = p->initfunc; - if (_PyImport_FixupExtensionObject(mod, name, name, - modules) < 0) { - return NULL; - } - return mod; - } - } - } - - // not found - Py_RETURN_NONE; -} - - - -/*[clinic input] -_imp.create_builtin - - spec: object - / - -Create an extension module. -[clinic start generated code]*/ - -static PyObject * -_imp_create_builtin(PyObject *module, PyObject *spec) -/*[clinic end generated code: output=ace7ff22271e6f39 input=37f966f890384e47]*/ -{ - PyThreadState *tstate = _PyThreadState_GET(); - - PyObject *name = PyObject_GetAttrString(spec, "name"); - if (name == NULL) { - return NULL; - } - - if (!PyUnicode_Check(name)) { - PyErr_Format(PyExc_TypeError, - "name must be string, not %.200s", - Py_TYPE(name)->tp_name); - Py_DECREF(name); - return NULL; - } - - PyObject *mod = create_builtin(tstate, name, spec); - Py_DECREF(name); - return mod; -} +/******************/ +/* frozen modules */ +/******************/ /* Return true if the name is an alias. In that case, "alias" is set to the original module name. If it is an alias but the original @@ -1210,14 +1374,11 @@ resolve_module_alias(const char *name, const struct _module_alias *aliases, } } - -/* Frozen modules */ - static bool use_frozen(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); - int override = interp->override_frozen_modules; + int override = OVERRIDE_FROZEN_MODULES(interp); if (override > 0) { return true; } @@ -1587,54 +1748,315 @@ PyImport_ImportFrozenModule(const char *name) } -/* Import a module, either built-in, frozen, or external, and return - its module object WITH INCREMENTED REFERENCE COUNT */ +/*************/ +/* importlib */ +/*************/ -PyObject * -PyImport_ImportModule(const char *name) +/* Import the _imp extension by calling manually _imp.create_builtin() and + _imp.exec_builtin() since importlib is not initialized yet. Initializing + importlib requires the _imp module: this function fix the bootstrap issue. + */ +static PyObject* +bootstrap_imp(PyThreadState *tstate) { - PyObject *pname; - PyObject *result; - - pname = PyUnicode_FromString(name); - if (pname == NULL) + PyObject *name = PyUnicode_FromString("_imp"); + if (name == NULL) { return NULL; - result = PyImport_Import(pname); - Py_DECREF(pname); - return result; -} + } + // Mock a ModuleSpec object just good enough for PyModule_FromDefAndSpec(): + // an object with just a name attribute. + // + // _imp.__spec__ is overridden by importlib._bootstrap._instal() anyway. + PyObject *attrs = Py_BuildValue("{sO}", "name", name); + if (attrs == NULL) { + goto error; + } + PyObject *spec = _PyNamespace_New(attrs); + Py_DECREF(attrs); + if (spec == NULL) { + goto error; + } -/* Import a module without blocking - * - * At first it tries to fetch the module from sys.modules. If the module was - * never loaded before it loads it with PyImport_ImportModule() unless another - * thread holds the import lock. In the latter case the function raises an - * ImportError instead of blocking. - * - * Returns the module object with incremented ref count. - */ -PyObject * -PyImport_ImportModuleNoBlock(const char *name) -{ - return PyImport_ImportModule(name); + // Create the _imp module from its definition. + PyObject *mod = create_builtin(tstate, name, spec); + Py_CLEAR(name); + Py_DECREF(spec); + if (mod == NULL) { + goto error; + } + assert(mod != Py_None); // not found + + // Execute the _imp module: call imp_module_exec(). + if (exec_builtin_or_dynamic(mod) < 0) { + Py_DECREF(mod); + goto error; + } + return mod; + +error: + Py_XDECREF(name); + return NULL; } +/* Global initializations. Can be undone by Py_FinalizeEx(). Don't + call this twice without an intervening Py_FinalizeEx() call. When + initializations fail, a fatal error is issued and the function does + not return. On return, the first thread and interpreter state have + been created. -/* Remove importlib frames from the traceback, - * except in Verbose mode. */ -static void -remove_importlib_frames(PyThreadState *tstate) + Locking: you must hold the interpreter lock while calling this. + (If the lock has not yet been initialized, that's equivalent to + having the lock, but you cannot use multiple threads.) + +*/ +static int +init_importlib(PyThreadState *tstate, PyObject *sysmod) { - const char *importlib_filename = ""; - const char *external_filename = ""; - const char *remove_frames = "_call_with_frames_removed"; - int always_trim = 0; - int in_importlib = 0; - PyObject *exception, *value, *base_tb, *tb; - PyObject **prev_link, **outer_link = NULL; + assert(!_PyErr_Occurred(tstate)); - /* Synopsis: if it's an ImportError, we trim all importlib chunks + PyInterpreterState *interp = tstate->interp; + int verbose = _PyInterpreterState_GetConfig(interp)->verbose; + + // Import _importlib through its frozen version, _frozen_importlib. + if (verbose) { + PySys_FormatStderr("import _frozen_importlib # frozen\n"); + } + if (PyImport_ImportFrozenModule("_frozen_importlib") <= 0) { + return -1; + } + PyObject *importlib = PyImport_AddModule("_frozen_importlib"); // borrowed + if (importlib == NULL) { + return -1; + } + IMPORTLIB(interp) = Py_NewRef(importlib); + + // Import the _imp module + if (verbose) { + PySys_FormatStderr("import _imp # builtin\n"); + } + PyObject *imp_mod = bootstrap_imp(tstate); + if (imp_mod == NULL) { + return -1; + } + if (_PyImport_SetModuleString("_imp", imp_mod) < 0) { + Py_DECREF(imp_mod); + return -1; + } + + // Install importlib as the implementation of import + PyObject *value = PyObject_CallMethod(importlib, "_install", + "OO", sysmod, imp_mod); + Py_DECREF(imp_mod); + if (value == NULL) { + return -1; + } + Py_DECREF(value); + + assert(!_PyErr_Occurred(tstate)); + return 0; +} + + +static int +init_importlib_external(PyInterpreterState *interp) +{ + PyObject *value; + value = PyObject_CallMethod(IMPORTLIB(interp), + "_install_external_importers", ""); + if (value == NULL) { + return -1; + } + Py_DECREF(value); + return 0; +} + +PyObject * +_PyImport_GetImportlibLoader(PyInterpreterState *interp, + const char *loader_name) +{ + return PyObject_GetAttrString(IMPORTLIB(interp), loader_name); +} + +PyObject * +_PyImport_GetImportlibExternalLoader(PyInterpreterState *interp, + const char *loader_name) +{ + PyObject *bootstrap = PyObject_GetAttrString(IMPORTLIB(interp), + "_bootstrap_external"); + if (bootstrap == NULL) { + return NULL; + } + + PyObject *loader_type = PyObject_GetAttrString(bootstrap, loader_name); + Py_DECREF(bootstrap); + return loader_type; +} + +PyObject * +_PyImport_BlessMyLoader(PyInterpreterState *interp, PyObject *module_globals) +{ + PyObject *external = PyObject_GetAttrString(IMPORTLIB(interp), + "_bootstrap_external"); + if (external == NULL) { + return NULL; + } + + PyObject *loader = PyObject_CallMethod(external, "_bless_my_loader", + "O", module_globals, NULL); + Py_DECREF(external); + return loader; +} + +PyObject * +_PyImport_ImportlibModuleRepr(PyInterpreterState *interp, PyObject *m) +{ + return PyObject_CallMethod(IMPORTLIB(interp), "_module_repr", "O", m); +} + + +/*******************/ + +/* Return a finder object for a sys.path/pkg.__path__ item 'p', + possibly by fetching it from the path_importer_cache dict. If it + wasn't yet cached, traverse path_hooks until a hook is found + that can handle the path item. Return None if no hook could; + this tells our caller that the path based finder could not find + a finder for this path item. Cache the result in + path_importer_cache. */ + +static PyObject * +get_path_importer(PyThreadState *tstate, PyObject *path_importer_cache, + PyObject *path_hooks, PyObject *p) +{ + PyObject *importer; + Py_ssize_t j, nhooks; + + /* These conditions are the caller's responsibility: */ + assert(PyList_Check(path_hooks)); + assert(PyDict_Check(path_importer_cache)); + + nhooks = PyList_Size(path_hooks); + if (nhooks < 0) + return NULL; /* Shouldn't happen */ + + importer = PyDict_GetItemWithError(path_importer_cache, p); + if (importer != NULL || _PyErr_Occurred(tstate)) { + return Py_XNewRef(importer); + } + + /* set path_importer_cache[p] to None to avoid recursion */ + if (PyDict_SetItem(path_importer_cache, p, Py_None) != 0) + return NULL; + + for (j = 0; j < nhooks; j++) { + PyObject *hook = PyList_GetItem(path_hooks, j); + if (hook == NULL) + return NULL; + importer = PyObject_CallOneArg(hook, p); + if (importer != NULL) + break; + + if (!_PyErr_ExceptionMatches(tstate, PyExc_ImportError)) { + return NULL; + } + _PyErr_Clear(tstate); + } + if (importer == NULL) { + Py_RETURN_NONE; + } + if (PyDict_SetItem(path_importer_cache, p, importer) < 0) { + Py_DECREF(importer); + return NULL; + } + return importer; +} + +PyObject * +PyImport_GetImporter(PyObject *path) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *path_importer_cache = PySys_GetObject("path_importer_cache"); + PyObject *path_hooks = PySys_GetObject("path_hooks"); + if (path_importer_cache == NULL || path_hooks == NULL) { + return NULL; + } + return get_path_importer(tstate, path_importer_cache, path_hooks, path); +} + + +/*********************/ +/* importing modules */ +/*********************/ + +int +_PyImport_InitDefaultImportFunc(PyInterpreterState *interp) +{ + // Get the __import__ function + PyObject *import_func = _PyDict_GetItemStringWithError(interp->builtins, + "__import__"); + if (import_func == NULL) { + return -1; + } + IMPORT_FUNC(interp) = Py_NewRef(import_func); + return 0; +} + +int +_PyImport_IsDefaultImportFunc(PyInterpreterState *interp, PyObject *func) +{ + return func == IMPORT_FUNC(interp); +} + + +/* Import a module, either built-in, frozen, or external, and return + its module object WITH INCREMENTED REFERENCE COUNT */ + +PyObject * +PyImport_ImportModule(const char *name) +{ + PyObject *pname; + PyObject *result; + + pname = PyUnicode_FromString(name); + if (pname == NULL) + return NULL; + result = PyImport_Import(pname); + Py_DECREF(pname); + return result; +} + + +/* Import a module without blocking + * + * At first it tries to fetch the module from sys.modules. If the module was + * never loaded before it loads it with PyImport_ImportModule() unless another + * thread holds the import lock. In the latter case the function raises an + * ImportError instead of blocking. + * + * Returns the module object with incremented ref count. + */ +PyObject * +PyImport_ImportModuleNoBlock(const char *name) +{ + return PyImport_ImportModule(name); +} + + +/* Remove importlib frames from the traceback, + * except in Verbose mode. */ +static void +remove_importlib_frames(PyThreadState *tstate) +{ + const char *importlib_filename = ""; + const char *external_filename = ""; + const char *remove_frames = "_call_with_frames_removed"; + int always_trim = 0; + int in_importlib = 0; + PyObject *exception, *value, *base_tb, *tb; + PyObject **prev_link, **outer_link = NULL; + + /* Synopsis: if it's an ImportError, we trim all importlib chunks from the traceback. We always trim chunks which end with a call to "_call_with_frames_removed". */ @@ -1851,8 +2273,8 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) PyObject *mod = NULL; PyInterpreterState *interp = tstate->interp; int import_time = _PyInterpreterState_GetConfig(interp)->import_time; -#define import_level _PyRuntime.imports.find_and_load.import_level -#define accumulated _PyRuntime.imports.find_and_load.accumulated +#define import_level FIND_AND_LOAD.import_level +#define accumulated FIND_AND_LOAD.accumulated _PyTime_t t1 = 0, accumulated_copy = accumulated; @@ -1873,7 +2295,7 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) * _PyDict_GetItemIdWithError(). */ if (import_time) { -#define header _PyRuntime.imports.find_and_load.header +#define header FIND_AND_LOAD.header if (header) { fputs("import time: self [us] | cumulative | imported package\n", stderr); @@ -1889,8 +2311,8 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) if (PyDTrace_IMPORT_FIND_LOAD_START_ENABLED()) PyDTrace_IMPORT_FIND_LOAD_START(PyUnicode_AsUTF8(abs_name)); - mod = PyObject_CallMethodObjArgs(interp->importlib, &_Py_ID(_find_and_load), - abs_name, interp->import_func, NULL); + mod = PyObject_CallMethodObjArgs(IMPORTLIB(interp), &_Py_ID(_find_and_load), + abs_name, IMPORT_FUNC(interp), NULL); if (PyDTrace_IMPORT_FIND_LOAD_DONE_ENABLED()) PyDTrace_IMPORT_FIND_LOAD_DONE(PyUnicode_AsUTF8(abs_name), @@ -1913,23 +2335,6 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) #undef accumulated } -PyObject * -PyImport_GetModule(PyObject *name) -{ - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *mod; - - mod = import_get_module(tstate, name); - if (mod != NULL && mod != Py_None) { - if (import_ensure_initialized(tstate->interp, mod, name) < 0) { - Py_DECREF(mod); - remove_importlib_frames(tstate); - return NULL; - } - } - return mod; -} - PyObject * PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, @@ -2059,8 +2464,8 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, if (path) { Py_DECREF(path); final_mod = PyObject_CallMethodObjArgs( - interp->importlib, &_Py_ID(_handle_fromlist), - mod, fromlist, interp->import_func, NULL); + IMPORTLIB(interp), &_Py_ID(_handle_fromlist), + mod, fromlist, IMPORT_FUNC(interp), NULL); } else { final_mod = Py_NewRef(mod); @@ -2129,72 +2534,429 @@ PyImport_ReloadModule(PyObject *m) PyObject * PyImport_Import(PyObject *module_name) { - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *globals = NULL; - PyObject *import = NULL; - PyObject *builtins = NULL; - PyObject *r = NULL; + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *globals = NULL; + PyObject *import = NULL; + PyObject *builtins = NULL; + PyObject *r = NULL; + + PyObject *from_list = PyList_New(0); + if (from_list == NULL) { + goto err; + } + + /* Get the builtins from current globals */ + globals = PyEval_GetGlobals(); + if (globals != NULL) { + Py_INCREF(globals); + builtins = PyObject_GetItem(globals, &_Py_ID(__builtins__)); + if (builtins == NULL) + goto err; + } + else { + /* No globals -- use standard builtins, and fake globals */ + builtins = PyImport_ImportModuleLevel("builtins", + NULL, NULL, NULL, 0); + if (builtins == NULL) { + goto err; + } + globals = Py_BuildValue("{OO}", &_Py_ID(__builtins__), builtins); + if (globals == NULL) + goto err; + } + + /* Get the __import__ function from the builtins */ + if (PyDict_Check(builtins)) { + import = PyObject_GetItem(builtins, &_Py_ID(__import__)); + if (import == NULL) { + _PyErr_SetObject(tstate, PyExc_KeyError, &_Py_ID(__import__)); + } + } + else + import = PyObject_GetAttr(builtins, &_Py_ID(__import__)); + if (import == NULL) + goto err; + + /* Call the __import__ function with the proper argument list + Always use absolute import here. + Calling for side-effect of import. */ + r = PyObject_CallFunction(import, "OOOOi", module_name, globals, + globals, from_list, 0, NULL); + if (r == NULL) + goto err; + Py_DECREF(r); + + r = import_get_module(tstate, module_name); + if (r == NULL && !_PyErr_Occurred(tstate)) { + _PyErr_SetObject(tstate, PyExc_KeyError, module_name); + } + + err: + Py_XDECREF(globals); + Py_XDECREF(builtins); + Py_XDECREF(import); + Py_XDECREF(from_list); + + return r; +} + + +/*********************/ +/* runtime lifecycle */ +/*********************/ + +PyStatus +_PyImport_Init(void) +{ + if (INITTAB != NULL) { + return _PyStatus_ERR("global import state already initialized"); + } + + PyStatus status = _PyStatus_OK(); + + /* Force default raw memory allocator to get a known allocator to be able + to release the memory in _PyImport_Fini() */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + if (init_builtin_modules_table() != 0) { + status = PyStatus_NoMemory(); + goto done; + } + +done: + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + return status; +} + +void +_PyImport_Fini(void) +{ + /* Destroy the database used by _PyImport_{Fixup,Find}Extension */ + _extensions_cache_clear(); + if (import_lock != NULL) { + PyThread_free_lock(import_lock); + import_lock = NULL; + } + + /* Use the same memory allocator as _PyImport_Init(). */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + /* Free memory allocated by _PyImport_Init() */ + fini_builtin_modules_table(); + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); +} + +void +_PyImport_Fini2(void) +{ + /* Use the same memory allocator than PyImport_ExtendInittab(). */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + // Reset PyImport_Inittab + PyImport_Inittab = _PyImport_Inittab; + + /* Free memory allocated by PyImport_ExtendInittab() */ + PyMem_RawFree(inittab_copy); + inittab_copy = NULL; + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); +} + + +/*************************/ +/* interpreter lifecycle */ +/*************************/ + +PyStatus +_PyImport_InitCore(PyThreadState *tstate, PyObject *sysmod, int importlib) +{ + // XXX Initialize here: interp->modules and interp->import_func. + // XXX Initialize here: sys.modules and sys.meta_path. + + if (importlib) { + /* This call sets up builtin and frozen import support */ + if (init_importlib(tstate, sysmod) < 0) { + return _PyStatus_ERR("failed to initialize importlib"); + } + } + + return _PyStatus_OK(); +} + +/* In some corner cases it is important to be sure that the import + machinery has been initialized (or not cleaned up yet). For + example, see issue #4236 and PyModule_Create2(). */ + +int +_PyImport_IsInitialized(PyInterpreterState *interp) +{ + if (MODULES(interp) == NULL) + return 0; + return 1; +} + +/* Clear the direct per-interpreter import state, if not cleared already. */ +void +_PyImport_ClearCore(PyInterpreterState *interp) +{ + /* interp->modules should have been cleaned up and cleared already + by _PyImport_FiniCore(). */ + Py_CLEAR(MODULES(interp)); + Py_CLEAR(MODULES_BY_INDEX(interp)); + Py_CLEAR(IMPORTLIB(interp)); + Py_CLEAR(IMPORT_FUNC(interp)); +} + +void +_PyImport_FiniCore(PyInterpreterState *interp) +{ + int verbose = _PyInterpreterState_GetConfig(interp)->verbose; + + if (_PySys_ClearAttrString(interp, "meta_path", verbose) < 0) { + PyErr_WriteUnraisable(NULL); + } + + // XXX Pull in most of finalize_modules() in pylifecycle.c. + + if (_PySys_ClearAttrString(interp, "modules", verbose) < 0) { + PyErr_WriteUnraisable(NULL); + } + + _PyImport_ClearCore(interp); +} + +// XXX Add something like _PyImport_Disable() for use early in interp fini? + + +/* "external" imports */ + +static int +init_zipimport(PyThreadState *tstate, int verbose) +{ + PyObject *path_hooks = PySys_GetObject("path_hooks"); + if (path_hooks == NULL) { + _PyErr_SetString(tstate, PyExc_RuntimeError, + "unable to get sys.path_hooks"); + return -1; + } + + if (verbose) { + PySys_WriteStderr("# installing zipimport hook\n"); + } + + PyObject *zipimporter = _PyImport_GetModuleAttrString("zipimport", "zipimporter"); + if (zipimporter == NULL) { + _PyErr_Clear(tstate); /* No zipimporter object -- okay */ + if (verbose) { + PySys_WriteStderr("# can't import zipimport.zipimporter\n"); + } + } + else { + /* sys.path_hooks.insert(0, zipimporter) */ + int err = PyList_Insert(path_hooks, 0, zipimporter); + Py_DECREF(zipimporter); + if (err < 0) { + return -1; + } + if (verbose) { + PySys_WriteStderr("# installed zipimport hook\n"); + } + } + + return 0; +} + +PyStatus +_PyImport_InitExternal(PyThreadState *tstate) +{ + int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose; + + // XXX Initialize here: sys.path_hooks and sys.path_importer_cache. + + if (init_importlib_external(tstate->interp) != 0) { + _PyErr_Print(tstate); + return _PyStatus_ERR("external importer setup failed"); + } + + if (init_zipimport(tstate, verbose) != 0) { + PyErr_Print(); + return _PyStatus_ERR("initializing zipimport failed"); + } + + return _PyStatus_OK(); +} + +void +_PyImport_FiniExternal(PyInterpreterState *interp) +{ + int verbose = _PyInterpreterState_GetConfig(interp)->verbose; + + // XXX Uninstall importlib metapath importers here? + + if (_PySys_ClearAttrString(interp, "path_importer_cache", verbose) < 0) { + PyErr_WriteUnraisable(NULL); + } + if (_PySys_ClearAttrString(interp, "path_hooks", verbose) < 0) { + PyErr_WriteUnraisable(NULL); + } +} + + +/******************/ +/* module helpers */ +/******************/ + +PyObject * +_PyImport_GetModuleAttr(PyObject *modname, PyObject *attrname) +{ + PyObject *mod = PyImport_Import(modname); + if (mod == NULL) { + return NULL; + } + PyObject *result = PyObject_GetAttr(mod, attrname); + Py_DECREF(mod); + return result; +} + +PyObject * +_PyImport_GetModuleAttrString(const char *modname, const char *attrname) +{ + PyObject *pmodname = PyUnicode_FromString(modname); + if (pmodname == NULL) { + return NULL; + } + PyObject *pattrname = PyUnicode_FromString(attrname); + if (pattrname == NULL) { + Py_DECREF(pmodname); + return NULL; + } + PyObject *result = _PyImport_GetModuleAttr(pmodname, pattrname); + Py_DECREF(pattrname); + Py_DECREF(pmodname); + return result; +} + + +/**************/ +/* the module */ +/**************/ + +/*[clinic input] +_imp.lock_held + +Return True if the import lock is currently held, else False. + +On platforms without threads, return False. +[clinic start generated code]*/ + +static PyObject * +_imp_lock_held_impl(PyObject *module) +/*[clinic end generated code: output=8b89384b5e1963fc input=9b088f9b217d9bdf]*/ +{ + return PyBool_FromLong(import_lock_thread != PYTHREAD_INVALID_THREAD_ID); +} + +/*[clinic input] +_imp.acquire_lock + +Acquires the interpreter's import lock for the current thread. + +This lock should be used by import hooks to ensure thread-safety when importing +modules. On platforms without threads, this function does nothing. +[clinic start generated code]*/ + +static PyObject * +_imp_acquire_lock_impl(PyObject *module) +/*[clinic end generated code: output=1aff58cb0ee1b026 input=4a2d4381866d5fdc]*/ +{ + _PyImport_AcquireLock(); + Py_RETURN_NONE; +} + +/*[clinic input] +_imp.release_lock + +Release the interpreter's import lock. + +On platforms without threads, this function does nothing. +[clinic start generated code]*/ + +static PyObject * +_imp_release_lock_impl(PyObject *module) +/*[clinic end generated code: output=7faab6d0be178b0a input=934fb11516dd778b]*/ +{ + if (_PyImport_ReleaseLock() < 0) { + PyErr_SetString(PyExc_RuntimeError, + "not holding the import lock"); + return NULL; + } + Py_RETURN_NONE; +} + + +/*[clinic input] +_imp._fix_co_filename + + code: object(type="PyCodeObject *", subclass_of="&PyCode_Type") + Code object to change. + + path: unicode + File path to use. + / + +Changes code.co_filename to specify the passed-in file path. +[clinic start generated code]*/ + +static PyObject * +_imp__fix_co_filename_impl(PyObject *module, PyCodeObject *code, + PyObject *path) +/*[clinic end generated code: output=1d002f100235587d input=895ba50e78b82f05]*/ + +{ + update_compiled_module(code, path); - PyObject *from_list = PyList_New(0); - if (from_list == NULL) { - goto err; - } + Py_RETURN_NONE; +} - /* Get the builtins from current globals */ - globals = PyEval_GetGlobals(); - if (globals != NULL) { - Py_INCREF(globals); - builtins = PyObject_GetItem(globals, &_Py_ID(__builtins__)); - if (builtins == NULL) - goto err; - } - else { - /* No globals -- use standard builtins, and fake globals */ - builtins = PyImport_ImportModuleLevel("builtins", - NULL, NULL, NULL, 0); - if (builtins == NULL) { - goto err; - } - globals = Py_BuildValue("{OO}", &_Py_ID(__builtins__), builtins); - if (globals == NULL) - goto err; - } - /* Get the __import__ function from the builtins */ - if (PyDict_Check(builtins)) { - import = PyObject_GetItem(builtins, &_Py_ID(__import__)); - if (import == NULL) { - _PyErr_SetObject(tstate, PyExc_KeyError, &_Py_ID(__import__)); - } - } - else - import = PyObject_GetAttr(builtins, &_Py_ID(__import__)); - if (import == NULL) - goto err; +/*[clinic input] +_imp.create_builtin - /* Call the __import__ function with the proper argument list - Always use absolute import here. - Calling for side-effect of import. */ - r = PyObject_CallFunction(import, "OOOOi", module_name, globals, - globals, from_list, 0, NULL); - if (r == NULL) - goto err; - Py_DECREF(r); + spec: object + / - r = import_get_module(tstate, module_name); - if (r == NULL && !_PyErr_Occurred(tstate)) { - _PyErr_SetObject(tstate, PyExc_KeyError, module_name); +Create an extension module. +[clinic start generated code]*/ + +static PyObject * +_imp_create_builtin(PyObject *module, PyObject *spec) +/*[clinic end generated code: output=ace7ff22271e6f39 input=37f966f890384e47]*/ +{ + PyThreadState *tstate = _PyThreadState_GET(); + + PyObject *name = PyObject_GetAttrString(spec, "name"); + if (name == NULL) { + return NULL; } - err: - Py_XDECREF(globals); - Py_XDECREF(builtins); - Py_XDECREF(import); - Py_XDECREF(from_list); + if (!PyUnicode_Check(name)) { + PyErr_Format(PyExc_TypeError, + "name must be string, not %.200s", + Py_TYPE(name)->tp_name); + Py_DECREF(name); + return NULL; + } - return r; + PyObject *mod = create_builtin(tstate, name, spec); + Py_DECREF(name); + return mod; } + /*[clinic input] _imp.extension_suffixes @@ -2459,34 +3221,10 @@ _imp__override_frozen_modules_for_tests_impl(PyObject *module, int override) /*[clinic end generated code: output=36d5cb1594160811 input=8f1f95a3ef21aec3]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - interp->override_frozen_modules = override; + OVERRIDE_FROZEN_MODULES(interp) = override; Py_RETURN_NONE; } -/* Common implementation for _imp.exec_dynamic and _imp.exec_builtin */ -static int -exec_builtin_or_dynamic(PyObject *mod) { - PyModuleDef *def; - void *state; - - if (!PyModule_Check(mod)) { - return 0; - } - - def = PyModule_GetDef(mod); - if (def == NULL) { - return 0; - } - - state = PyModule_GetState(mod); - if (state) { - /* Already initialized; skip reload */ - return 0; - } - - return PyModule_ExecDef(mod, def); -} - #ifdef HAVE_DYNAMIC_LOADING /*[clinic input] @@ -2674,158 +3412,6 @@ PyInit__imp(void) } -// Import the _imp extension by calling manually _imp.create_builtin() and -// _imp.exec_builtin() since importlib is not initialized yet. Initializing -// importlib requires the _imp module: this function fix the bootstrap issue. -PyObject* -_PyImport_BootstrapImp(PyThreadState *tstate) -{ - PyObject *name = PyUnicode_FromString("_imp"); - if (name == NULL) { - return NULL; - } - - // Mock a ModuleSpec object just good enough for PyModule_FromDefAndSpec(): - // an object with just a name attribute. - // - // _imp.__spec__ is overridden by importlib._bootstrap._instal() anyway. - PyObject *attrs = Py_BuildValue("{sO}", "name", name); - if (attrs == NULL) { - goto error; - } - PyObject *spec = _PyNamespace_New(attrs); - Py_DECREF(attrs); - if (spec == NULL) { - goto error; - } - - // Create the _imp module from its definition. - PyObject *mod = create_builtin(tstate, name, spec); - Py_CLEAR(name); - Py_DECREF(spec); - if (mod == NULL) { - goto error; - } - assert(mod != Py_None); // not found - - // Execute the _imp module: call imp_module_exec(). - if (exec_builtin_or_dynamic(mod) < 0) { - Py_DECREF(mod); - goto error; - } - return mod; - -error: - Py_XDECREF(name); - return NULL; -} - - -/* API for embedding applications that want to add their own entries - to the table of built-in modules. This should normally be called - *before* Py_Initialize(). When the table resize fails, -1 is - returned and the existing table is unchanged. - - After a similar function by Just van Rossum. */ - -int -PyImport_ExtendInittab(struct _inittab *newtab) -{ - struct _inittab *p; - size_t i, n; - int res = 0; - - if (_PyRuntime.imports.inittab != NULL) { - Py_FatalError("PyImport_ExtendInittab() may not be called after Py_Initialize()"); - } - - /* Count the number of entries in both tables */ - for (n = 0; newtab[n].name != NULL; n++) - ; - if (n == 0) - return 0; /* Nothing to do */ - for (i = 0; PyImport_Inittab[i].name != NULL; i++) - ; - - /* Force default raw memory allocator to get a known allocator to be able - to release the memory in _PyImport_Fini2() */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - /* Allocate new memory for the combined table */ - p = NULL; - if (i + n <= SIZE_MAX / sizeof(struct _inittab) - 1) { - size_t size = sizeof(struct _inittab) * (i + n + 1); - p = PyMem_RawRealloc(inittab_copy, size); - } - if (p == NULL) { - res = -1; - goto done; - } - - /* Copy the tables into the new memory at the first call - to PyImport_ExtendInittab(). */ - if (inittab_copy != PyImport_Inittab) { - memcpy(p, PyImport_Inittab, (i+1) * sizeof(struct _inittab)); - } - memcpy(p + i, newtab, (n + 1) * sizeof(struct _inittab)); - PyImport_Inittab = inittab_copy = p; - -done: - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - return res; -} - -/* Shorthand to add a single entry given a name and a function */ - -int -PyImport_AppendInittab(const char *name, PyObject* (*initfunc)(void)) -{ - struct _inittab newtab[2]; - - if (_PyRuntime.imports.inittab != NULL) { - Py_FatalError("PyImport_AppendInittab() may not be called after Py_Initialize()"); - } - - memset(newtab, '\0', sizeof newtab); - - newtab[0].name = name; - newtab[0].initfunc = initfunc; - - return PyImport_ExtendInittab(newtab); -} - - -PyObject * -_PyImport_GetModuleAttr(PyObject *modname, PyObject *attrname) -{ - PyObject *mod = PyImport_Import(modname); - if (mod == NULL) { - return NULL; - } - PyObject *result = PyObject_GetAttr(mod, attrname); - Py_DECREF(mod); - return result; -} - -PyObject * -_PyImport_GetModuleAttrString(const char *modname, const char *attrname) -{ - PyObject *pmodname = PyUnicode_FromString(modname); - if (pmodname == NULL) { - return NULL; - } - PyObject *pattrname = PyUnicode_FromString(attrname); - if (pattrname == NULL) { - Py_DECREF(pmodname); - return NULL; - } - PyObject *result = _PyImport_GetModuleAttr(pmodname, pattrname); - Py_DECREF(pattrname); - Py_DECREF(pmodname); - return result; -} - #ifdef __cplusplus } #endif diff --git a/Python/importdl.c b/Python/importdl.c index 91fa06f49c2897..6dafb4541486e9 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -99,7 +99,7 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) #endif PyObject *name_unicode = NULL, *name = NULL, *path = NULL, *m = NULL; const char *name_buf, *hook_prefix; - const char *oldcontext; + const char *oldcontext, *newcontext; dl_funcptr exportfunc; PyModuleDef *def; PyModInitFunction p0; @@ -113,6 +113,10 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) "spec.name must be a string"); goto error; } + newcontext = PyUnicode_AsUTF8(name_unicode); + if (newcontext == NULL) { + goto error; + } name = get_encoded_name(name_unicode, &hook_prefix); if (name == NULL) { @@ -160,16 +164,9 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) p0 = (PyModInitFunction)exportfunc; /* Package context is needed for single-phase init */ -#define _Py_PackageContext (_PyRuntime.imports.pkgcontext) - oldcontext = _Py_PackageContext; - _Py_PackageContext = PyUnicode_AsUTF8(name_unicode); - if (_Py_PackageContext == NULL) { - _Py_PackageContext = oldcontext; - goto error; - } + oldcontext = _PyImport_SwapPackageContext(newcontext); m = _PyImport_InitFunc_TrampolineCall(p0); - _Py_PackageContext = oldcontext; -#undef _Py_PackageContext + _PyImport_SwapPackageContext(oldcontext); if (m == NULL) { if (!PyErr_Occurred()) { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 045a2996e8988b..281035dafa9577 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -156,79 +156,6 @@ Py_IsInitialized(void) } -/* Global initializations. Can be undone by Py_FinalizeEx(). Don't - call this twice without an intervening Py_FinalizeEx() call. When - initializations fail, a fatal error is issued and the function does - not return. On return, the first thread and interpreter state have - been created. - - Locking: you must hold the interpreter lock while calling this. - (If the lock has not yet been initialized, that's equivalent to - having the lock, but you cannot use multiple threads.) - -*/ -static int -init_importlib(PyThreadState *tstate, PyObject *sysmod) -{ - assert(!_PyErr_Occurred(tstate)); - - PyInterpreterState *interp = tstate->interp; - int verbose = _PyInterpreterState_GetConfig(interp)->verbose; - - // Import _importlib through its frozen version, _frozen_importlib. - if (verbose) { - PySys_FormatStderr("import _frozen_importlib # frozen\n"); - } - if (PyImport_ImportFrozenModule("_frozen_importlib") <= 0) { - return -1; - } - PyObject *importlib = PyImport_AddModule("_frozen_importlib"); // borrowed - if (importlib == NULL) { - return -1; - } - interp->importlib = Py_NewRef(importlib); - - // Import the _imp module - if (verbose) { - PySys_FormatStderr("import _imp # builtin\n"); - } - PyObject *imp_mod = _PyImport_BootstrapImp(tstate); - if (imp_mod == NULL) { - return -1; - } - if (_PyImport_SetModuleString("_imp", imp_mod) < 0) { - Py_DECREF(imp_mod); - return -1; - } - - // Install importlib as the implementation of import - PyObject *value = PyObject_CallMethod(importlib, "_install", - "OO", sysmod, imp_mod); - Py_DECREF(imp_mod); - if (value == NULL) { - return -1; - } - Py_DECREF(value); - - assert(!_PyErr_Occurred(tstate)); - return 0; -} - - -static PyStatus -init_importlib_external(PyThreadState *tstate) -{ - PyObject *value; - value = PyObject_CallMethod(tstate->interp->importlib, - "_install_external_importers", ""); - if (value == NULL) { - _PyErr_Print(tstate); - return _PyStatus_ERR("external importer setup failed"); - } - Py_DECREF(value); - return _PyImportZip_Init(tstate); -} - /* Helper functions to better handle the legacy C locale * * The legacy C locale assumes ASCII as the default text encoding, which @@ -814,7 +741,8 @@ pycore_init_builtins(PyThreadState *tstate) goto error; } - if (_PyImport_FixupBuiltin(bimod, "builtins", interp->modules) < 0) { + PyObject *modules = _PyImport_GetModules(interp); + if (_PyImport_FixupBuiltin(bimod, "builtins", modules) < 0) { goto error; } @@ -850,13 +778,9 @@ pycore_init_builtins(PyThreadState *tstate) } Py_DECREF(bimod); - // Get the __import__ function - PyObject *import_func = _PyDict_GetItemStringWithError(interp->builtins, - "__import__"); - if (import_func == NULL) { + if (_PyImport_InitDefaultImportFunc(interp) < 0) { goto error; } - interp->import_func = Py_NewRef(import_func); assert(!_PyErr_Occurred(tstate)); return _PyStatus_OK(); @@ -918,11 +842,10 @@ pycore_interp_init(PyThreadState *tstate) } const PyConfig *config = _PyInterpreterState_GetConfig(interp); - if (config->_install_importlib) { - /* This call sets up builtin and frozen import support */ - if (init_importlib(tstate, sysmod) < 0) { - return _PyStatus_ERR("failed to initialize importlib"); - } + + status = _PyImport_InitCore(tstate, sysmod, config->_install_importlib); + if (_PyStatus_EXCEPTION(status)) { + goto done; } done: @@ -1172,7 +1095,7 @@ init_interp_main(PyThreadState *tstate) return _PyStatus_ERR("failed to update the Python config"); } - status = init_importlib_external(tstate); + status = _PyImport_InitExternal(tstate); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -1379,8 +1302,11 @@ finalize_modules_delete_special(PyThreadState *tstate, int verbose) static const char * const sys_deletes[] = { "path", "argv", "ps1", "ps2", "last_type", "last_value", "last_traceback", - "path_hooks", "path_importer_cache", "meta_path", "__interactivehook__", + // path_hooks and path_importer_cache are cleared + // by _PyImport_FiniExternal(). + // XXX Clear meta_path in _PyImport_FiniCore(). + "meta_path", NULL }; @@ -1401,10 +1327,7 @@ finalize_modules_delete_special(PyThreadState *tstate, int verbose) const char * const *p; for (p = sys_deletes; *p != NULL; p++) { - if (verbose) { - PySys_WriteStderr("# clear sys.%s\n", *p); - } - if (PyDict_SetItemString(interp->sysdict, *p, Py_None) < 0) { + if (_PySys_ClearAttrString(interp, *p, verbose) < 0) { PyErr_WriteUnraisable(NULL); } } @@ -1576,11 +1499,12 @@ finalize_clear_sys_builtins_dict(PyInterpreterState *interp, int verbose) /* Clear modules, as good as we can */ +// XXX Move most of this to import.c. static void finalize_modules(PyThreadState *tstate) { PyInterpreterState *interp = tstate->interp; - PyObject *modules = interp->modules; + PyObject *modules = _PyImport_GetModules(interp); if (modules == NULL) { // Already done return; @@ -1645,12 +1569,12 @@ finalize_modules(PyThreadState *tstate) // clear PyInterpreterState.modules_by_index and // clear PyModuleDef.m_base.m_copy (of extensions not using the multi-phase // initialization API) - _PyInterpreterState_ClearModules(interp); + _PyImport_ClearModulesByIndex(interp); // Clear and delete the modules directory. Actual modules will // still be there only if imported during the execution of some // destructor. - Py_SETREF(interp->modules, NULL); + _PyImport_ClearModules(interp); // Collect garbage once more _PyGC_CollectNoFail(tstate); @@ -1861,6 +1785,8 @@ Py_FinalizeEx(void) runtime->initialized = 0; runtime->core_initialized = 0; + // XXX Call something like _PyImport_Disable() here? + /* Destroy the state of all threads of the interpreter, except of the current thread. In practice, only daemon threads should still be alive, except if wait_for_thread_shutdown() has been cancelled by CTRL+C. @@ -1910,6 +1836,7 @@ Py_FinalizeEx(void) PyGC_Collect(); /* Destroy all modules */ + _PyImport_FiniExternal(tstate->interp); finalize_modules(tstate); /* Print debug stats if any */ @@ -1943,7 +1870,9 @@ Py_FinalizeEx(void) so it is possible to use tracemalloc in objects destructor. */ _PyTraceMalloc_Fini(); - /* Destroy the database used by _PyImport_{Fixup,Find}Extension */ + /* Finalize any remaining import state */ + // XXX Move these up to where finalize_modules() is currently. + _PyImport_FiniCore(tstate->interp); _PyImport_Fini(); /* unload faulthandler module */ @@ -2183,7 +2112,11 @@ Py_EndInterpreter(PyThreadState *tstate) Py_FatalError("not the last thread"); } + // XXX Call something like _PyImport_Disable() here? + + _PyImport_FiniExternal(tstate->interp); finalize_modules(tstate); + _PyImport_FiniCore(tstate->interp); finalize_interp_clear(tstate); finalize_interp_delete(tstate->interp); @@ -2232,8 +2165,8 @@ add_main_module(PyInterpreterState *interp) if (PyErr_Occurred()) { return _PyStatus_ERR("Failed to test __main__.__loader__"); } - PyObject *loader = PyObject_GetAttrString(interp->importlib, - "BuiltinImporter"); + PyObject *loader = _PyImport_GetImportlibLoader(interp, + "BuiltinImporter"); if (loader == NULL) { return _PyStatus_ERR("Failed to retrieve BuiltinImporter"); } @@ -2739,7 +2672,7 @@ _Py_DumpExtensionModules(int fd, PyInterpreterState *interp) if (interp == NULL) { return; } - PyObject *modules = interp->modules; + PyObject *modules = _PyImport_GetModules(interp); if (modules == NULL || !PyDict_Check(modules)) { return; } diff --git a/Python/pystate.c b/Python/pystate.c index 1261092d1435fa..4770caaed0a363 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -772,11 +772,13 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->codec_search_path); Py_CLEAR(interp->codec_search_cache); Py_CLEAR(interp->codec_error_registry); - Py_CLEAR(interp->modules); - Py_CLEAR(interp->modules_by_index); + + assert(interp->imports.modules == NULL); + assert(interp->imports.modules_by_index == NULL); + assert(interp->imports.importlib == NULL); + assert(interp->imports.import_func == NULL); + Py_CLEAR(interp->builtins_copy); - Py_CLEAR(interp->importlib); - Py_CLEAR(interp->import_func); Py_CLEAR(interp->dict); #ifdef HAVE_FORK Py_CLEAR(interp->before_forkers); @@ -836,6 +838,7 @@ PyInterpreterState_Clear(PyInterpreterState *interp) // garbage. It can be different than the current Python thread state // of 'interp'. PyThreadState *current_tstate = current_fast_get(interp->runtime); + _PyImport_ClearCore(interp); interpreter_clear(interp, current_tstate); } @@ -843,6 +846,7 @@ PyInterpreterState_Clear(PyInterpreterState *interp) void _PyInterpreterState_Clear(PyThreadState *tstate) { + _PyImport_ClearCore(tstate->interp); interpreter_clear(tstate->interp, tstate); } @@ -945,36 +949,6 @@ _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime) #endif -// Used by finalize_modules() -void -_PyInterpreterState_ClearModules(PyInterpreterState *interp) -{ - if (!interp->modules_by_index) { - return; - } - - Py_ssize_t i; - for (i = 0; i < PyList_GET_SIZE(interp->modules_by_index); i++) { - PyObject *m = PyList_GET_ITEM(interp->modules_by_index, i); - if (PyModule_Check(m)) { - /* cleanup the saved copy of module dicts */ - PyModuleDef *md = PyModule_GetDef(m); - if (md) { - Py_CLEAR(md->m_base.m_copy); - } - } - } - - /* Setting modules_by_index to NULL could be dangerous, so we - clear the list instead. */ - if (PyList_SetSlice(interp->modules_by_index, - 0, PyList_GET_SIZE(interp->modules_by_index), - NULL)) { - PyErr_WriteUnraisable(interp->modules_by_index); - } -} - - //---------- // accessors //---------- @@ -1058,11 +1032,12 @@ _PyInterpreterState_RequireIDRef(PyInterpreterState *interp, int required) PyObject * _PyInterpreterState_GetMainModule(PyInterpreterState *interp) { - if (interp->modules == NULL) { + PyObject *modules = _PyImport_GetModules(interp); + if (modules == NULL) { PyErr_SetString(PyExc_RuntimeError, "interpreter not initialized"); return NULL; } - return PyMapping_GetItemString(interp->modules, "__main__"); + return PyMapping_GetItemString(modules, "__main__"); } PyObject * @@ -1922,110 +1897,6 @@ _PyThread_CurrentExceptions(void) } -/****************/ -/* module state */ -/****************/ - -PyObject* -PyState_FindModule(PyModuleDef* module) -{ - Py_ssize_t index = module->m_base.m_index; - PyInterpreterState *state = _PyInterpreterState_GET(); - PyObject *res; - if (module->m_slots) { - return NULL; - } - if (index == 0) - return NULL; - if (state->modules_by_index == NULL) - return NULL; - if (index >= PyList_GET_SIZE(state->modules_by_index)) - return NULL; - res = PyList_GET_ITEM(state->modules_by_index, index); - return res==Py_None ? NULL : res; -} - -int -_PyState_AddModule(PyThreadState *tstate, PyObject* module, PyModuleDef* def) -{ - if (!def) { - assert(_PyErr_Occurred(tstate)); - return -1; - } - if (def->m_slots) { - _PyErr_SetString(tstate, - PyExc_SystemError, - "PyState_AddModule called on module with slots"); - return -1; - } - - PyInterpreterState *interp = tstate->interp; - if (!interp->modules_by_index) { - interp->modules_by_index = PyList_New(0); - if (!interp->modules_by_index) { - return -1; - } - } - - while (PyList_GET_SIZE(interp->modules_by_index) <= def->m_base.m_index) { - if (PyList_Append(interp->modules_by_index, Py_None) < 0) { - return -1; - } - } - - return PyList_SetItem(interp->modules_by_index, - def->m_base.m_index, Py_NewRef(module)); -} - -int -PyState_AddModule(PyObject* module, PyModuleDef* def) -{ - if (!def) { - Py_FatalError("module definition is NULL"); - return -1; - } - - PyThreadState *tstate = current_fast_get(&_PyRuntime); - PyInterpreterState *interp = tstate->interp; - Py_ssize_t index = def->m_base.m_index; - if (interp->modules_by_index && - index < PyList_GET_SIZE(interp->modules_by_index) && - module == PyList_GET_ITEM(interp->modules_by_index, index)) - { - _Py_FatalErrorFormat(__func__, "module %p already added", module); - return -1; - } - return _PyState_AddModule(tstate, module, def); -} - -int -PyState_RemoveModule(PyModuleDef* def) -{ - PyThreadState *tstate = current_fast_get(&_PyRuntime); - PyInterpreterState *interp = tstate->interp; - - if (def->m_slots) { - _PyErr_SetString(tstate, - PyExc_SystemError, - "PyState_RemoveModule called on module with slots"); - return -1; - } - - Py_ssize_t index = def->m_base.m_index; - if (index == 0) { - Py_FatalError("invalid module index"); - } - if (interp->modules_by_index == NULL) { - Py_FatalError("Interpreters module-list not accessible."); - } - if (index > PyList_GET_SIZE(interp->modules_by_index)) { - Py_FatalError("Module index out of bounds."); - } - - return PyList_SetItem(interp->modules_by_index, index, Py_NewRef(Py_None)); -} - - /***********************************/ /* Python "auto thread state" API. */ /***********************************/ diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 6a4d593768690a..ce993ea8796cb7 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -350,14 +350,8 @@ static int set_main_loader(PyObject *d, PyObject *filename, const char *loader_name) { PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *bootstrap = PyObject_GetAttrString(interp->importlib, - "_bootstrap_external"); - if (bootstrap == NULL) { - return -1; - } - - PyObject *loader_type = PyObject_GetAttrString(bootstrap, loader_name); - Py_DECREF(bootstrap); + PyObject *loader_type = _PyImport_GetImportlibExternalLoader(interp, + loader_name); if (loader_type == NULL) { return -1; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 6e81ef92b67f70..b69b803560924c 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -142,6 +142,20 @@ PySys_SetObject(const char *name, PyObject *v) return sys_set_object_str(interp, name, v); } +int +_PySys_ClearAttrString(PyInterpreterState *interp, + const char *name, int verbose) +{ + if (verbose) { + PySys_WriteStderr("# clear sys.%s\n", name); + } + /* To play it safe, we set the attr to None instead of deleting it. */ + if (PyDict_SetItemString(interp->sysdict, name, Py_None) < 0) { + return -1; + } + return 0; +} + static int should_audit(PyInterpreterState *interp) @@ -1650,7 +1664,7 @@ sys_setdlopenflags_impl(PyObject *module, int new_val) /*[clinic end generated code: output=ec918b7fe0a37281 input=4c838211e857a77f]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - interp->dlopenflags = new_val; + _PyImport_SetDLOpenFlags(interp, new_val); Py_RETURN_NONE; } @@ -1668,7 +1682,8 @@ sys_getdlopenflags_impl(PyObject *module) /*[clinic end generated code: output=e92cd1bc5005da6e input=dc4ea0899c53b4b6]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - return PyLong_FromLong(interp->dlopenflags); + return PyLong_FromLong( + _PyImport_GetDLOpenFlags(interp)); } #endif /* HAVE_DLOPEN */ @@ -2279,22 +2294,10 @@ static PyMethodDef sys_methods[] = { static PyObject * list_builtin_module_names(void) { - PyObject *list = PyList_New(0); + PyObject *list = _PyImport_GetBuiltinModuleNames(); if (list == NULL) { return NULL; } - struct _inittab *inittab = _PyRuntime.imports.inittab; - for (Py_ssize_t i = 0; inittab[i].name != NULL; i++) { - PyObject *name = PyUnicode_FromString(inittab[i].name); - if (name == NULL) { - goto error; - } - if (PyList_Append(list, name) < 0) { - Py_DECREF(name); - goto error; - } - Py_DECREF(name); - } if (PyList_Sort(list) != 0) { goto error; } @@ -3411,11 +3414,10 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) PyInterpreterState *interp = tstate->interp; - PyObject *modules = PyDict_New(); + PyObject *modules = _PyImport_InitModules(interp); if (modules == NULL) { goto error; } - interp->modules = modules; PyObject *sysmod = _PyModule_CreateInitialized(&sysmodule, PYTHON_API_VERSION); if (sysmod == NULL) { @@ -3428,7 +3430,7 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) } interp->sysdict = Py_NewRef(sysdict); - if (PyDict_SetItemString(sysdict, "modules", interp->modules) < 0) { + if (PyDict_SetItemString(sysdict, "modules", modules) < 0) { goto error; } @@ -3442,7 +3444,7 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) return status; } - if (_PyImport_FixupBuiltin(sysmod, "sys", interp->modules) < 0) { + if (_PyImport_FixupBuiltin(sysmod, "sys", modules) < 0) { goto error; } From b365d88465d9228ce4e9e0be20b88e9e4056ad88 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 15 Feb 2023 16:05:07 -0700 Subject: [PATCH 15/76] gh-101758: Add a Test For Single-Phase Init Modules in Multiple Interpreters (gh-101920) The test verifies the behavior of single-phase init modules when loaded in multiple interpreters. https://github.com/python/cpython/issues/101758 --- Include/internal/pycore_import.h | 3 ++ Lib/test/test_imp.py | 73 ++++++++++++++++++++++++++++++ Modules/_testinternalcapi.c | 15 +++++++ Modules/_testsinglephase.c | 49 +++++++++++++++++++- Python/import.c | 76 +++++++++++++++++++++++++++++++- 5 files changed, 212 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index da766253ef6b9c..6ee7356b41c021 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -153,6 +153,9 @@ PyAPI_DATA(const struct _frozen *) _PyImport_FrozenStdlib; PyAPI_DATA(const struct _frozen *) _PyImport_FrozenTest; extern const struct _module_alias * _PyImport_FrozenAliases; +// for testing +PyAPI_FUNC(int) _PyImport_ClearExtension(PyObject *name, PyObject *filename); + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index c85ab92307de78..e81eb6f0a86fe8 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -10,10 +10,13 @@ from test.support import os_helper from test.support import script_helper from test.support import warnings_helper +import textwrap import unittest import warnings imp = warnings_helper.import_deprecated('imp') import _imp +import _testinternalcapi +import _xxsubinterpreters as _interpreters OS_PATH_NAME = os.path.__name__ @@ -251,6 +254,71 @@ def test_issue16421_multiple_modules_in_one_dll(self): with self.assertRaises(ImportError): imp.load_dynamic('nonexistent', pathname) + @requires_load_dynamic + def test_singlephase_multiple_interpreters(self): + # Currently, for every single-phrase init module loaded + # in multiple interpreters, those interpreters share a + # PyModuleDef for that object, which can be a problem. + + # This single-phase module has global state, which is shared + # by the interpreters. + import _testsinglephase + name = _testsinglephase.__name__ + filename = _testsinglephase.__file__ + + del sys.modules[name] + _testsinglephase._clear_globals() + _testinternalcapi.clear_extension(name, filename) + init_count = _testsinglephase.initialized_count() + assert init_count == -1, (init_count,) + + def clean_up(): + _testsinglephase._clear_globals() + _testinternalcapi.clear_extension(name, filename) + self.addCleanup(clean_up) + + interp1 = _interpreters.create(isolated=False) + self.addCleanup(_interpreters.destroy, interp1) + interp2 = _interpreters.create(isolated=False) + self.addCleanup(_interpreters.destroy, interp2) + + script = textwrap.dedent(f''' + import _testsinglephase + + expected = %d + init_count = _testsinglephase.initialized_count() + if init_count != expected: + raise Exception(init_count) + + lookedup = _testsinglephase.look_up_self() + if lookedup is not _testsinglephase: + raise Exception((_testsinglephase, lookedup)) + + # Attrs set in the module init func are in m_copy. + _initialized = _testsinglephase._initialized + initialized = _testsinglephase.initialized() + if _initialized != initialized: + raise Exception((_initialized, initialized)) + + # Attrs set after loading are not in m_copy. + if hasattr(_testsinglephase, 'spam'): + raise Exception(_testsinglephase.spam) + _testsinglephase.spam = expected + ''') + + # Use an interpreter that gets destroyed right away. + ret = support.run_in_subinterp(script % 1) + self.assertEqual(ret, 0) + + # The module's init func gets run again. + # The module's globals did not get destroyed. + _interpreters.run_string(interp1, script % 2) + + # The module's init func is not run again. + # The second interpreter copies the module's m_copy. + # However, globals are still shared. + _interpreters.run_string(interp2, script % 2) + @requires_load_dynamic def test_singlephase_variants(self): '''Exercise the most meaningful variants described in Python/import.c.''' @@ -260,6 +328,11 @@ def test_singlephase_variants(self): fileobj, pathname, _ = imp.find_module(basename) fileobj.close() + def clean_up(): + import _testsinglephase + _testsinglephase._clear_globals() + self.addCleanup(clean_up) + modules = {} def load(name): assert name not in modules diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index ba57719d92096b..632fac2de0c419 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -671,6 +671,20 @@ get_interp_settings(PyObject *self, PyObject *args) } +static PyObject * +clear_extension(PyObject *self, PyObject *args) +{ + PyObject *name = NULL, *filename = NULL; + if (!PyArg_ParseTuple(args, "OO:clear_extension", &name, &filename)) { + return NULL; + } + if (_PyImport_ClearExtension(name, filename) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -692,6 +706,7 @@ static PyMethodDef module_functions[] = { _TESTINTERNALCAPI_COMPILER_CODEGEN_METHODDEF _TESTINTERNALCAPI_OPTIMIZE_CFG_METHODDEF {"get_interp_settings", get_interp_settings, METH_VARARGS, NULL}, + {"clear_extension", clear_extension, METH_VARARGS, NULL}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index 9e8dd647ee761a..565221c887e5ae 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -17,12 +17,27 @@ typedef struct { PyObject *str_const; } module_state; + /* Process-global state is only used by _testsinglephase since it's the only one that does not support re-init. */ static struct { int initialized_count; module_state module; -} global_state = { .initialized_count = -1 }; +} global_state = { + +#define NOT_INITIALIZED -1 + .initialized_count = NOT_INITIALIZED, +}; + +static void clear_state(module_state *state); + +static void +clear_global_state(void) +{ + clear_state(&global_state.module); + global_state.initialized_count = NOT_INITIALIZED; +} + static inline module_state * get_module_state(PyObject *module) @@ -106,6 +121,7 @@ init_state(module_state *state) return -1; } + static int init_module(PyObject *module, module_state *state) { @@ -118,6 +134,16 @@ init_module(PyObject *module, module_state *state) if (PyModule_AddObjectRef(module, "str_const", state->str_const) != 0) { return -1; } + + double d = _PyTime_AsSecondsDouble(state->initialized); + PyObject *initialized = PyFloat_FromDouble(d); + if (initialized == NULL) { + return -1; + } + if (PyModule_AddObjectRef(module, "_initialized", initialized) != 0) { + return -1; + } + return 0; } @@ -198,10 +224,28 @@ basic_initialized_count(PyObject *self, PyObject *Py_UNUSED(ignored)) } #define INITIALIZED_COUNT_METHODDEF \ - {"initialized_count", basic_initialized_count, METH_VARARGS, \ + {"initialized_count", basic_initialized_count, METH_NOARGS, \ basic_initialized_count_doc} +PyDoc_STRVAR(basic__clear_globals_doc, +"_clear_globals()\n\ +\n\ +Free all global state and set it to uninitialized."); + +static PyObject * +basic__clear_globals(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + assert(PyModule_GetDef(self)->m_size == -1); + clear_global_state(); + Py_RETURN_NONE; +} + +#define _CLEAR_GLOBALS_METHODDEF \ + {"_clear_globals", basic__clear_globals, METH_NOARGS, \ + basic__clear_globals_doc} + + /*********************************************/ /* the _testsinglephase module (and aliases) */ /*********************************************/ @@ -223,6 +267,7 @@ static PyMethodDef TestMethods_Basic[] = { SUM_METHODDEF, INITIALIZED_METHODDEF, INITIALIZED_COUNT_METHODDEF, + _CLEAR_GLOBALS_METHODDEF, {NULL, NULL} /* sentinel */ }; diff --git a/Python/import.c b/Python/import.c index ae27aaf56848d6..87981668a30505 100644 --- a/Python/import.c +++ b/Python/import.c @@ -632,6 +632,28 @@ exec_builtin_or_dynamic(PyObject *mod) { } +static int clear_singlephase_extension(PyInterpreterState *interp, + PyObject *name, PyObject *filename); + +// Currently, this is only used for testing. +// (See _testinternalcapi.clear_extension().) +int +_PyImport_ClearExtension(PyObject *name, PyObject *filename) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + + /* Clearing a module's C globals is up to the module. */ + if (clear_singlephase_extension(interp, name, filename) < 0) { + return -1; + } + + // In the future we'll probably also make sure the extension's + // file handle (and DL handle) is closed (requires saving it). + + return 0; +} + + /*******************/ #if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) @@ -766,8 +788,30 @@ _extensions_cache_set(PyObject *filename, PyObject *name, PyModuleDef *def) return 0; } +static int +_extensions_cache_delete(PyObject *filename, PyObject *name) +{ + PyObject *extensions = EXTENSIONS; + if (extensions == NULL) { + return 0; + } + PyObject *key = PyTuple_Pack(2, filename, name); + if (key == NULL) { + return -1; + } + if (PyDict_DelItem(extensions, key) < 0) { + if (!PyErr_ExceptionMatches(PyExc_KeyError)) { + Py_DECREF(key); + return -1; + } + PyErr_Clear(); + } + Py_DECREF(key); + return 0; +} + static void -_extensions_cache_clear(void) +_extensions_cache_clear_all(void) { Py_CLEAR(EXTENSIONS); } @@ -890,6 +934,34 @@ import_find_extension(PyThreadState *tstate, PyObject *name, return mod; } +static int +clear_singlephase_extension(PyInterpreterState *interp, + PyObject *name, PyObject *filename) +{ + PyModuleDef *def = _extensions_cache_get(filename, name); + if (def == NULL) { + if (PyErr_Occurred()) { + return -1; + } + return 0; + } + + /* Clear data set when the module was initially loaded. */ + def->m_base.m_init = NULL; + Py_CLEAR(def->m_base.m_copy); + // We leave m_index alone since there's no reason to reset it. + + /* Clear the PyState_*Module() cache entry. */ + if (_modules_by_index_check(interp, def->m_base.m_index) == NULL) { + if (_modules_by_index_clear(interp, def) < 0) { + return -1; + } + } + + /* Clear the cached module def. */ + return _extensions_cache_delete(filename, name); +} + /*******************/ /* builtin modules */ @@ -2633,7 +2705,7 @@ void _PyImport_Fini(void) { /* Destroy the database used by _PyImport_{Fixup,Find}Extension */ - _extensions_cache_clear(); + _extensions_cache_clear_all(); if (import_lock != NULL) { PyThread_free_lock(import_lock); import_lock = NULL; From 3dea4ba6c1b9237893d23574f931f33c940b74e8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 15 Feb 2023 17:54:05 -0700 Subject: [PATCH 16/76] gh-101758: Fix the wasm Buildbots (gh-101943) They were broken by gh-101920. https://github.com/python/cpython/issues/101758 --- Lib/test/test_imp.py | 12 +++++++++++- Python/pystate.c | 4 ++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index e81eb6f0a86fe8..5997ffad8e1232 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -16,12 +16,21 @@ imp = warnings_helper.import_deprecated('imp') import _imp import _testinternalcapi -import _xxsubinterpreters as _interpreters +try: + import _xxsubinterpreters as _interpreters +except ModuleNotFoundError: + _interpreters = None OS_PATH_NAME = os.path.__name__ +def requires_subinterpreters(meth): + """Decorator to skip a test if subinterpreters are not supported.""" + return unittest.skipIf(_interpreters is None, + 'subinterpreters required')(meth) + + def requires_load_dynamic(meth): """Decorator to skip a test if not running under CPython or lacking imp.load_dynamic().""" @@ -254,6 +263,7 @@ def test_issue16421_multiple_modules_in_one_dll(self): with self.assertRaises(ImportError): imp.load_dynamic('nonexistent', pathname) + @requires_subinterpreters @requires_load_dynamic def test_singlephase_multiple_interpreters(self): # Currently, for every single-phrase init module loaded diff --git a/Python/pystate.c b/Python/pystate.c index 4770caaed0a363..32b17fd19e348f 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -197,6 +197,7 @@ gilstate_tss_clear(_PyRuntimeState *runtime) } +#ifndef NDEBUG static inline int tstate_is_alive(PyThreadState *tstate); static inline int @@ -204,6 +205,7 @@ tstate_is_bound(PyThreadState *tstate) { return tstate->_status.bound && !tstate->_status.unbound; } +#endif // !NDEBUG static void bind_gilstate_tstate(PyThreadState *); static void unbind_gilstate_tstate(PyThreadState *); @@ -1119,6 +1121,7 @@ _PyInterpreterState_LookUpID(int64_t requested_id) /* the per-thread runtime state */ /********************************/ +#ifndef NDEBUG static inline int tstate_is_alive(PyThreadState *tstate) { @@ -1127,6 +1130,7 @@ tstate_is_alive(PyThreadState *tstate) !tstate->_status.cleared && !tstate->_status.finalizing); } +#endif //---------- From 89ac665891dec1988bedec2ce9b2c4d016502a49 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 15 Feb 2023 18:16:00 -0700 Subject: [PATCH 17/76] gh-98627: Add an Optional Check for Extension Module Subinterpreter Compatibility (gh-99040) Enforcing (optionally) the restriction set by PEP 489 makes sense. Furthermore, this sets the stage for a potential restriction related to a per-interpreter GIL. This change includes the following: * add tests for extension module subinterpreter compatibility * add _PyInterpreterConfig.check_multi_interp_extensions * add Py_RTFLAGS_MULTI_INTERP_EXTENSIONS * add _PyImport_CheckSubinterpIncompatibleExtensionAllowed() * fail iff the module does not implement multi-phase init and the current interpreter is configured to check https://github.com/python/cpython/issues/98627 --- Include/cpython/initconfig.h | 3 + Include/cpython/pystate.h | 3 + Include/internal/pycore_import.h | 5 + Lib/test/support/import_helper.py | 18 ++ Lib/test/test_capi/check_config.py | 77 ++++++ Lib/test/test_capi/test_misc.py | 98 +++++++- Lib/test/test_embed.py | 4 +- Lib/test/test_import/__init__.py | 220 +++++++++++++++++- Lib/test/test_threading.py | 1 + ...2-11-02-20-23-47.gh-issue-98627.VJkdRM.rst | 5 + Modules/_testcapimodule.c | 12 +- Python/clinic/import.c.h | 33 ++- Python/import.c | 88 ++++++- Python/importdl.c | 5 + Python/pylifecycle.c | 4 + 15 files changed, 557 insertions(+), 19 deletions(-) create mode 100644 Lib/test/test_capi/check_config.py create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 6ce42b4c09502f..a070fa9ff3a038 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -248,6 +248,7 @@ typedef struct { int allow_exec; int allow_threads; int allow_daemon_threads; + int check_multi_interp_extensions; } _PyInterpreterConfig; #define _PyInterpreterConfig_INIT \ @@ -256,6 +257,7 @@ typedef struct { .allow_exec = 0, \ .allow_threads = 1, \ .allow_daemon_threads = 0, \ + .check_multi_interp_extensions = 1, \ } #define _PyInterpreterConfig_LEGACY_INIT \ @@ -264,6 +266,7 @@ typedef struct { .allow_exec = 1, \ .allow_threads = 1, \ .allow_daemon_threads = 1, \ + .check_multi_interp_extensions = 0, \ } /* --- Helper functions --------------------------------------- */ diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index be1fcb61fa2244..3efb241e8237e7 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -11,6 +11,9 @@ is available in a given context. For example, forking the process might not be allowed in the current interpreter (i.e. os.fork() would fail). */ +/* Set if import should check a module for subinterpreter support. */ +#define Py_RTFLAGS_MULTI_INTERP_EXTENSIONS (1UL << 8) + /* Set if threads are allowed. */ #define Py_RTFLAGS_THREADS (1UL << 10) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 6ee7356b41c021..b7ffe01c0c0e20 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -64,6 +64,7 @@ struct _import_state { /* override for config->use_frozen_modules (for tests) (-1: "off", 1: "on", 0: no override) */ int override_frozen_modules; + int override_multi_interp_extensions_check; #ifdef HAVE_DLOPEN int dlopenflags; #endif @@ -153,6 +154,10 @@ PyAPI_DATA(const struct _frozen *) _PyImport_FrozenStdlib; PyAPI_DATA(const struct _frozen *) _PyImport_FrozenTest; extern const struct _module_alias * _PyImport_FrozenAliases; +PyAPI_FUNC(int) _PyImport_CheckSubinterpIncompatibleExtensionAllowed( + const char *name); + + // for testing PyAPI_FUNC(int) _PyImport_ClearExtension(PyObject *name, PyObject *filename); diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 63a8a7952db7a6..772c0987c2ebef 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -105,6 +105,24 @@ def frozen_modules(enabled=True): _imp._override_frozen_modules_for_tests(0) +@contextlib.contextmanager +def multi_interp_extensions_check(enabled=True): + """Force legacy modules to be allowed in subinterpreters (or not). + + ("legacy" == single-phase init) + + This only applies to modules that haven't been imported yet. + It overrides the PyInterpreterConfig.check_multi_interp_extensions + setting (see support.run_in_subinterp_with_config() and + _xxsubinterpreters.create()). + """ + old = _imp._override_multi_interp_extensions_check(1 if enabled else -1) + try: + yield + finally: + _imp._override_multi_interp_extensions_check(old) + + def import_fresh_module(name, fresh=(), blocked=(), *, deprecated=False, usefrozen=False, diff --git a/Lib/test/test_capi/check_config.py b/Lib/test/test_capi/check_config.py new file mode 100644 index 00000000000000..aaedd82f39af50 --- /dev/null +++ b/Lib/test/test_capi/check_config.py @@ -0,0 +1,77 @@ +# This script is used by test_misc. + +import _imp +import _testinternalcapi +import json +import os +import sys + + +def import_singlephase(): + assert '_testsinglephase' not in sys.modules + try: + import _testsinglephase + except ImportError: + sys.modules.pop('_testsinglephase') + return False + else: + del sys.modules['_testsinglephase'] + return True + + +def check_singlephase(override): + # Check using the default setting. + settings_initial = _testinternalcapi.get_interp_settings() + allowed_initial = import_singlephase() + assert(_testinternalcapi.get_interp_settings() == settings_initial) + + # Apply the override and check. + override_initial = _imp._override_multi_interp_extensions_check(override) + settings_after = _testinternalcapi.get_interp_settings() + allowed_after = import_singlephase() + + # Apply the override again and check. + noop = {} + override_after = _imp._override_multi_interp_extensions_check(override) + settings_noop = _testinternalcapi.get_interp_settings() + if settings_noop != settings_after: + noop['settings_noop'] = settings_noop + allowed_noop = import_singlephase() + if allowed_noop != allowed_after: + noop['allowed_noop'] = allowed_noop + + # Restore the original setting and check. + override_noop = _imp._override_multi_interp_extensions_check(override_initial) + if override_noop != override_after: + noop['override_noop'] = override_noop + settings_restored = _testinternalcapi.get_interp_settings() + allowed_restored = import_singlephase() + + # Restore the original setting again. + override_restored = _imp._override_multi_interp_extensions_check(override_initial) + assert(_testinternalcapi.get_interp_settings() == settings_restored) + + return dict({ + 'requested': override, + 'override__initial': override_initial, + 'override_after': override_after, + 'override_restored': override_restored, + 'settings__initial': settings_initial, + 'settings_after': settings_after, + 'settings_restored': settings_restored, + 'allowed__initial': allowed_initial, + 'allowed_after': allowed_after, + 'allowed_restored': allowed_restored, + }, **noop) + + +def run_singlephase_check(override, outfd): + with os.fdopen(outfd, 'w') as outfile: + sys.stdout = outfile + sys.stderr = outfile + try: + results = check_singlephase(override) + json.dump(results, outfile) + finally: + sys.stdout = sys.__stdout__ + sys.stderr = sys.__stderr__ diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 7612cddb1f6576..f26b4723d1e68b 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -31,6 +31,10 @@ import _testmultiphase except ImportError: _testmultiphase = None +try: + import _testsinglephase +except ImportError: + _testsinglephase = None # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module('_testcapi') @@ -1297,17 +1301,20 @@ def test_configured_settings(self): """ import json + EXTENSIONS = 1<<8 THREADS = 1<<10 DAEMON_THREADS = 1<<11 FORK = 1<<15 EXEC = 1<<16 - features = ['fork', 'exec', 'threads', 'daemon_threads'] + features = ['fork', 'exec', 'threads', 'daemon_threads', 'extensions'] kwlist = [f'allow_{n}' for n in features] + kwlist[-1] = 'check_multi_interp_extensions' for config, expected in { - (True, True, True, True): FORK | EXEC | THREADS | DAEMON_THREADS, - (False, False, False, False): 0, - (False, False, True, False): THREADS, + (True, True, True, True, True): + FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS, + (False, False, False, False, False): 0, + (False, False, True, False, True): THREADS | EXTENSIONS, }.items(): kwargs = dict(zip(kwlist, config)) expected = { @@ -1322,12 +1329,93 @@ def test_configured_settings(self): json.dump(settings, stdin) ''') with os.fdopen(r) as stdout: - support.run_in_subinterp_with_config(script, **kwargs) + ret = support.run_in_subinterp_with_config(script, **kwargs) + self.assertEqual(ret, 0) out = stdout.read() settings = json.loads(out) self.assertEqual(settings, expected) + @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_overridden_setting_extensions_subinterp_check(self): + """ + PyInterpreterConfig.check_multi_interp_extensions can be overridden + with PyInterpreterState.override_multi_interp_extensions_check. + This verifies that the override works but does not modify + the underlying setting. + """ + import json + + EXTENSIONS = 1<<8 + THREADS = 1<<10 + DAEMON_THREADS = 1<<11 + FORK = 1<<15 + EXEC = 1<<16 + BASE_FLAGS = FORK | EXEC | THREADS | DAEMON_THREADS + base_kwargs = { + 'allow_fork': True, + 'allow_exec': True, + 'allow_threads': True, + 'allow_daemon_threads': True, + } + + def check(enabled, override): + kwargs = dict( + base_kwargs, + check_multi_interp_extensions=enabled, + ) + flags = BASE_FLAGS | EXTENSIONS if enabled else BASE_FLAGS + settings = { + 'feature_flags': flags, + } + + expected = { + 'requested': override, + 'override__initial': 0, + 'override_after': override, + 'override_restored': 0, + # The override should not affect the config or settings. + 'settings__initial': settings, + 'settings_after': settings, + 'settings_restored': settings, + # These are the most likely values to be wrong. + 'allowed__initial': not enabled, + 'allowed_after': not ((override > 0) if override else enabled), + 'allowed_restored': not enabled, + } + + r, w = os.pipe() + script = textwrap.dedent(f''' + from test.test_capi.check_config import run_singlephase_check + run_singlephase_check({override}, {w}) + ''') + with os.fdopen(r) as stdout: + ret = support.run_in_subinterp_with_config(script, **kwargs) + self.assertEqual(ret, 0) + out = stdout.read() + results = json.loads(out) + + self.assertEqual(results, expected) + + self.maxDiff = None + + # setting: check disabled + with self.subTest('config: check disabled; override: disabled'): + check(False, -1) + with self.subTest('config: check disabled; override: use config'): + check(False, 0) + with self.subTest('config: check disabled; override: enabled'): + check(False, 1) + + # setting: check enabled + with self.subTest('config: check enabled; override: disabled'): + check(True, -1) + with self.subTest('config: check enabled; override: use config'): + check(True, 0) + with self.subTest('config: check enabled; override: enabled'): + check(True, 1) + def test_mutate_exception(self): """ Exceptions saved in global module state get shared between diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 4d422da5b99f44..e56d0db8627e91 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1656,13 +1656,15 @@ def test_init_use_frozen_modules(self): api=API_PYTHON, env=env) def test_init_main_interpreter_settings(self): + EXTENSIONS = 1<<8 THREADS = 1<<10 DAEMON_THREADS = 1<<11 FORK = 1<<15 EXEC = 1<<16 expected = { # All optional features should be enabled. - 'feature_flags': FORK | EXEC | THREADS | DAEMON_THREADS, + 'feature_flags': + FORK | EXEC | THREADS | DAEMON_THREADS, } out, err = self.run_embedded_interpreter( 'test_init_main_interpreter_settings', diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 1e4429ed7efe13..96815b3f758a5b 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -21,7 +21,7 @@ from test.support import os_helper from test.support import ( STDLIB_DIR, swap_attr, swap_item, cpython_only, is_emscripten, - is_wasi) + is_wasi, run_in_subinterp_with_config) from test.support.import_helper import ( forget, make_legacy_pyc, unlink, unload, DirsOnSysPath, CleanImport) from test.support.os_helper import ( @@ -30,6 +30,14 @@ from test.support import threading_helper from test.test_importlib.util import uncache from types import ModuleType +try: + import _testsinglephase +except ImportError: + _testsinglephase = None +try: + import _testmultiphase +except ImportError: + _testmultiphase = None skip_if_dont_write_bytecode = unittest.skipIf( @@ -1392,6 +1400,216 @@ def test_unwritable_module(self): unwritable.x = 42 +class SubinterpImportTests(unittest.TestCase): + + RUN_KWARGS = dict( + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + ) + + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def pipe(self): + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + if hasattr(os, 'set_blocking'): + os.set_blocking(r, False) + return (r, w) + + def import_script(self, name, fd, check_override=None): + override_text = '' + if check_override is not None: + override_text = f''' + import _imp + _imp._override_multi_interp_extensions_check({check_override}) + ''' + return textwrap.dedent(f''' + import os, sys + {override_text} + try: + import {name} + except ImportError as exc: + text = 'ImportError: ' + str(exc) + else: + text = 'okay' + os.write({fd}, text.encode('utf-8')) + ''') + + def run_shared(self, name, *, + check_singlephase_setting=False, + check_singlephase_override=None, + ): + """ + Try importing the named module in a subinterpreter. + + The subinterpreter will be in the current process. + The module will have already been imported in the main interpreter. + Thus, for extension/builtin modules, the module definition will + have been loaded already and cached globally. + + "check_singlephase_setting" determines whether or not + the interpreter will be configured to check for modules + that are not compatible with use in multiple interpreters. + + This should always return "okay" for all modules if the + setting is False (with no override). + """ + __import__(name) + + kwargs = dict( + **self.RUN_KWARGS, + check_multi_interp_extensions=check_singlephase_setting, + ) + + r, w = self.pipe() + script = self.import_script(name, w, check_singlephase_override) + + ret = run_in_subinterp_with_config(script, **kwargs) + self.assertEqual(ret, 0) + return os.read(r, 100) + + def check_compatible_shared(self, name, *, strict=False): + # Verify that the named module may be imported in a subinterpreter. + # (See run_shared() for more info.) + out = self.run_shared(name, check_singlephase_setting=strict) + self.assertEqual(out, b'okay') + + def check_incompatible_shared(self, name): + # Differences from check_compatible_shared(): + # * verify that import fails + # * "strict" is always True + out = self.run_shared(name, check_singlephase_setting=True) + self.assertEqual( + out.decode('utf-8'), + f'ImportError: module {name} does not support loading in subinterpreters', + ) + + def check_compatible_isolated(self, name, *, strict=False): + # Differences from check_compatible_shared(): + # * subinterpreter in a new process + # * module has never been imported before in that process + # * this tests importing the module for the first time + _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' + import _testcapi, sys + assert ( + {name!r} in sys.builtin_module_names or + {name!r} not in sys.modules + ), repr({name!r}) + ret = _testcapi.run_in_subinterp_with_config( + {self.import_script(name, "sys.stdout.fileno()")!r}, + **{self.RUN_KWARGS}, + check_multi_interp_extensions={strict}, + ) + assert ret == 0, ret + ''')) + self.assertEqual(err, b'') + self.assertEqual(out, b'okay') + + def check_incompatible_isolated(self, name): + # Differences from check_compatible_isolated(): + # * verify that import fails + # * "strict" is always True + _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' + import _testcapi, sys + assert {name!r} not in sys.modules, {name!r} + ret = _testcapi.run_in_subinterp_with_config( + {self.import_script(name, "sys.stdout.fileno()")!r}, + **{self.RUN_KWARGS}, + check_multi_interp_extensions=True, + ) + assert ret == 0, ret + ''')) + self.assertEqual(err, b'') + self.assertEqual( + out.decode('utf-8'), + f'ImportError: module {name} does not support loading in subinterpreters', + ) + + def test_builtin_compat(self): + module = 'sys' + with self.subTest(f'{module}: not strict'): + self.check_compatible_shared(module, strict=False) + with self.subTest(f'{module}: strict, shared'): + self.check_compatible_shared(module, strict=True) + + @cpython_only + def test_frozen_compat(self): + module = '_frozen_importlib' + if __import__(module).__spec__.origin != 'frozen': + raise unittest.SkipTest(f'{module} is unexpectedly not frozen') + with self.subTest(f'{module}: not strict'): + self.check_compatible_shared(module, strict=False) + with self.subTest(f'{module}: strict, shared'): + self.check_compatible_shared(module, strict=True) + + @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") + def test_single_init_extension_compat(self): + module = '_testsinglephase' + with self.subTest(f'{module}: not strict'): + self.check_compatible_shared(module, strict=False) + with self.subTest(f'{module}: strict, shared'): + self.check_incompatible_shared(module) + with self.subTest(f'{module}: strict, isolated'): + self.check_incompatible_isolated(module) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_multi_init_extension_compat(self): + module = '_testmultiphase' + with self.subTest(f'{module}: not strict'): + self.check_compatible_shared(module, strict=False) + with self.subTest(f'{module}: strict, shared'): + self.check_compatible_shared(module, strict=True) + with self.subTest(f'{module}: strict, isolated'): + self.check_compatible_isolated(module, strict=True) + + def test_python_compat(self): + module = 'threading' + if __import__(module).__spec__.origin == 'frozen': + raise unittest.SkipTest(f'{module} is unexpectedly frozen') + with self.subTest(f'{module}: not strict'): + self.check_compatible_shared(module, strict=False) + with self.subTest(f'{module}: strict, shared'): + self.check_compatible_shared(module, strict=True) + with self.subTest(f'{module}: strict, isolated'): + self.check_compatible_isolated(module, strict=True) + + @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") + def test_singlephase_check_with_setting_and_override(self): + module = '_testsinglephase' + + def check_compatible(setting, override): + out = self.run_shared( + module, + check_singlephase_setting=setting, + check_singlephase_override=override, + ) + self.assertEqual(out, b'okay') + + def check_incompatible(setting, override): + out = self.run_shared( + module, + check_singlephase_setting=setting, + check_singlephase_override=override, + ) + self.assertNotEqual(out, b'okay') + + with self.subTest('config: check enabled; override: enabled'): + check_incompatible(True, 1) + with self.subTest('config: check enabled; override: use config'): + check_incompatible(True, 0) + with self.subTest('config: check enabled; override: disabled'): + check_compatible(True, -1) + + with self.subTest('config: check disabled; override: enabled'): + check_incompatible(False, 1) + with self.subTest('config: check disabled; override: use config'): + check_compatible(False, 0) + with self.subTest('config: check disabled; override: disabled'): + check_compatible(False, -1) + + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. unittest.main() diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 31bf46311a80dc..7fea2d38673eff 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1347,6 +1347,7 @@ def func(): allow_exec=True, allow_threads={allowed}, allow_daemon_threads={daemon_allowed}, + check_multi_interp_extensions=False, ) """) with test.support.SuppressCrashReport(): diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst new file mode 100644 index 00000000000000..3d2d6f6eb0c41f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst @@ -0,0 +1,5 @@ +When an interpreter is configured to check (and only then), importing an +extension module will now fail when the extension does not support multiple +interpreters (i.e. doesn't implement PEP 489 multi-phase init). This does +not apply to the main interpreter, nor to subinterpreters created with +``Py_NewInterpreter()``. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 5610a7689136f6..0d8d1d73fb2390 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1618,6 +1618,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) int allow_exec = -1; int allow_threads = -1; int allow_daemon_threads = -1; + int check_multi_interp_extensions = -1; int r; PyThreadState *substate, *mainstate; /* only initialise 'cflags.cf_flags' to test backwards compatibility */ @@ -1628,11 +1629,13 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) "allow_exec", "allow_threads", "allow_daemon_threads", + "check_multi_interp_extensions", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "s$pppp:run_in_subinterp_with_config", kwlist, + "s$ppppp:run_in_subinterp_with_config", kwlist, &code, &allow_fork, &allow_exec, - &allow_threads, &allow_daemon_threads)) { + &allow_threads, &allow_daemon_threads, + &check_multi_interp_extensions)) { return NULL; } if (allow_fork < 0) { @@ -1651,6 +1654,10 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) PyErr_SetString(PyExc_ValueError, "missing allow_daemon_threads"); return NULL; } + if (check_multi_interp_extensions < 0) { + PyErr_SetString(PyExc_ValueError, "missing check_multi_interp_extensions"); + return NULL; + } mainstate = PyThreadState_Get(); @@ -1661,6 +1668,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) .allow_exec = allow_exec, .allow_threads = allow_threads, .allow_daemon_threads = allow_daemon_threads, + .check_multi_interp_extensions = check_multi_interp_extensions, }; substate = _Py_NewInterpreterFromConfig(&config); if (substate == NULL) { diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h index 819fb1c75c15c3..cb74be6a422124 100644 --- a/Python/clinic/import.c.h +++ b/Python/clinic/import.c.h @@ -442,6 +442,37 @@ _imp__override_frozen_modules_for_tests(PyObject *module, PyObject *arg) return return_value; } +PyDoc_STRVAR(_imp__override_multi_interp_extensions_check__doc__, +"_override_multi_interp_extensions_check($module, override, /)\n" +"--\n" +"\n" +"(internal-only) Override PyInterpreterConfig.check_multi_interp_extensions.\n" +"\n" +"(-1: \"never\", 1: \"always\", 0: no override)"); + +#define _IMP__OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK_METHODDEF \ + {"_override_multi_interp_extensions_check", (PyCFunction)_imp__override_multi_interp_extensions_check, METH_O, _imp__override_multi_interp_extensions_check__doc__}, + +static PyObject * +_imp__override_multi_interp_extensions_check_impl(PyObject *module, + int override); + +static PyObject * +_imp__override_multi_interp_extensions_check(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int override; + + override = _PyLong_AsInt(arg); + if (override == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _imp__override_multi_interp_extensions_check_impl(module, override); + +exit: + return return_value; +} + #if defined(HAVE_DYNAMIC_LOADING) PyDoc_STRVAR(_imp_create_dynamic__doc__, @@ -617,4 +648,4 @@ _imp_source_hash(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb #ifndef _IMP_EXEC_DYNAMIC_METHODDEF #define _IMP_EXEC_DYNAMIC_METHODDEF #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */ -/*[clinic end generated code: output=806352838c3f7008 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b18d46e0036eff49 input=a9049054013a1b77]*/ diff --git a/Python/import.c b/Python/import.c index 87981668a30505..ec126f28b85816 100644 --- a/Python/import.c +++ b/Python/import.c @@ -74,6 +74,8 @@ static struct _inittab *inittab_copy = NULL; (interp)->imports.modules_by_index #define IMPORTLIB(interp) \ (interp)->imports.importlib +#define OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp) \ + (interp)->imports.override_multi_interp_extensions_check #define OVERRIDE_FROZEN_MODULES(interp) \ (interp)->imports.override_frozen_modules #ifdef HAVE_DLOPEN @@ -816,6 +818,38 @@ _extensions_cache_clear_all(void) Py_CLEAR(EXTENSIONS); } + +static bool +check_multi_interp_extensions(PyInterpreterState *interp) +{ + int override = OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp); + if (override < 0) { + return false; + } + else if (override > 0) { + return true; + } + else if (_PyInterpreterState_HasFeature( + interp, Py_RTFLAGS_MULTI_INTERP_EXTENSIONS)) { + return true; + } + return false; +} + +int +_PyImport_CheckSubinterpIncompatibleExtensionAllowed(const char *name) +{ + PyInterpreterState *interp = _PyInterpreterState_Get(); + if (check_multi_interp_extensions(interp)) { + assert(!_Py_IsMainInterpreter(interp)); + PyErr_Format(PyExc_ImportError, + "module %s does not support loading in subinterpreters", + name); + return -1; + } + return 0; +} + static int fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) { @@ -3297,6 +3331,34 @@ _imp__override_frozen_modules_for_tests_impl(PyObject *module, int override) Py_RETURN_NONE; } +/*[clinic input] +_imp._override_multi_interp_extensions_check + + override: int + / + +(internal-only) Override PyInterpreterConfig.check_multi_interp_extensions. + +(-1: "never", 1: "always", 0: no override) +[clinic start generated code]*/ + +static PyObject * +_imp__override_multi_interp_extensions_check_impl(PyObject *module, + int override) +/*[clinic end generated code: output=3ff043af52bbf280 input=e086a2ea181f92ae]*/ +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_Py_IsMainInterpreter(interp)) { + PyErr_SetString(PyExc_RuntimeError, + "_imp._override_multi_interp_extensions_check() " + "cannot be used in the main interpreter"); + return NULL; + } + int oldvalue = OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp); + OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp) = override; + return PyLong_FromLong(oldvalue); +} + #ifdef HAVE_DYNAMIC_LOADING /*[clinic input] @@ -3329,18 +3391,23 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) PyThreadState *tstate = _PyThreadState_GET(); mod = import_find_extension(tstate, name, path); - if (mod != NULL || PyErr_Occurred()) { - Py_DECREF(name); - Py_DECREF(path); - return mod; + if (mod != NULL) { + const char *name_buf = PyUnicode_AsUTF8(name); + assert(name_buf != NULL); + if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { + Py_DECREF(mod); + mod = NULL; + } + goto finally; + } + else if (PyErr_Occurred()) { + goto finally; } if (file != NULL) { fp = _Py_fopen_obj(path, "r"); if (fp == NULL) { - Py_DECREF(name); - Py_DECREF(path); - return NULL; + goto finally; } } else @@ -3348,10 +3415,12 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) mod = _PyImport_LoadDynamicModuleWithSpec(spec, fp); - Py_DECREF(name); - Py_DECREF(path); if (fp) fclose(fp); + +finally: + Py_DECREF(name); + Py_DECREF(path); return mod; } @@ -3436,6 +3505,7 @@ static PyMethodDef imp_methods[] = { _IMP_IS_FROZEN_METHODDEF _IMP__FROZEN_MODULE_NAMES_METHODDEF _IMP__OVERRIDE_FROZEN_MODULES_FOR_TESTS_METHODDEF + _IMP__OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK_METHODDEF _IMP_CREATE_DYNAMIC_METHODDEF _IMP_EXEC_DYNAMIC_METHODDEF _IMP_EXEC_BUILTIN_METHODDEF diff --git a/Python/importdl.c b/Python/importdl.c index 6dafb4541486e9..3a3a30ddbdcdb5 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_call.h" +#include "pycore_import.h" #include "pycore_pystate.h" #include "pycore_runtime.h" @@ -203,6 +204,10 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) /* Fall back to single-phase init mechanism */ + if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { + goto error; + } + if (hook_prefix == nonascii_prefix) { /* don't allow legacy init for non-ASCII module names */ PyErr_Format( diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 281035dafa9577..e80dd30c89dfd0 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -565,6 +565,10 @@ init_interp_settings(PyInterpreterState *interp, const _PyInterpreterConfig *con if (config->allow_daemon_threads) { interp->feature_flags |= Py_RTFLAGS_DAEMON_THREADS; } + + if (config->check_multi_interp_extensions) { + interp->feature_flags |= Py_RTFLAGS_MULTI_INTERP_EXTENSIONS; + } } From 0b13575e74ff3321364a3389eda6b4e92792afe1 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Wed, 15 Feb 2023 22:08:20 -0800 Subject: [PATCH 18/76] gh-99108: Refactor _sha256 & _sha512 into _sha2. (#101924) This merges their code. They're backed by the same single HACL* static library, having them be a single module simplifies maintenance. This should unbreak the wasm enscripten builds that currently fail due to linking in --whole-archive mode and the HACL* library appearing twice. Long unnoticed error fixed: _sha512.SHA384Type was doubly assigned and was actually SHA512Type. Nobody depends on those internal names. Also rename LIBHACL_ make vars to LIBHACL_SHA2_ in preperation for other future HACL things. --- Lib/hashlib.py | 12 +- Lib/test/test_hashlib.py | 22 +- Makefile.pre.in | 15 +- ...3-02-15-01-54-06.gh-issue-99108.rjTSic.rst | 3 + Modules/Setup | 3 +- Modules/Setup.stdlib.in | 3 +- Modules/clinic/sha256module.c.h | 225 ----- Modules/clinic/sha2module.c.h | 440 ++++++++++ Modules/clinic/sha512module.c.h | 225 ----- Modules/sha256module.c | 465 ---------- Modules/sha2module.c | 805 ++++++++++++++++++ Modules/sha512module.c | 456 ---------- PC/config.c | 6 +- PCbuild/pythoncore.vcxproj | 3 +- PCbuild/pythoncore.vcxproj.filters | 5 +- Python/stdlib_module_names.h | 3 +- configure | 96 +-- configure.ac | 16 +- 18 files changed, 1310 insertions(+), 1493 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst delete mode 100644 Modules/clinic/sha256module.c.h create mode 100644 Modules/clinic/sha2module.c.h delete mode 100644 Modules/clinic/sha512module.c.h delete mode 100644 Modules/sha256module.c create mode 100644 Modules/sha2module.c delete mode 100644 Modules/sha512module.c diff --git a/Lib/hashlib.py b/Lib/hashlib.py index 21b5e912f3c771..1b16441cb60ba7 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -92,13 +92,13 @@ def __get_builtin_constructor(name): import _md5 cache['MD5'] = cache['md5'] = _md5.md5 elif name in {'SHA256', 'sha256', 'SHA224', 'sha224'}: - import _sha256 - cache['SHA224'] = cache['sha224'] = _sha256.sha224 - cache['SHA256'] = cache['sha256'] = _sha256.sha256 + import _sha2 + cache['SHA224'] = cache['sha224'] = _sha2.sha224 + cache['SHA256'] = cache['sha256'] = _sha2.sha256 elif name in {'SHA512', 'sha512', 'SHA384', 'sha384'}: - import _sha512 - cache['SHA384'] = cache['sha384'] = _sha512.sha384 - cache['SHA512'] = cache['sha512'] = _sha512.sha512 + import _sha2 + cache['SHA384'] = cache['sha384'] = _sha2.sha384 + cache['SHA512'] = cache['sha512'] = _sha2.sha512 elif name in {'blake2b', 'blake2s'}: import _blake2 cache['blake2b'] = _blake2.blake2b diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 9c92b4e9c280dc..5ead8857943592 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -1,6 +1,4 @@ -# Test hashlib module -# -# $Id$ +# Test the hashlib module. # # Copyright (C) 2005-2010 Gregory P. Smith (greg@krypto.org) # Licensed to PSF under a Contributor Agreement. @@ -28,7 +26,6 @@ from http.client import HTTPException -# default builtin hash module default_builtin_hashes = {'md5', 'sha1', 'sha256', 'sha512', 'sha3', 'blake2'} # --with-builtin-hashlib-hashes override builtin_hashes = sysconfig.get_config_var("PY_BUILTIN_HASHLIB_HASHES") @@ -66,6 +63,7 @@ def get_fips_mode(): requires_blake2 = unittest.skipUnless(_blake2, 'requires _blake2') # bpo-46913: Don't test the _sha3 extension on a Python UBSAN build +# TODO(gh-99108): Revisit this after _sha3 uses HACL*. SKIP_SHA3 = support.check_sanitizer(ub=True) requires_sha3 = unittest.skipUnless(not SKIP_SHA3, 'requires _sha3') @@ -107,7 +105,7 @@ class HashLibTestCase(unittest.TestCase): shakes = {'shake_128', 'shake_256'} - # Issue #14693: fallback modules are always compiled under POSIX + # gh-58898: Fallback modules are always compiled under POSIX. _warn_on_extension_import = (os.name == 'posix' or support.Py_DEBUG) def _conditional_import_module(self, module_name): @@ -116,7 +114,7 @@ def _conditional_import_module(self, module_name): return importlib.import_module(module_name) except ModuleNotFoundError as error: if self._warn_on_extension_import and module_name in builtin_hashes: - warnings.warn('Did a C extension fail to compile? %s' % error) + warnings.warn(f'Did a C extension fail to compile? {error}') return None def __init__(self, *args, **kwargs): @@ -147,7 +145,7 @@ def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs): _hashlib = self._conditional_import_module('_hashlib') self._hashlib = _hashlib if _hashlib: - # These two algorithms should always be present when this module + # These algorithms should always be present when this module # is compiled. If not, something was compiled wrong. self.assertTrue(hasattr(_hashlib, 'openssl_md5')) self.assertTrue(hasattr(_hashlib, 'openssl_sha1')) @@ -172,12 +170,10 @@ def add_builtin_constructor(name): _sha1 = self._conditional_import_module('_sha1') if _sha1: add_builtin_constructor('sha1') - _sha256 = self._conditional_import_module('_sha256') - if _sha256: + _sha2 = self._conditional_import_module('_sha2') + if _sha2: add_builtin_constructor('sha224') add_builtin_constructor('sha256') - _sha512 = self._conditional_import_module('_sha512') - if _sha512: add_builtin_constructor('sha384') add_builtin_constructor('sha512') if _blake2: @@ -460,9 +456,9 @@ def check_blocksize_name(self, name, block_size=0, digest_size=0, self.assertEqual(len(m.hexdigest()), 2*digest_size) self.assertEqual(m.name, name) # split for sha3_512 / _sha3.sha3 object - self.assertIn(name.split("_")[0], repr(m)) + self.assertIn(name.split("_")[0], repr(m).lower()) - def test_blocksize_name(self): + def test_blocksize_and_name(self): self.check_blocksize_name('md5', 64, 16) self.check_blocksize_name('sha1', 64, 20) self.check_blocksize_name('sha224', 64, 28) diff --git a/Makefile.pre.in b/Makefile.pre.in index ce3fed3d648536..490483a712014c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -207,7 +207,7 @@ ENSUREPIP= @ENSUREPIP@ # Internal static libraries LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a LIBEXPAT_A= Modules/expat/libexpat.a -LIBHACL_A= Modules/_hacl/libHacl_Streaming_SHA2.a +LIBHACL_SHA2_A= Modules/_hacl/libHacl_Streaming_SHA2.a # Module state, compiler flags and linker flags # Empty CFLAGS and LDFLAGS are omitted. @@ -575,10 +575,10 @@ LIBEXPAT_HEADERS= \ ########################################################################## # hashlib's HACL* library -LIBHACL_OBJS= \ +LIBHACL_SHA2_OBJS= \ Modules/_hacl/Hacl_Streaming_SHA2.o -LIBHACL_HEADERS= \ +LIBHACL_SHA2_HEADERS= \ Modules/_hacl/Hacl_Streaming_SHA2.h \ Modules/_hacl/include/krml/FStar_UInt128_Verified.h \ Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h \ @@ -912,12 +912,12 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS) # Build HACL* static libraries for hashlib: libHacl_Streaming_SHA2.a LIBHACL_CFLAGS=-I$(srcdir)/Modules/_hacl/include -D_BSD_SOURCE -D_DEFAULT_SOURCE $(PY_STDMODULE_CFLAGS) $(CCSHARED) -Modules/_hacl/Hacl_Streaming_SHA2.o: $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.c $(LIBHACL_HEADERS) +Modules/_hacl/Hacl_Streaming_SHA2.o: $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.c $(LIBHACL_SHA2_HEADERS) $(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.c -$(LIBHACL_A): $(LIBHACL_OBJS) +$(LIBHACL_SHA2_A): $(LIBHACL_SHA2_OBJS) -rm -f $@ - $(AR) $(ARFLAGS) $@ $(LIBHACL_OBJS) + $(AR) $(ARFLAGS) $@ $(LIBHACL_SHA2_OBJS) # create relative links from build/lib.platform/egg.so to Modules/egg.so # pybuilddir.txt is created too late. We cannot use it in Makefile @@ -2635,9 +2635,8 @@ MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h -MODULE__SHA256_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) $(LIBHACL_A) +MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_A) MODULE__SHA3_DEPS=$(srcdir)/Modules/_sha3/sha3.c $(srcdir)/Modules/_sha3/sha3.h $(srcdir)/Modules/hashlib.h -MODULE__SHA512_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) $(LIBHACL_A) MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data.h $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h diff --git a/Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst b/Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst new file mode 100644 index 00000000000000..1612c89c0ea6be --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst @@ -0,0 +1,3 @@ +The built-in extension modules for :mod:`hashlib` SHA2 algorithms, used when +OpenSSL does not provide them, now live in a single internal ``_sha2`` module +instead of separate ``_sha256`` and ``_sha512`` modules. diff --git a/Modules/Setup b/Modules/Setup index 428be0a1bf8fa1..1d5183bc2df118 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -165,8 +165,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c #_md5 md5module.c #_sha1 sha1module.c -#_sha256 sha256module.c -#_sha512 sha512module.c +#_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a #_sha3 _sha3/sha3module.c # text encodings and unicode diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 22bcb423db233f..8f5e14a4e80e22 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -79,8 +79,7 @@ # hashing builtins, can be disabled with --without-builtin-hashlib-hashes @MODULE__MD5_TRUE@_md5 md5module.c @MODULE__SHA1_TRUE@_sha1 sha1module.c -@MODULE__SHA256_TRUE@_sha256 sha256module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a -@MODULE__SHA512_TRUE@_sha512 sha512module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a +@MODULE__SHA2_TRUE@_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a @MODULE__SHA3_TRUE@_sha3 _sha3/sha3module.c @MODULE__BLAKE2_TRUE@_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c diff --git a/Modules/clinic/sha256module.c.h b/Modules/clinic/sha256module.c.h deleted file mode 100644 index 10d09fac695fc4..00000000000000 --- a/Modules/clinic/sha256module.c.h +++ /dev/null @@ -1,225 +0,0 @@ -/*[clinic input] -preserve -[clinic start generated code]*/ - -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - -PyDoc_STRVAR(SHA256Type_copy__doc__, -"copy($self, /)\n" -"--\n" -"\n" -"Return a copy of the hash object."); - -#define SHA256TYPE_COPY_METHODDEF \ - {"copy", _PyCFunction_CAST(SHA256Type_copy), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, SHA256Type_copy__doc__}, - -static PyObject * -SHA256Type_copy_impl(SHAobject *self, PyTypeObject *cls); - -static PyObject * -SHA256Type_copy(SHAobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - if (nargs) { - PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); - return NULL; - } - return SHA256Type_copy_impl(self, cls); -} - -PyDoc_STRVAR(SHA256Type_digest__doc__, -"digest($self, /)\n" -"--\n" -"\n" -"Return the digest value as a bytes object."); - -#define SHA256TYPE_DIGEST_METHODDEF \ - {"digest", (PyCFunction)SHA256Type_digest, METH_NOARGS, SHA256Type_digest__doc__}, - -static PyObject * -SHA256Type_digest_impl(SHAobject *self); - -static PyObject * -SHA256Type_digest(SHAobject *self, PyObject *Py_UNUSED(ignored)) -{ - return SHA256Type_digest_impl(self); -} - -PyDoc_STRVAR(SHA256Type_hexdigest__doc__, -"hexdigest($self, /)\n" -"--\n" -"\n" -"Return the digest value as a string of hexadecimal digits."); - -#define SHA256TYPE_HEXDIGEST_METHODDEF \ - {"hexdigest", (PyCFunction)SHA256Type_hexdigest, METH_NOARGS, SHA256Type_hexdigest__doc__}, - -static PyObject * -SHA256Type_hexdigest_impl(SHAobject *self); - -static PyObject * -SHA256Type_hexdigest(SHAobject *self, PyObject *Py_UNUSED(ignored)) -{ - return SHA256Type_hexdigest_impl(self); -} - -PyDoc_STRVAR(SHA256Type_update__doc__, -"update($self, obj, /)\n" -"--\n" -"\n" -"Update this hash object\'s state with the provided string."); - -#define SHA256TYPE_UPDATE_METHODDEF \ - {"update", (PyCFunction)SHA256Type_update, METH_O, SHA256Type_update__doc__}, - -PyDoc_STRVAR(_sha256_sha256__doc__, -"sha256($module, /, string=b\'\', *, usedforsecurity=True)\n" -"--\n" -"\n" -"Return a new SHA-256 hash object; optionally initialized with a string."); - -#define _SHA256_SHA256_METHODDEF \ - {"sha256", _PyCFunction_CAST(_sha256_sha256), METH_FASTCALL|METH_KEYWORDS, _sha256_sha256__doc__}, - -static PyObject * -_sha256_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity); - -static PyObject * -_sha256_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "sha256", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; - int usedforsecurity = 1; - - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); - if (!args) { - goto exit; - } - if (!noptargs) { - goto skip_optional_pos; - } - if (args[0]) { - string = args[0]; - if (!--noptargs) { - goto skip_optional_pos; - } - } -skip_optional_pos: - if (!noptargs) { - goto skip_optional_kwonly; - } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; - } -skip_optional_kwonly: - return_value = _sha256_sha256_impl(module, string, usedforsecurity); - -exit: - return return_value; -} - -PyDoc_STRVAR(_sha256_sha224__doc__, -"sha224($module, /, string=b\'\', *, usedforsecurity=True)\n" -"--\n" -"\n" -"Return a new SHA-224 hash object; optionally initialized with a string."); - -#define _SHA256_SHA224_METHODDEF \ - {"sha224", _PyCFunction_CAST(_sha256_sha224), METH_FASTCALL|METH_KEYWORDS, _sha256_sha224__doc__}, - -static PyObject * -_sha256_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity); - -static PyObject * -_sha256_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "sha224", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; - int usedforsecurity = 1; - - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); - if (!args) { - goto exit; - } - if (!noptargs) { - goto skip_optional_pos; - } - if (args[0]) { - string = args[0]; - if (!--noptargs) { - goto skip_optional_pos; - } - } -skip_optional_pos: - if (!noptargs) { - goto skip_optional_kwonly; - } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; - } -skip_optional_kwonly: - return_value = _sha256_sha224_impl(module, string, usedforsecurity); - -exit: - return return_value; -} -/*[clinic end generated code: output=ae926f7ec85e7c97 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha2module.c.h b/Modules/clinic/sha2module.c.h new file mode 100644 index 00000000000000..8f855ca345e47a --- /dev/null +++ b/Modules/clinic/sha2module.c.h @@ -0,0 +1,440 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +PyDoc_STRVAR(SHA256Type_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a copy of the hash object."); + +#define SHA256TYPE_COPY_METHODDEF \ + {"copy", _PyCFunction_CAST(SHA256Type_copy), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, SHA256Type_copy__doc__}, + +static PyObject * +SHA256Type_copy_impl(SHA256object *self, PyTypeObject *cls); + +static PyObject * +SHA256Type_copy(SHA256object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs) { + PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); + return NULL; + } + return SHA256Type_copy_impl(self, cls); +} + +PyDoc_STRVAR(SHA512Type_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a copy of the hash object."); + +#define SHA512TYPE_COPY_METHODDEF \ + {"copy", _PyCFunction_CAST(SHA512Type_copy), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, SHA512Type_copy__doc__}, + +static PyObject * +SHA512Type_copy_impl(SHA512object *self, PyTypeObject *cls); + +static PyObject * +SHA512Type_copy(SHA512object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs) { + PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); + return NULL; + } + return SHA512Type_copy_impl(self, cls); +} + +PyDoc_STRVAR(SHA256Type_digest__doc__, +"digest($self, /)\n" +"--\n" +"\n" +"Return the digest value as a bytes object."); + +#define SHA256TYPE_DIGEST_METHODDEF \ + {"digest", (PyCFunction)SHA256Type_digest, METH_NOARGS, SHA256Type_digest__doc__}, + +static PyObject * +SHA256Type_digest_impl(SHA256object *self); + +static PyObject * +SHA256Type_digest(SHA256object *self, PyObject *Py_UNUSED(ignored)) +{ + return SHA256Type_digest_impl(self); +} + +PyDoc_STRVAR(SHA512Type_digest__doc__, +"digest($self, /)\n" +"--\n" +"\n" +"Return the digest value as a bytes object."); + +#define SHA512TYPE_DIGEST_METHODDEF \ + {"digest", (PyCFunction)SHA512Type_digest, METH_NOARGS, SHA512Type_digest__doc__}, + +static PyObject * +SHA512Type_digest_impl(SHA512object *self); + +static PyObject * +SHA512Type_digest(SHA512object *self, PyObject *Py_UNUSED(ignored)) +{ + return SHA512Type_digest_impl(self); +} + +PyDoc_STRVAR(SHA256Type_hexdigest__doc__, +"hexdigest($self, /)\n" +"--\n" +"\n" +"Return the digest value as a string of hexadecimal digits."); + +#define SHA256TYPE_HEXDIGEST_METHODDEF \ + {"hexdigest", (PyCFunction)SHA256Type_hexdigest, METH_NOARGS, SHA256Type_hexdigest__doc__}, + +static PyObject * +SHA256Type_hexdigest_impl(SHA256object *self); + +static PyObject * +SHA256Type_hexdigest(SHA256object *self, PyObject *Py_UNUSED(ignored)) +{ + return SHA256Type_hexdigest_impl(self); +} + +PyDoc_STRVAR(SHA512Type_hexdigest__doc__, +"hexdigest($self, /)\n" +"--\n" +"\n" +"Return the digest value as a string of hexadecimal digits."); + +#define SHA512TYPE_HEXDIGEST_METHODDEF \ + {"hexdigest", (PyCFunction)SHA512Type_hexdigest, METH_NOARGS, SHA512Type_hexdigest__doc__}, + +static PyObject * +SHA512Type_hexdigest_impl(SHA512object *self); + +static PyObject * +SHA512Type_hexdigest(SHA512object *self, PyObject *Py_UNUSED(ignored)) +{ + return SHA512Type_hexdigest_impl(self); +} + +PyDoc_STRVAR(SHA256Type_update__doc__, +"update($self, obj, /)\n" +"--\n" +"\n" +"Update this hash object\'s state with the provided string."); + +#define SHA256TYPE_UPDATE_METHODDEF \ + {"update", (PyCFunction)SHA256Type_update, METH_O, SHA256Type_update__doc__}, + +PyDoc_STRVAR(SHA512Type_update__doc__, +"update($self, obj, /)\n" +"--\n" +"\n" +"Update this hash object\'s state with the provided string."); + +#define SHA512TYPE_UPDATE_METHODDEF \ + {"update", (PyCFunction)SHA512Type_update, METH_O, SHA512Type_update__doc__}, + +PyDoc_STRVAR(_sha2_sha256__doc__, +"sha256($module, /, string=b\'\', *, usedforsecurity=True)\n" +"--\n" +"\n" +"Return a new SHA-256 hash object; optionally initialized with a string."); + +#define _SHA2_SHA256_METHODDEF \ + {"sha256", _PyCFunction_CAST(_sha2_sha256), METH_FASTCALL|METH_KEYWORDS, _sha2_sha256__doc__}, + +static PyObject * +_sha2_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity); + +static PyObject * +_sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "sha256", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *string = NULL; + int usedforsecurity = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + string = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _sha2_sha256_impl(module, string, usedforsecurity); + +exit: + return return_value; +} + +PyDoc_STRVAR(_sha2_sha224__doc__, +"sha224($module, /, string=b\'\', *, usedforsecurity=True)\n" +"--\n" +"\n" +"Return a new SHA-224 hash object; optionally initialized with a string."); + +#define _SHA2_SHA224_METHODDEF \ + {"sha224", _PyCFunction_CAST(_sha2_sha224), METH_FASTCALL|METH_KEYWORDS, _sha2_sha224__doc__}, + +static PyObject * +_sha2_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity); + +static PyObject * +_sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "sha224", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *string = NULL; + int usedforsecurity = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + string = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _sha2_sha224_impl(module, string, usedforsecurity); + +exit: + return return_value; +} + +PyDoc_STRVAR(_sha2_sha512__doc__, +"sha512($module, /, string=b\'\', *, usedforsecurity=True)\n" +"--\n" +"\n" +"Return a new SHA-512 hash object; optionally initialized with a string."); + +#define _SHA2_SHA512_METHODDEF \ + {"sha512", _PyCFunction_CAST(_sha2_sha512), METH_FASTCALL|METH_KEYWORDS, _sha2_sha512__doc__}, + +static PyObject * +_sha2_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity); + +static PyObject * +_sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "sha512", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *string = NULL; + int usedforsecurity = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + string = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _sha2_sha512_impl(module, string, usedforsecurity); + +exit: + return return_value; +} + +PyDoc_STRVAR(_sha2_sha384__doc__, +"sha384($module, /, string=b\'\', *, usedforsecurity=True)\n" +"--\n" +"\n" +"Return a new SHA-384 hash object; optionally initialized with a string."); + +#define _SHA2_SHA384_METHODDEF \ + {"sha384", _PyCFunction_CAST(_sha2_sha384), METH_FASTCALL|METH_KEYWORDS, _sha2_sha384__doc__}, + +static PyObject * +_sha2_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity); + +static PyObject * +_sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "sha384", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *string = NULL; + int usedforsecurity = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + string = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _sha2_sha384_impl(module, string, usedforsecurity); + +exit: + return return_value; +} +/*[clinic end generated code: output=f81dacb48f3fee72 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha512module.c.h b/Modules/clinic/sha512module.c.h deleted file mode 100644 index f8d326363c398e..00000000000000 --- a/Modules/clinic/sha512module.c.h +++ /dev/null @@ -1,225 +0,0 @@ -/*[clinic input] -preserve -[clinic start generated code]*/ - -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - -PyDoc_STRVAR(SHA512Type_copy__doc__, -"copy($self, /)\n" -"--\n" -"\n" -"Return a copy of the hash object."); - -#define SHA512TYPE_COPY_METHODDEF \ - {"copy", _PyCFunction_CAST(SHA512Type_copy), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, SHA512Type_copy__doc__}, - -static PyObject * -SHA512Type_copy_impl(SHAobject *self, PyTypeObject *cls); - -static PyObject * -SHA512Type_copy(SHAobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - if (nargs) { - PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); - return NULL; - } - return SHA512Type_copy_impl(self, cls); -} - -PyDoc_STRVAR(SHA512Type_digest__doc__, -"digest($self, /)\n" -"--\n" -"\n" -"Return the digest value as a bytes object."); - -#define SHA512TYPE_DIGEST_METHODDEF \ - {"digest", (PyCFunction)SHA512Type_digest, METH_NOARGS, SHA512Type_digest__doc__}, - -static PyObject * -SHA512Type_digest_impl(SHAobject *self); - -static PyObject * -SHA512Type_digest(SHAobject *self, PyObject *Py_UNUSED(ignored)) -{ - return SHA512Type_digest_impl(self); -} - -PyDoc_STRVAR(SHA512Type_hexdigest__doc__, -"hexdigest($self, /)\n" -"--\n" -"\n" -"Return the digest value as a string of hexadecimal digits."); - -#define SHA512TYPE_HEXDIGEST_METHODDEF \ - {"hexdigest", (PyCFunction)SHA512Type_hexdigest, METH_NOARGS, SHA512Type_hexdigest__doc__}, - -static PyObject * -SHA512Type_hexdigest_impl(SHAobject *self); - -static PyObject * -SHA512Type_hexdigest(SHAobject *self, PyObject *Py_UNUSED(ignored)) -{ - return SHA512Type_hexdigest_impl(self); -} - -PyDoc_STRVAR(SHA512Type_update__doc__, -"update($self, obj, /)\n" -"--\n" -"\n" -"Update this hash object\'s state with the provided string."); - -#define SHA512TYPE_UPDATE_METHODDEF \ - {"update", (PyCFunction)SHA512Type_update, METH_O, SHA512Type_update__doc__}, - -PyDoc_STRVAR(_sha512_sha512__doc__, -"sha512($module, /, string=b\'\', *, usedforsecurity=True)\n" -"--\n" -"\n" -"Return a new SHA-512 hash object; optionally initialized with a string."); - -#define _SHA512_SHA512_METHODDEF \ - {"sha512", _PyCFunction_CAST(_sha512_sha512), METH_FASTCALL|METH_KEYWORDS, _sha512_sha512__doc__}, - -static PyObject * -_sha512_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity); - -static PyObject * -_sha512_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "sha512", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; - int usedforsecurity = 1; - - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); - if (!args) { - goto exit; - } - if (!noptargs) { - goto skip_optional_pos; - } - if (args[0]) { - string = args[0]; - if (!--noptargs) { - goto skip_optional_pos; - } - } -skip_optional_pos: - if (!noptargs) { - goto skip_optional_kwonly; - } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; - } -skip_optional_kwonly: - return_value = _sha512_sha512_impl(module, string, usedforsecurity); - -exit: - return return_value; -} - -PyDoc_STRVAR(_sha512_sha384__doc__, -"sha384($module, /, string=b\'\', *, usedforsecurity=True)\n" -"--\n" -"\n" -"Return a new SHA-384 hash object; optionally initialized with a string."); - -#define _SHA512_SHA384_METHODDEF \ - {"sha384", _PyCFunction_CAST(_sha512_sha384), METH_FASTCALL|METH_KEYWORDS, _sha512_sha384__doc__}, - -static PyObject * -_sha512_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity); - -static PyObject * -_sha512_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "sha384", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; - int usedforsecurity = 1; - - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); - if (!args) { - goto exit; - } - if (!noptargs) { - goto skip_optional_pos; - } - if (args[0]) { - string = args[0]; - if (!--noptargs) { - goto skip_optional_pos; - } - } -skip_optional_pos: - if (!noptargs) { - goto skip_optional_kwonly; - } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; - } -skip_optional_kwonly: - return_value = _sha512_sha384_impl(module, string, usedforsecurity); - -exit: - return return_value; -} -/*[clinic end generated code: output=dd168f3f21097afe input=a9049054013a1b77]*/ diff --git a/Modules/sha256module.c b/Modules/sha256module.c deleted file mode 100644 index 301c9837bb6720..00000000000000 --- a/Modules/sha256module.c +++ /dev/null @@ -1,465 +0,0 @@ -/* SHA256 module */ - -/* This module provides an interface to NIST's SHA-256 and SHA-224 Algorithms */ - -/* See below for information about the original code this module was - based upon. Additional work performed by: - - Andrew Kuchling (amk@amk.ca) - Greg Stein (gstein@lyra.org) - Trevor Perrin (trevp@trevp.net) - Jonathan Protzenko (jonathan@protzenko.fr) - - Copyright (C) 2005-2007 Gregory P. Smith (greg@krypto.org) - Licensed to PSF under a Contributor Agreement. - -*/ - -/* SHA objects */ -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 -#endif - -#include "Python.h" -#include "pycore_bitutils.h" // _Py_bswap32() -#include "pycore_strhex.h" // _Py_strhex() -#include "structmember.h" // PyMemberDef -#include "hashlib.h" - -/*[clinic input] -module _sha256 -class SHA256Type "SHAobject *" "&PyType_Type" -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=71a39174d4f0a744]*/ - - -/* The SHA block size and maximum message digest sizes, in bytes */ - -#define SHA_BLOCKSIZE 64 -#define SHA_DIGESTSIZE 32 - -/* The SHA2-224 and SHA2-256 implementations defer to the HACL* verified - * library. */ - -#include "_hacl/Hacl_Streaming_SHA2.h" - -typedef struct { - PyObject_HEAD - // Even though one could conceivably perform run-type checks to tell apart a - // sha224_type from a sha256_type (and thus deduce the digest size), we must - // keep this field because it's exposed as a member field on the underlying - // python object. - // TODO: could we transform this into a getter and get rid of the redundant - // field? - int digestsize; - Hacl_Streaming_SHA2_state_sha2_256 *state; -} SHAobject; - -#include "clinic/sha256module.c.h" - -/* We shall use run-time type information in the remainder of this module to - * tell apart SHA2-224 and SHA2-256 */ -typedef struct { - PyTypeObject* sha224_type; - PyTypeObject* sha256_type; -} _sha256_state; - -static inline _sha256_state* -_sha256_get_state(PyObject *module) -{ - void *state = PyModule_GetState(module); - assert(state != NULL); - return (_sha256_state *)state; -} - -static void SHAcopy(SHAobject *src, SHAobject *dest) -{ - dest->digestsize = src->digestsize; - dest->state = Hacl_Streaming_SHA2_copy_256(src->state); -} - -static SHAobject * -newSHA224object(_sha256_state *state) -{ - SHAobject *sha = (SHAobject *)PyObject_GC_New(SHAobject, - state->sha224_type); - PyObject_GC_Track(sha); - return sha; -} - -static SHAobject * -newSHA256object(_sha256_state *state) -{ - SHAobject *sha = (SHAobject *)PyObject_GC_New(SHAobject, - state->sha256_type); - PyObject_GC_Track(sha); - return sha; -} - -/* Internal methods for a hash object */ -static int -SHA_traverse(PyObject *ptr, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(ptr)); - return 0; -} - -static void -SHA_dealloc(SHAobject *ptr) -{ - Hacl_Streaming_SHA2_free_256(ptr->state); - PyTypeObject *tp = Py_TYPE(ptr); - PyObject_GC_UnTrack(ptr); - PyObject_GC_Del(ptr); - Py_DECREF(tp); -} - -/* HACL* takes a uint32_t for the length of its parameter, but Py_ssize_t can be - * 64 bits. */ -static void update_256(Hacl_Streaming_SHA2_state_sha2_256 *state, uint8_t *buf, Py_ssize_t len) { - /* Note: we explicitly ignore the error code on the basis that it would take > - * 1 billion years to overflow the maximum admissible length for SHA2-256 - * (namely, 2^61-1 bytes). */ - while (len > UINT32_MAX) { - Hacl_Streaming_SHA2_update_256(state, buf, UINT32_MAX); - len -= UINT32_MAX; - buf += UINT32_MAX; - } - /* Cast to uint32_t is safe: upon exiting the loop, len <= UINT32_MAX, and - * therefore fits in a uint32_t */ - Hacl_Streaming_SHA2_update_256(state, buf, (uint32_t) len); -} - - -/* External methods for a hash object */ - -/*[clinic input] -SHA256Type.copy - - cls:defining_class - -Return a copy of the hash object. -[clinic start generated code]*/ - -static PyObject * -SHA256Type_copy_impl(SHAobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=9273f92c382be12f input=3137146fcb88e212]*/ -{ - SHAobject *newobj; - _sha256_state *state = PyType_GetModuleState(cls); - if (Py_IS_TYPE(self, state->sha256_type)) { - if ( (newobj = newSHA256object(state)) == NULL) { - return NULL; - } - } else { - if ( (newobj = newSHA224object(state))==NULL) { - return NULL; - } - } - - SHAcopy(self, newobj); - return (PyObject *)newobj; -} - -/*[clinic input] -SHA256Type.digest - -Return the digest value as a bytes object. -[clinic start generated code]*/ - -static PyObject * -SHA256Type_digest_impl(SHAobject *self) -/*[clinic end generated code: output=46616a5e909fbc3d input=f1f4cfea5cbde35c]*/ -{ - uint8_t digest[SHA_DIGESTSIZE]; - // HACL performs copies under the hood so that self->state remains valid - // after this call. - Hacl_Streaming_SHA2_finish_256(self->state, digest); - return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); -} - -/*[clinic input] -SHA256Type.hexdigest - -Return the digest value as a string of hexadecimal digits. -[clinic start generated code]*/ - -static PyObject * -SHA256Type_hexdigest_impl(SHAobject *self) -/*[clinic end generated code: output=725f8a7041ae97f3 input=0cc4c714693010d1]*/ -{ - uint8_t digest[SHA_DIGESTSIZE]; - Hacl_Streaming_SHA2_finish_256(self->state, digest); - return _Py_strhex((const char *)digest, self->digestsize); -} - -/*[clinic input] -SHA256Type.update - - obj: object - / - -Update this hash object's state with the provided string. -[clinic start generated code]*/ - -static PyObject * -SHA256Type_update(SHAobject *self, PyObject *obj) -/*[clinic end generated code: output=0967fb2860c66af7 input=b2d449d5b30f0f5a]*/ -{ - Py_buffer buf; - - GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - - update_256(self->state, buf.buf, buf.len); - - PyBuffer_Release(&buf); - Py_RETURN_NONE; -} - -static PyMethodDef SHA_methods[] = { - SHA256TYPE_COPY_METHODDEF - SHA256TYPE_DIGEST_METHODDEF - SHA256TYPE_HEXDIGEST_METHODDEF - SHA256TYPE_UPDATE_METHODDEF - {NULL, NULL} /* sentinel */ -}; - -static PyObject * -SHA256_get_block_size(PyObject *self, void *closure) -{ - return PyLong_FromLong(SHA_BLOCKSIZE); -} - -static PyObject * -SHA256_get_name(SHAobject *self, void *closure) -{ - if (self->digestsize == 28) { - return PyUnicode_FromStringAndSize("sha224", 6); - } - return PyUnicode_FromStringAndSize("sha256", 6); -} - -static PyGetSetDef SHA_getseters[] = { - {"block_size", - (getter)SHA256_get_block_size, NULL, - NULL, - NULL}, - {"name", - (getter)SHA256_get_name, NULL, - NULL, - NULL}, - {NULL} /* Sentinel */ -}; - -static PyMemberDef SHA_members[] = { - {"digest_size", T_INT, offsetof(SHAobject, digestsize), READONLY, NULL}, - {NULL} /* Sentinel */ -}; - -static PyType_Slot sha256_types_slots[] = { - {Py_tp_dealloc, SHA_dealloc}, - {Py_tp_methods, SHA_methods}, - {Py_tp_members, SHA_members}, - {Py_tp_getset, SHA_getseters}, - {Py_tp_traverse, SHA_traverse}, - {0,0} -}; - -static PyType_Spec sha224_type_spec = { - .name = "_sha256.sha224", - .basicsize = sizeof(SHAobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | - Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), - .slots = sha256_types_slots -}; - -static PyType_Spec sha256_type_spec = { - .name = "_sha256.sha256", - .basicsize = sizeof(SHAobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | - Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), - .slots = sha256_types_slots -}; - -/* The single module-level function: new() */ - -/*[clinic input] -_sha256.sha256 - - string: object(c_default="NULL") = b'' - * - usedforsecurity: bool = True - -Return a new SHA-256 hash object; optionally initialized with a string. -[clinic start generated code]*/ - -static PyObject * -_sha256_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=a1de327e8e1185cf input=9be86301aeb14ea5]*/ -{ - Py_buffer buf; - - if (string) { - GET_BUFFER_VIEW_OR_ERROUT(string, &buf); - } - - _sha256_state *state = PyModule_GetState(module); - - SHAobject *new; - if ((new = newSHA256object(state)) == NULL) { - if (string) { - PyBuffer_Release(&buf); - } - return NULL; - } - - new->state = Hacl_Streaming_SHA2_create_in_256(); - new->digestsize = 32; - - if (PyErr_Occurred()) { - Py_DECREF(new); - if (string) { - PyBuffer_Release(&buf); - } - return NULL; - } - if (string) { - update_256(new->state, buf.buf, buf.len); - PyBuffer_Release(&buf); - } - - return (PyObject *)new; -} - -/*[clinic input] -_sha256.sha224 - - string: object(c_default="NULL") = b'' - * - usedforsecurity: bool = True - -Return a new SHA-224 hash object; optionally initialized with a string. -[clinic start generated code]*/ - -static PyObject * -_sha256_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=08be6b36569bc69c input=9fcfb46e460860ac]*/ -{ - Py_buffer buf; - if (string) { - GET_BUFFER_VIEW_OR_ERROUT(string, &buf); - } - - _sha256_state *state = PyModule_GetState(module); - SHAobject *new; - if ((new = newSHA224object(state)) == NULL) { - if (string) { - PyBuffer_Release(&buf); - } - return NULL; - } - - new->state = Hacl_Streaming_SHA2_create_in_224(); - new->digestsize = 28; - - if (PyErr_Occurred()) { - Py_DECREF(new); - if (string) { - PyBuffer_Release(&buf); - } - return NULL; - } - if (string) { - update_256(new->state, buf.buf, buf.len); - PyBuffer_Release(&buf); - } - - return (PyObject *)new; -} - - -/* List of functions exported by this module */ - -static struct PyMethodDef SHA_functions[] = { - _SHA256_SHA256_METHODDEF - _SHA256_SHA224_METHODDEF - {NULL, NULL} /* Sentinel */ -}; - -static int -_sha256_traverse(PyObject *module, visitproc visit, void *arg) -{ - _sha256_state *state = _sha256_get_state(module); - Py_VISIT(state->sha224_type); - Py_VISIT(state->sha256_type); - return 0; -} - -static int -_sha256_clear(PyObject *module) -{ - _sha256_state *state = _sha256_get_state(module); - Py_CLEAR(state->sha224_type); - Py_CLEAR(state->sha256_type); - return 0; -} - -static void -_sha256_free(void *module) -{ - _sha256_clear((PyObject *)module); -} - -static int sha256_exec(PyObject *module) -{ - _sha256_state *state = _sha256_get_state(module); - - state->sha224_type = (PyTypeObject *)PyType_FromModuleAndSpec( - module, &sha224_type_spec, NULL); - - if (state->sha224_type == NULL) { - return -1; - } - - state->sha256_type = (PyTypeObject *)PyType_FromModuleAndSpec( - module, &sha256_type_spec, NULL); - - if (state->sha256_type == NULL) { - return -1; - } - - Py_INCREF((PyObject *)state->sha224_type); - if (PyModule_AddObject(module, "SHA224Type", (PyObject *)state->sha224_type) < 0) { - Py_DECREF((PyObject *)state->sha224_type); - return -1; - } - Py_INCREF((PyObject *)state->sha256_type); - if (PyModule_AddObject(module, "SHA256Type", (PyObject *)state->sha256_type) < 0) { - Py_DECREF((PyObject *)state->sha256_type); - return -1; - } - return 0; -} - -static PyModuleDef_Slot _sha256_slots[] = { - {Py_mod_exec, sha256_exec}, - {0, NULL} -}; - -static struct PyModuleDef _sha256module = { - PyModuleDef_HEAD_INIT, - .m_name = "_sha256", - .m_size = sizeof(_sha256_state), - .m_methods = SHA_functions, - .m_slots = _sha256_slots, - .m_traverse = _sha256_traverse, - .m_clear = _sha256_clear, - .m_free = _sha256_free -}; - -/* Initialize this module. */ -PyMODINIT_FUNC -PyInit__sha256(void) -{ - return PyModuleDef_Init(&_sha256module); -} diff --git a/Modules/sha2module.c b/Modules/sha2module.c new file mode 100644 index 00000000000000..9999f255cd578a --- /dev/null +++ b/Modules/sha2module.c @@ -0,0 +1,805 @@ +/* SHA2 module */ + +/* This provides an interface to NIST's SHA2 224, 256, 384, & 512 Algorithms */ + +/* See below for information about the original code this module was + based upon. Additional work performed by: + + Andrew Kuchling (amk@amk.ca) + Greg Stein (gstein@lyra.org) + Trevor Perrin (trevp@trevp.net) + Jonathan Protzenko (jonathan@protzenko.fr) + + Copyright (C) 2005-2007 Gregory P. Smith (greg@krypto.org) + Licensed to PSF under a Contributor Agreement. + +*/ + +/* SHA objects */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +#include "Python.h" +#include "pycore_bitutils.h" // _Py_bswap32() +#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_strhex.h" // _Py_strhex() +#include "structmember.h" // PyMemberDef +#include "hashlib.h" + +/*[clinic input] +module _sha2 +class SHA256Type "SHA256object *" "&PyType_Type" +class SHA512Type "SHA512object *" "&PyType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b5315a7b611c9afc]*/ + + +/* The SHA block sizes and maximum message digest sizes, in bytes */ + +#define SHA256_BLOCKSIZE 64 +#define SHA256_DIGESTSIZE 32 +#define SHA512_BLOCKSIZE 128 +#define SHA512_DIGESTSIZE 64 + +/* Our SHA2 implementations defer to the HACL* verified library. */ + +#include "_hacl/Hacl_Streaming_SHA2.h" + +// TODO: Get rid of int digestsize in favor of Hacl state info? + +typedef struct { + PyObject_HEAD + int digestsize; + Hacl_Streaming_SHA2_state_sha2_256 *state; +} SHA256object; + +typedef struct { + PyObject_HEAD + int digestsize; + Hacl_Streaming_SHA2_state_sha2_512 *state; +} SHA512object; + +#include "clinic/sha2module.c.h" + +/* We shall use run-time type information in the remainder of this module to + * tell apart SHA2-224 and SHA2-256 */ +typedef struct { + PyTypeObject* sha224_type; + PyTypeObject* sha256_type; + PyTypeObject* sha384_type; + PyTypeObject* sha512_type; +} sha2_state; + +static inline sha2_state* +sha2_get_state(PyObject *module) +{ + void *state = _PyModule_GetState(module); + assert(state != NULL); + return (sha2_state *)state; +} + +static void SHA256copy(SHA256object *src, SHA256object *dest) +{ + dest->digestsize = src->digestsize; + dest->state = Hacl_Streaming_SHA2_copy_256(src->state); +} + +static void SHA512copy(SHA512object *src, SHA512object *dest) +{ + dest->digestsize = src->digestsize; + dest->state = Hacl_Streaming_SHA2_copy_512(src->state); +} + +static SHA256object * +newSHA224object(sha2_state *state) +{ + SHA256object *sha = (SHA256object *)PyObject_GC_New( + SHA256object, state->sha224_type); + if (!sha) { + return NULL; + } + PyObject_GC_Track(sha); + return sha; +} + +static SHA256object * +newSHA256object(sha2_state *state) +{ + SHA256object *sha = (SHA256object *)PyObject_GC_New( + SHA256object, state->sha256_type); + if (!sha) { + return NULL; + } + PyObject_GC_Track(sha); + return sha; +} + +static SHA512object * +newSHA384object(sha2_state *state) +{ + SHA512object *sha = (SHA512object *)PyObject_GC_New( + SHA512object, state->sha384_type); + if (!sha) { + return NULL; + } + PyObject_GC_Track(sha); + return sha; +} + +static SHA512object * +newSHA512object(sha2_state *state) +{ + SHA512object *sha = (SHA512object *)PyObject_GC_New( + SHA512object, state->sha512_type); + if (!sha) { + return NULL; + } + PyObject_GC_Track(sha); + return sha; +} + +/* Internal methods for our hash objects. */ + +static int +SHA2_traverse(PyObject *ptr, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(ptr)); + return 0; +} + +static void +SHA256_dealloc(SHA256object *ptr) +{ + Hacl_Streaming_SHA2_free_256(ptr->state); + PyTypeObject *tp = Py_TYPE(ptr); + PyObject_GC_UnTrack(ptr); + PyObject_GC_Del(ptr); + Py_DECREF(tp); +} + +static void +SHA512_dealloc(SHA512object *ptr) +{ + Hacl_Streaming_SHA2_free_512(ptr->state); + PyTypeObject *tp = Py_TYPE(ptr); + PyObject_GC_UnTrack(ptr); + PyObject_GC_Del(ptr); + Py_DECREF(tp); +} + +/* HACL* takes a uint32_t for the length of its parameter, but Py_ssize_t can be + * 64 bits so we loop in <4gig chunks when needed. */ + +static void update_256(Hacl_Streaming_SHA2_state_sha2_256 *state, uint8_t *buf, Py_ssize_t len) { + /* Note: we explicitly ignore the error code on the basis that it would take > + * 1 billion years to overflow the maximum admissible length for SHA2-256 + * (namely, 2^61-1 bytes). */ +#if PY_SSIZE_T_MAX > UINT32_MAX + while (len > UINT32_MAX) { + Hacl_Streaming_SHA2_update_256(state, buf, UINT32_MAX); + len -= UINT32_MAX; + buf += UINT32_MAX; + } +#endif + /* Cast to uint32_t is safe: len <= UINT32_MAX at this point. */ + Hacl_Streaming_SHA2_update_256(state, buf, (uint32_t) len); +} + +static void update_512(Hacl_Streaming_SHA2_state_sha2_512 *state, uint8_t *buf, Py_ssize_t len) { + /* Note: we explicitly ignore the error code on the basis that it would take > + * 1 billion years to overflow the maximum admissible length for this API + * (namely, 2^64-1 bytes). */ +#if PY_SSIZE_T_MAX > UINT32_MAX + while (len > UINT32_MAX) { + Hacl_Streaming_SHA2_update_512(state, buf, UINT32_MAX); + len -= UINT32_MAX; + buf += UINT32_MAX; + } +#endif + /* Cast to uint32_t is safe: len <= UINT32_MAX at this point. */ + Hacl_Streaming_SHA2_update_512(state, buf, (uint32_t) len); +} + + +/* External methods for our hash objects */ + +/*[clinic input] +SHA256Type.copy + + cls:defining_class + +Return a copy of the hash object. +[clinic start generated code]*/ + +static PyObject * +SHA256Type_copy_impl(SHA256object *self, PyTypeObject *cls) +/*[clinic end generated code: output=fabd515577805cd3 input=3137146fcb88e212]*/ +{ + SHA256object *newobj; + sha2_state *state = PyType_GetModuleState(cls); + if (Py_IS_TYPE(self, state->sha256_type)) { + if ((newobj = newSHA256object(state)) == NULL) { + return NULL; + } + } else { + if ((newobj = newSHA224object(state)) == NULL) { + return NULL; + } + } + + SHA256copy(self, newobj); + return (PyObject *)newobj; +} + +/*[clinic input] +SHA512Type.copy + + cls: defining_class + +Return a copy of the hash object. +[clinic start generated code]*/ + +static PyObject * +SHA512Type_copy_impl(SHA512object *self, PyTypeObject *cls) +/*[clinic end generated code: output=66d2a8ef20de8302 input=f673a18f66527c90]*/ +{ + SHA512object *newobj; + sha2_state *state = PyType_GetModuleState(cls); + + if (Py_IS_TYPE((PyObject*)self, state->sha512_type)) { + if ((newobj = newSHA512object(state)) == NULL) { + return NULL; + } + } + else { + if ((newobj = newSHA384object(state)) == NULL) { + return NULL; + } + } + + SHA512copy(self, newobj); + return (PyObject *)newobj; +} + +/*[clinic input] +SHA256Type.digest + +Return the digest value as a bytes object. +[clinic start generated code]*/ + +static PyObject * +SHA256Type_digest_impl(SHA256object *self) +/*[clinic end generated code: output=3a2e3997a98ee792 input=f1f4cfea5cbde35c]*/ +{ + uint8_t digest[SHA256_DIGESTSIZE]; + assert(self->digestsize <= SHA256_DIGESTSIZE); + // HACL* performs copies under the hood so that self->state remains valid + // after this call. + Hacl_Streaming_SHA2_finish_256(self->state, digest); + return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); +} + +/*[clinic input] +SHA512Type.digest + +Return the digest value as a bytes object. +[clinic start generated code]*/ + +static PyObject * +SHA512Type_digest_impl(SHA512object *self) +/*[clinic end generated code: output=dd8c6320070458e0 input=f6470dd359071f4b]*/ +{ + uint8_t digest[SHA512_DIGESTSIZE]; + assert(self->digestsize <= SHA512_DIGESTSIZE); + // HACL* performs copies under the hood so that self->state remains valid + // after this call. + Hacl_Streaming_SHA2_finish_512(self->state, digest); + return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); +} + +/*[clinic input] +SHA256Type.hexdigest + +Return the digest value as a string of hexadecimal digits. +[clinic start generated code]*/ + +static PyObject * +SHA256Type_hexdigest_impl(SHA256object *self) +/*[clinic end generated code: output=96cb68996a780ab3 input=0cc4c714693010d1]*/ +{ + uint8_t digest[SHA256_DIGESTSIZE]; + assert(self->digestsize <= SHA256_DIGESTSIZE); + Hacl_Streaming_SHA2_finish_256(self->state, digest); + return _Py_strhex((const char *)digest, self->digestsize); +} + +/*[clinic input] +SHA512Type.hexdigest + +Return the digest value as a string of hexadecimal digits. +[clinic start generated code]*/ + +static PyObject * +SHA512Type_hexdigest_impl(SHA512object *self) +/*[clinic end generated code: output=cbd6f844aba1fe7c input=498b877b25cbe0a2]*/ +{ + uint8_t digest[SHA512_DIGESTSIZE]; + assert(self->digestsize <= SHA512_DIGESTSIZE); + Hacl_Streaming_SHA2_finish_512(self->state, digest); + return _Py_strhex((const char *)digest, self->digestsize); +} + +/*[clinic input] +SHA256Type.update + + obj: object + / + +Update this hash object's state with the provided string. +[clinic start generated code]*/ + +static PyObject * +SHA256Type_update(SHA256object *self, PyObject *obj) +/*[clinic end generated code: output=1b240f965ddbd8c6 input=b2d449d5b30f0f5a]*/ +{ + Py_buffer buf; + + GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); + + update_256(self->state, buf.buf, buf.len); + + PyBuffer_Release(&buf); + Py_RETURN_NONE; +} + +/*[clinic input] +SHA512Type.update + + obj: object + / + +Update this hash object's state with the provided string. +[clinic start generated code]*/ + +static PyObject * +SHA512Type_update(SHA512object *self, PyObject *obj) +/*[clinic end generated code: output=745f51057a985884 input=ded2b46656566283]*/ +{ + Py_buffer buf; + + GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); + + update_512(self->state, buf.buf, buf.len); + + PyBuffer_Release(&buf); + Py_RETURN_NONE; +} + +static PyMethodDef SHA256_methods[] = { + SHA256TYPE_COPY_METHODDEF + SHA256TYPE_DIGEST_METHODDEF + SHA256TYPE_HEXDIGEST_METHODDEF + SHA256TYPE_UPDATE_METHODDEF + {NULL, NULL} /* sentinel */ +}; + +static PyMethodDef SHA512_methods[] = { + SHA512TYPE_COPY_METHODDEF + SHA512TYPE_DIGEST_METHODDEF + SHA512TYPE_HEXDIGEST_METHODDEF + SHA512TYPE_UPDATE_METHODDEF + {NULL, NULL} /* sentinel */ +}; + +static PyObject * +SHA256_get_block_size(PyObject *self, void *closure) +{ + return PyLong_FromLong(SHA256_BLOCKSIZE); +} + +static PyObject * +SHA512_get_block_size(PyObject *self, void *closure) +{ + return PyLong_FromLong(SHA512_BLOCKSIZE); +} + +static PyObject * +SHA256_get_digest_size(SHA256object *self, void *closure) +{ + return PyLong_FromLong(self->digestsize); +} + +static PyObject * +SHA512_get_digest_size(SHA512object *self, void *closure) +{ + return PyLong_FromLong(self->digestsize); +} + +static PyObject * +SHA256_get_name(SHA256object *self, void *closure) +{ + if (self->digestsize == 28) { + return PyUnicode_FromStringAndSize("sha224", 6); + } + return PyUnicode_FromStringAndSize("sha256", 6); +} + +static PyObject * +SHA512_get_name(SHA512object *self, void *closure) +{ + if (self->digestsize == 64) { + return PyUnicode_FromStringAndSize("sha512", 6); + } + return PyUnicode_FromStringAndSize("sha384", 6); +} + +static PyGetSetDef SHA256_getseters[] = { + {"block_size", + (getter)SHA256_get_block_size, NULL, + NULL, + NULL}, + {"name", + (getter)SHA256_get_name, NULL, + NULL, + NULL}, + {"digest_size", + (getter)SHA256_get_digest_size, NULL, + NULL, + NULL}, + {NULL} /* Sentinel */ +}; + +static PyGetSetDef SHA512_getseters[] = { + {"block_size", + (getter)SHA512_get_block_size, NULL, + NULL, + NULL}, + {"name", + (getter)SHA512_get_name, NULL, + NULL, + NULL}, + {"digest_size", + (getter)SHA512_get_digest_size, NULL, + NULL, + NULL}, + {NULL} /* Sentinel */ +}; + +static PyType_Slot sha256_types_slots[] = { + {Py_tp_dealloc, SHA256_dealloc}, + {Py_tp_methods, SHA256_methods}, + {Py_tp_getset, SHA256_getseters}, + {Py_tp_traverse, SHA2_traverse}, + {0,0} +}; + +static PyType_Slot sha512_type_slots[] = { + {Py_tp_dealloc, SHA512_dealloc}, + {Py_tp_methods, SHA512_methods}, + {Py_tp_getset, SHA512_getseters}, + {Py_tp_traverse, SHA2_traverse}, + {0,0} +}; + +// Using PyType_GetModuleState() on these types is safe since they +// cannot be subclassed: they don't have the Py_TPFLAGS_BASETYPE flag. +static PyType_Spec sha224_type_spec = { + .name = "_sha2.SHA224Type", + .basicsize = sizeof(SHA256object), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), + .slots = sha256_types_slots +}; + +static PyType_Spec sha256_type_spec = { + .name = "_sha2.SHA256Type", + .basicsize = sizeof(SHA256object), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), + .slots = sha256_types_slots +}; + +static PyType_Spec sha384_type_spec = { + .name = "_sha2.SHA384Type", + .basicsize = sizeof(SHA512object), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), + .slots = sha512_type_slots +}; + +static PyType_Spec sha512_type_spec = { + .name = "_sha2.SHA512Type", + .basicsize = sizeof(SHA512object), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), + .slots = sha512_type_slots +}; + +/* The module-level constructors. */ + +/*[clinic input] +_sha2.sha256 + + string: object(c_default="NULL") = b'' + * + usedforsecurity: bool = True + +Return a new SHA-256 hash object; optionally initialized with a string. +[clinic start generated code]*/ + +static PyObject * +_sha2_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity) +/*[clinic end generated code: output=243c9dd289931f87 input=6249da1de607280a]*/ +{ + Py_buffer buf; + + if (string) { + GET_BUFFER_VIEW_OR_ERROUT(string, &buf); + } + + sha2_state *state = sha2_get_state(module); + + SHA256object *new; + if ((new = newSHA256object(state)) == NULL) { + if (string) { + PyBuffer_Release(&buf); + } + return NULL; + } + + new->state = Hacl_Streaming_SHA2_create_in_256(); + new->digestsize = 32; + + if (PyErr_Occurred()) { + Py_DECREF(new); + if (string) { + PyBuffer_Release(&buf); + } + return NULL; + } + if (string) { + update_256(new->state, buf.buf, buf.len); + PyBuffer_Release(&buf); + } + + return (PyObject *)new; +} + +/*[clinic input] +_sha2.sha224 + + string: object(c_default="NULL") = b'' + * + usedforsecurity: bool = True + +Return a new SHA-224 hash object; optionally initialized with a string. +[clinic start generated code]*/ + +static PyObject * +_sha2_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity) +/*[clinic end generated code: output=68191f232e4a3843 input=c42bcba47fd7d2b7]*/ +{ + Py_buffer buf; + if (string) { + GET_BUFFER_VIEW_OR_ERROUT(string, &buf); + } + + sha2_state *state = sha2_get_state(module); + SHA256object *new; + if ((new = newSHA224object(state)) == NULL) { + if (string) { + PyBuffer_Release(&buf); + } + return NULL; + } + + new->state = Hacl_Streaming_SHA2_create_in_224(); + new->digestsize = 28; + + if (PyErr_Occurred()) { + Py_DECREF(new); + if (string) { + PyBuffer_Release(&buf); + } + return NULL; + } + if (string) { + update_256(new->state, buf.buf, buf.len); + PyBuffer_Release(&buf); + } + + return (PyObject *)new; +} + +/*[clinic input] +_sha2.sha512 + + string: object(c_default="NULL") = b'' + * + usedforsecurity: bool = True + +Return a new SHA-512 hash object; optionally initialized with a string. +[clinic start generated code]*/ + +static PyObject * +_sha2_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) +/*[clinic end generated code: output=d55c8996eca214d7 input=0576ae2a6ebfad25]*/ +{ + SHA512object *new; + Py_buffer buf; + + sha2_state *state = sha2_get_state(module); + + if (string) + GET_BUFFER_VIEW_OR_ERROUT(string, &buf); + + if ((new = newSHA512object(state)) == NULL) { + if (string) + PyBuffer_Release(&buf); + return NULL; + } + + new->state = Hacl_Streaming_SHA2_create_in_512(); + new->digestsize = 64; + + if (PyErr_Occurred()) { + Py_DECREF(new); + if (string) + PyBuffer_Release(&buf); + return NULL; + } + if (string) { + update_512(new->state, buf.buf, buf.len); + PyBuffer_Release(&buf); + } + + return (PyObject *)new; +} + +/*[clinic input] +_sha2.sha384 + + string: object(c_default="NULL") = b'' + * + usedforsecurity: bool = True + +Return a new SHA-384 hash object; optionally initialized with a string. +[clinic start generated code]*/ + +static PyObject * +_sha2_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity) +/*[clinic end generated code: output=b29a0d81d51d1368 input=4e9199d8de0d2f9b]*/ +{ + SHA512object *new; + Py_buffer buf; + + sha2_state *state = sha2_get_state(module); + + if (string) + GET_BUFFER_VIEW_OR_ERROUT(string, &buf); + + if ((new = newSHA384object(state)) == NULL) { + if (string) + PyBuffer_Release(&buf); + return NULL; + } + + new->state = Hacl_Streaming_SHA2_create_in_384(); + new->digestsize = 48; + + if (PyErr_Occurred()) { + Py_DECREF(new); + if (string) + PyBuffer_Release(&buf); + return NULL; + } + if (string) { + update_512(new->state, buf.buf, buf.len); + PyBuffer_Release(&buf); + } + + return (PyObject *)new; +} + +/* List of functions exported by this module */ + +static struct PyMethodDef SHA2_functions[] = { + _SHA2_SHA256_METHODDEF + _SHA2_SHA224_METHODDEF + _SHA2_SHA512_METHODDEF + _SHA2_SHA384_METHODDEF + {NULL, NULL} /* Sentinel */ +}; + +static int +_sha2_traverse(PyObject *module, visitproc visit, void *arg) +{ + sha2_state *state = sha2_get_state(module); + Py_VISIT(state->sha224_type); + Py_VISIT(state->sha256_type); + Py_VISIT(state->sha384_type); + Py_VISIT(state->sha512_type); + return 0; +} + +static int +_sha2_clear(PyObject *module) +{ + sha2_state *state = sha2_get_state(module); + Py_CLEAR(state->sha224_type); + Py_CLEAR(state->sha256_type); + Py_CLEAR(state->sha384_type); + Py_CLEAR(state->sha512_type); + return 0; +} + +static void +_sha2_free(void *module) +{ + _sha2_clear((PyObject *)module); +} + +/* Initialize this module. */ +static int sha2_exec(PyObject *module) +{ + sha2_state *state = sha2_get_state(module); + + state->sha224_type = (PyTypeObject *)PyType_FromModuleAndSpec( + module, &sha224_type_spec, NULL); + if (state->sha224_type == NULL) { + return -1; + } + state->sha256_type = (PyTypeObject *)PyType_FromModuleAndSpec( + module, &sha256_type_spec, NULL); + if (state->sha256_type == NULL) { + return -1; + } + state->sha384_type = (PyTypeObject *)PyType_FromModuleAndSpec( + module, &sha384_type_spec, NULL); + if (state->sha384_type == NULL) { + return -1; + } + state->sha512_type = (PyTypeObject *)PyType_FromModuleAndSpec( + module, &sha512_type_spec, NULL); + if (state->sha512_type == NULL) { + return -1; + } + + if (PyModule_AddType(module, state->sha224_type) < 0) { + return -1; + } + if (PyModule_AddType(module, state->sha256_type) < 0) { + return -1; + } + if (PyModule_AddType(module, state->sha384_type) < 0) { + return -1; + } + if (PyModule_AddType(module, state->sha512_type) < 0) { + return -1; + } + + return 0; +} + +static PyModuleDef_Slot _sha2_slots[] = { + {Py_mod_exec, sha2_exec}, + {0, NULL} +}; + +static struct PyModuleDef _sha2module = { + PyModuleDef_HEAD_INIT, + .m_name = "_sha2", + .m_size = sizeof(sha2_state), + .m_methods = SHA2_functions, + .m_slots = _sha2_slots, + .m_traverse = _sha2_traverse, + .m_clear = _sha2_clear, + .m_free = _sha2_free +}; + +PyMODINIT_FUNC +PyInit__sha2(void) +{ + return PyModuleDef_Init(&_sha2module); +} diff --git a/Modules/sha512module.c b/Modules/sha512module.c deleted file mode 100644 index d7dfed4e5db03a..00000000000000 --- a/Modules/sha512module.c +++ /dev/null @@ -1,456 +0,0 @@ -/* SHA512 module */ - -/* This module provides an interface to NIST's SHA-512 and SHA-384 Algorithms */ - -/* See below for information about the original code this module was - based upon. Additional work performed by: - - Andrew Kuchling (amk@amk.ca) - Greg Stein (gstein@lyra.org) - Trevor Perrin (trevp@trevp.net) - Jonathan Protzenko (jonathan@protzenko.fr) - - Copyright (C) 2005-2007 Gregory P. Smith (greg@krypto.org) - Licensed to PSF under a Contributor Agreement. - -*/ - -/* SHA objects */ -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 -#endif - -#include "Python.h" -#include "pycore_bitutils.h" // _Py_bswap64() -#include "pycore_strhex.h" // _Py_strhex() -#include "structmember.h" // PyMemberDef -#include "hashlib.h" - -/*[clinic input] -module _sha512 -class SHA512Type "SHAobject *" "&PyType_Type" -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=81a3ccde92bcfe8d]*/ - - -/* The SHA block size and message digest sizes, in bytes */ - -#define SHA_BLOCKSIZE 128 -#define SHA_DIGESTSIZE 64 - -/* The SHA2-384 and SHA2-512 implementations defer to the HACL* verified - * library. */ - -#include "_hacl/Hacl_Streaming_SHA2.h" - -typedef struct { - PyObject_HEAD - int digestsize; - Hacl_Streaming_SHA2_state_sha2_512 *state; -} SHAobject; - -#include "clinic/sha512module.c.h" - - -static void SHAcopy(SHAobject *src, SHAobject *dest) -{ - dest->digestsize = src->digestsize; - dest->state = Hacl_Streaming_SHA2_copy_512(src->state); -} - -typedef struct { - PyTypeObject* sha384_type; - PyTypeObject* sha512_type; -} SHA512State; - -static inline SHA512State* -sha512_get_state(PyObject *module) -{ - void *state = PyModule_GetState(module); - assert(state != NULL); - return (SHA512State *)state; -} - -static SHAobject * -newSHA384object(SHA512State *st) -{ - SHAobject *sha = (SHAobject *)PyObject_GC_New(SHAobject, st->sha384_type); - PyObject_GC_Track(sha); - return sha; -} - -static SHAobject * -newSHA512object(SHA512State *st) -{ - SHAobject *sha = (SHAobject *)PyObject_GC_New(SHAobject, st->sha512_type); - PyObject_GC_Track(sha); - return sha; -} - -/* Internal methods for a hash object */ -static int -SHA_traverse(PyObject *ptr, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(ptr)); - return 0; -} - -static void -SHA512_dealloc(SHAobject *ptr) -{ - Hacl_Streaming_SHA2_free_512(ptr->state); - PyTypeObject *tp = Py_TYPE(ptr); - PyObject_GC_UnTrack(ptr); - PyObject_GC_Del(ptr); - Py_DECREF(tp); -} - -/* HACL* takes a uint32_t for the length of its parameter, but Py_ssize_t can be - * 64 bits. */ -static void update_512(Hacl_Streaming_SHA2_state_sha2_512 *state, uint8_t *buf, Py_ssize_t len) { - /* Note: we explicitly ignore the error code on the basis that it would take > - * 1 billion years to overflow the maximum admissible length for this API - * (namely, 2^64-1 bytes). */ - while (len > UINT32_MAX) { - Hacl_Streaming_SHA2_update_512(state, buf, UINT32_MAX); - len -= UINT32_MAX; - buf += UINT32_MAX; - } - /* Cast to uint32_t is safe: upon exiting the loop, len <= UINT32_MAX, and - * therefore fits in a uint32_t */ - Hacl_Streaming_SHA2_update_512(state, buf, (uint32_t) len); -} - - -/* External methods for a hash object */ - -/*[clinic input] -SHA512Type.copy - - cls: defining_class - -Return a copy of the hash object. -[clinic start generated code]*/ - -static PyObject * -SHA512Type_copy_impl(SHAobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=85ea5b47837a08e6 input=f673a18f66527c90]*/ -{ - SHAobject *newobj; - SHA512State *st = PyType_GetModuleState(cls); - - if (Py_IS_TYPE((PyObject*)self, st->sha512_type)) { - if ( (newobj = newSHA512object(st))==NULL) { - return NULL; - } - } - else { - if ( (newobj = newSHA384object(st))==NULL) { - return NULL; - } - } - - SHAcopy(self, newobj); - return (PyObject *)newobj; -} - -/*[clinic input] -SHA512Type.digest - -Return the digest value as a bytes object. -[clinic start generated code]*/ - -static PyObject * -SHA512Type_digest_impl(SHAobject *self) -/*[clinic end generated code: output=1080bbeeef7dde1b input=f6470dd359071f4b]*/ -{ - uint8_t digest[SHA_DIGESTSIZE]; - // HACL performs copies under the hood so that self->state remains valid - // after this call. - Hacl_Streaming_SHA2_finish_512(self->state, digest); - return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); -} - -/*[clinic input] -SHA512Type.hexdigest - -Return the digest value as a string of hexadecimal digits. -[clinic start generated code]*/ - -static PyObject * -SHA512Type_hexdigest_impl(SHAobject *self) -/*[clinic end generated code: output=7373305b8601e18b input=498b877b25cbe0a2]*/ -{ - uint8_t digest[SHA_DIGESTSIZE]; - Hacl_Streaming_SHA2_finish_512(self->state, digest); - return _Py_strhex((const char *)digest, self->digestsize); -} - -/*[clinic input] -SHA512Type.update - - obj: object - / - -Update this hash object's state with the provided string. -[clinic start generated code]*/ - -static PyObject * -SHA512Type_update(SHAobject *self, PyObject *obj) -/*[clinic end generated code: output=1cf333e73995a79e input=ded2b46656566283]*/ -{ - Py_buffer buf; - - GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - - update_512(self->state, buf.buf, buf.len); - - PyBuffer_Release(&buf); - Py_RETURN_NONE; -} - -static PyMethodDef SHA_methods[] = { - SHA512TYPE_COPY_METHODDEF - SHA512TYPE_DIGEST_METHODDEF - SHA512TYPE_HEXDIGEST_METHODDEF - SHA512TYPE_UPDATE_METHODDEF - {NULL, NULL} /* sentinel */ -}; - -static PyObject * -SHA512_get_block_size(PyObject *self, void *closure) -{ - return PyLong_FromLong(SHA_BLOCKSIZE); -} - -static PyObject * -SHA512_get_name(PyObject *self, void *closure) -{ - if (((SHAobject *)self)->digestsize == 64) - return PyUnicode_FromStringAndSize("sha512", 6); - else - return PyUnicode_FromStringAndSize("sha384", 6); -} - -static PyGetSetDef SHA_getseters[] = { - {"block_size", - (getter)SHA512_get_block_size, NULL, - NULL, - NULL}, - {"name", - (getter)SHA512_get_name, NULL, - NULL, - NULL}, - {NULL} /* Sentinel */ -}; - -static PyMemberDef SHA_members[] = { - {"digest_size", T_INT, offsetof(SHAobject, digestsize), READONLY, NULL}, - {NULL} /* Sentinel */ -}; - -static PyType_Slot sha512_sha384_type_slots[] = { - {Py_tp_dealloc, SHA512_dealloc}, - {Py_tp_methods, SHA_methods}, - {Py_tp_members, SHA_members}, - {Py_tp_getset, SHA_getseters}, - {Py_tp_traverse, SHA_traverse}, - {0,0} -}; - -static PyType_Spec sha512_sha384_type_spec = { - .name = "_sha512.sha384", - .basicsize = sizeof(SHAobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | - Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), - .slots = sha512_sha384_type_slots -}; - -// Using PyType_GetModuleState() on this type is safe since -// it cannot be subclassed: it does not have the Py_TPFLAGS_BASETYPE flag. -static PyType_Spec sha512_sha512_type_spec = { - .name = "_sha512.sha512", - .basicsize = sizeof(SHAobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | - Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), - .slots = sha512_sha384_type_slots -}; - -/* The single module-level function: new() */ - -/*[clinic input] -_sha512.sha512 - - string: object(c_default="NULL") = b'' - * - usedforsecurity: bool = True - -Return a new SHA-512 hash object; optionally initialized with a string. -[clinic start generated code]*/ - -static PyObject * -_sha512_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=a8d9e5f9e6a0831c input=23b4daebc2ebb9c9]*/ -{ - SHAobject *new; - Py_buffer buf; - - SHA512State *st = sha512_get_state(module); - - if (string) - GET_BUFFER_VIEW_OR_ERROUT(string, &buf); - - if ((new = newSHA512object(st)) == NULL) { - if (string) - PyBuffer_Release(&buf); - return NULL; - } - - new->state = Hacl_Streaming_SHA2_create_in_512(); - new->digestsize = 64; - - if (PyErr_Occurred()) { - Py_DECREF(new); - if (string) - PyBuffer_Release(&buf); - return NULL; - } - if (string) { - update_512(new->state, buf.buf, buf.len); - PyBuffer_Release(&buf); - } - - return (PyObject *)new; -} - -/*[clinic input] -_sha512.sha384 - - string: object(c_default="NULL") = b'' - * - usedforsecurity: bool = True - -Return a new SHA-384 hash object; optionally initialized with a string. -[clinic start generated code]*/ - -static PyObject * -_sha512_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=da7d594a08027ac3 input=59ef72f039a6b431]*/ -{ - SHAobject *new; - Py_buffer buf; - - SHA512State *st = sha512_get_state(module); - - if (string) - GET_BUFFER_VIEW_OR_ERROUT(string, &buf); - - if ((new = newSHA384object(st)) == NULL) { - if (string) - PyBuffer_Release(&buf); - return NULL; - } - - new->state = Hacl_Streaming_SHA2_create_in_384(); - new->digestsize = 48; - - if (PyErr_Occurred()) { - Py_DECREF(new); - if (string) - PyBuffer_Release(&buf); - return NULL; - } - if (string) { - update_512(new->state, buf.buf, buf.len); - PyBuffer_Release(&buf); - } - - return (PyObject *)new; -} - - -/* List of functions exported by this module */ - -static struct PyMethodDef SHA_functions[] = { - _SHA512_SHA512_METHODDEF - _SHA512_SHA384_METHODDEF - {NULL, NULL} /* Sentinel */ -}; - -static int -_sha512_traverse(PyObject *module, visitproc visit, void *arg) -{ - SHA512State *state = sha512_get_state(module); - Py_VISIT(state->sha384_type); - Py_VISIT(state->sha512_type); - return 0; -} - -static int -_sha512_clear(PyObject *module) -{ - SHA512State *state = sha512_get_state(module); - Py_CLEAR(state->sha384_type); - Py_CLEAR(state->sha512_type); - return 0; -} - -static void -_sha512_free(void *module) -{ - _sha512_clear((PyObject *)module); -} - - -/* Initialize this module. */ -static int -_sha512_exec(PyObject *m) -{ - SHA512State* st = sha512_get_state(m); - - st->sha384_type = (PyTypeObject *)PyType_FromModuleAndSpec( - m, &sha512_sha384_type_spec, NULL); - - st->sha512_type = (PyTypeObject *)PyType_FromModuleAndSpec( - m, &sha512_sha512_type_spec, NULL); - - if (st->sha384_type == NULL || st->sha512_type == NULL) { - return -1; - } - - Py_INCREF(st->sha384_type); - if (PyModule_AddObject(m, "SHA384Type", (PyObject *)st->sha384_type) < 0) { - Py_DECREF(st->sha384_type); - return -1; - } - - Py_INCREF(st->sha512_type); - if (PyModule_AddObject(m, "SHA384Type", (PyObject *)st->sha512_type) < 0) { - Py_DECREF(st->sha512_type); - return -1; - } - - return 0; -} - -static PyModuleDef_Slot _sha512_slots[] = { - {Py_mod_exec, _sha512_exec}, - {0, NULL} -}; - -static struct PyModuleDef _sha512module = { - PyModuleDef_HEAD_INIT, - .m_name = "_sha512", - .m_size = sizeof(SHA512State), - .m_methods = SHA_functions, - .m_slots = _sha512_slots, - .m_traverse = _sha512_traverse, - .m_clear = _sha512_clear, - .m_free = _sha512_free -}; - -PyMODINIT_FUNC -PyInit__sha512(void) -{ - return PyModuleDef_Init(&_sha512module); -} diff --git a/PC/config.c b/PC/config.c index cdb5db23c4ae49..b1481d79e6508d 100644 --- a/PC/config.c +++ b/PC/config.c @@ -20,8 +20,7 @@ extern PyObject* PyInit_nt(void); extern PyObject* PyInit__operator(void); extern PyObject* PyInit__signal(void); extern PyObject* PyInit__sha1(void); -extern PyObject* PyInit__sha256(void); -extern PyObject* PyInit__sha512(void); +extern PyObject* PyInit__sha2(void); extern PyObject* PyInit__sha3(void); extern PyObject* PyInit__statistics(void); extern PyObject* PyInit__typing(void); @@ -98,8 +97,7 @@ struct _inittab _PyImport_Inittab[] = { {"_signal", PyInit__signal}, {"_md5", PyInit__md5}, {"_sha1", PyInit__sha1}, - {"_sha256", PyInit__sha256}, - {"_sha512", PyInit__sha512}, + {"_sha2", PyInit__sha2}, {"_sha3", PyInit__sha3}, {"_blake2", PyInit__blake2}, {"time", PyInit_time}, diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index e8e9ff01e306bc..222963bc42d17c 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -408,8 +408,7 @@ - - + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 4820db6f2c32dc..efb96222043ac2 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -869,10 +869,7 @@ Modules - - Modules - - + Modules diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 4e7dfb14d19dec..e9f0061a59d3ba 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -63,9 +63,8 @@ static const char* _Py_stdlib_module_names[] = { "_random", "_scproxy", "_sha1", -"_sha256", +"_sha2", "_sha3", -"_sha512", "_signal", "_sitebuiltins", "_socket", diff --git a/configure b/configure index c00a1e1d2ec986..7c4254f3cb176f 100755 --- a/configure +++ b/configure @@ -686,10 +686,8 @@ MODULE__BLAKE2_FALSE MODULE__BLAKE2_TRUE MODULE__SHA3_FALSE MODULE__SHA3_TRUE -MODULE__SHA512_FALSE -MODULE__SHA512_TRUE -MODULE__SHA256_FALSE -MODULE__SHA256_TRUE +MODULE__SHA2_FALSE +MODULE__SHA2_TRUE MODULE__SHA1_FALSE MODULE__SHA1_TRUE MODULE__MD5_FALSE @@ -1891,9 +1889,9 @@ Optional Packages: leave OpenSSL's defaults untouched, STRING: use a custom string, python and STRING also set TLS 1.2 as minimum TLS version - --with-builtin-hashlib-hashes=md5,sha1,sha256,sha512,sha3,blake2 - builtin hash modules, md5, sha1, sha256, sha512, - sha3 (with shake), blake2 + --with-builtin-hashlib-hashes=md5,sha1,sha2,sha3,blake2 + builtin hash modules, md5, sha1, sha2, sha3 (with + shake), blake2 Some influential environment variables: PKG_CONFIG path to pkg-config utility @@ -25346,7 +25344,7 @@ fi # builtin hash modules -default_hashlib_hashes="md5,sha1,sha256,sha512,sha3,blake2" +default_hashlib_hashes="md5,sha1,sha2,sha3,blake2" $as_echo "#define PY_BUILTIN_HASHLIB_HASHES /**/" >>confdefs.h @@ -25386,10 +25384,8 @@ for builtin_hash in $with_builtin_hashlib_hashes; do with_builtin_md5=yes ;; #( sha1) : with_builtin_sha1=yes ;; #( - sha256) : - with_builtin_sha256=yes ;; #( - sha512) : - with_builtin_sha512=yes ;; #( + sha2) : + with_builtin_sha2=yes ;; #( sha3) : with_builtin_sha3=yes ;; #( blake2) : @@ -26898,72 +26894,38 @@ fi $as_echo "$py_cv_module__sha1" >&6; } - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _sha256" >&5 -$as_echo_n "checking for stdlib extension module _sha256... " >&6; } - if test "$py_cv_module__sha256" != "n/a"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _sha2" >&5 +$as_echo_n "checking for stdlib extension module _sha2... " >&6; } + if test "$py_cv_module__sha2" != "n/a"; then : - if test "$with_builtin_sha256" = yes; then : + if test "$with_builtin_sha2" = yes; then : if true; then : - py_cv_module__sha256=yes + py_cv_module__sha2=yes else - py_cv_module__sha256=missing + py_cv_module__sha2=missing fi else - py_cv_module__sha256=disabled + py_cv_module__sha2=disabled fi fi - as_fn_append MODULE_BLOCK "MODULE__SHA256_STATE=$py_cv_module__sha256$as_nl" - if test "x$py_cv_module__sha256" = xyes; then : + as_fn_append MODULE_BLOCK "MODULE__SHA2_STATE=$py_cv_module__sha2$as_nl" + if test "x$py_cv_module__sha2" = xyes; then : - as_fn_append MODULE_BLOCK "MODULE__SHA256_CFLAGS=-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE$as_nl" fi - if test "$py_cv_module__sha256" = yes; then - MODULE__SHA256_TRUE= - MODULE__SHA256_FALSE='#' + if test "$py_cv_module__sha2" = yes; then + MODULE__SHA2_TRUE= + MODULE__SHA2_FALSE='#' else - MODULE__SHA256_TRUE='#' - MODULE__SHA256_FALSE= + MODULE__SHA2_TRUE='#' + MODULE__SHA2_FALSE= fi - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__sha256" >&5 -$as_echo "$py_cv_module__sha256" >&6; } - - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _sha512" >&5 -$as_echo_n "checking for stdlib extension module _sha512... " >&6; } - if test "$py_cv_module__sha512" != "n/a"; then : - - if test "$with_builtin_sha512" = yes; then : - if true; then : - py_cv_module__sha512=yes -else - py_cv_module__sha512=missing -fi -else - py_cv_module__sha512=disabled -fi - -fi - as_fn_append MODULE_BLOCK "MODULE__SHA512_STATE=$py_cv_module__sha512$as_nl" - if test "x$py_cv_module__sha512" = xyes; then : - - as_fn_append MODULE_BLOCK "MODULE__SHA512_CFLAGS=-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE$as_nl" - - -fi - if test "$py_cv_module__sha512" = yes; then - MODULE__SHA512_TRUE= - MODULE__SHA512_FALSE='#' -else - MODULE__SHA512_TRUE='#' - MODULE__SHA512_FALSE= -fi - - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__sha512" >&5 -$as_echo "$py_cv_module__sha512" >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__sha2" >&5 +$as_echo "$py_cv_module__sha2" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _sha3" >&5 @@ -28337,12 +28299,8 @@ if test -z "${MODULE__SHA1_TRUE}" && test -z "${MODULE__SHA1_FALSE}"; then as_fn_error $? "conditional \"MODULE__SHA1\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__SHA256_TRUE}" && test -z "${MODULE__SHA256_FALSE}"; then - as_fn_error $? "conditional \"MODULE__SHA256\" was never defined. -Usually this means the macro was only invoked conditionally." "$LINENO" 5 -fi -if test -z "${MODULE__SHA512_TRUE}" && test -z "${MODULE__SHA512_FALSE}"; then - as_fn_error $? "conditional \"MODULE__SHA512\" was never defined. +if test -z "${MODULE__SHA2_TRUE}" && test -z "${MODULE__SHA2_FALSE}"; then + as_fn_error $? "conditional \"MODULE__SHA2\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${MODULE__SHA3_TRUE}" && test -z "${MODULE__SHA3_FALSE}"; then diff --git a/configure.ac b/configure.ac index 92a05c011026f2..370bbe07c57634 100644 --- a/configure.ac +++ b/configure.ac @@ -6928,14 +6928,14 @@ AC_DEFINE(PY_SSL_DEFAULT_CIPHERS, 1) ]) # builtin hash modules -default_hashlib_hashes="md5,sha1,sha256,sha512,sha3,blake2" +default_hashlib_hashes="md5,sha1,sha2,sha3,blake2" AC_DEFINE([PY_BUILTIN_HASHLIB_HASHES], [], [enabled builtin hash modules] ) AC_MSG_CHECKING(for --with-builtin-hashlib-hashes) AC_ARG_WITH(builtin-hashlib-hashes, - AS_HELP_STRING([--with-builtin-hashlib-hashes=md5,sha1,sha256,sha512,sha3,blake2], + AS_HELP_STRING([--with-builtin-hashlib-hashes=md5,sha1,sha2,sha3,blake2], [builtin hash modules, - md5, sha1, sha256, sha512, sha3 (with shake), blake2]), + md5, sha1, sha2, sha3 (with shake), blake2]), [ AS_CASE([$with_builtin_hashlib_hashes], [yes], [with_builtin_hashlib_hashes=$default_hashlib_hashes], @@ -6952,8 +6952,7 @@ for builtin_hash in $with_builtin_hashlib_hashes; do AS_CASE($builtin_hash, [md5], [with_builtin_md5=yes], [sha1], [with_builtin_sha1=yes], - [sha256], [with_builtin_sha256=yes], - [sha512], [with_builtin_sha512=yes], + [sha2], [with_builtin_sha2=yes], [sha3], [with_builtin_sha3=yes], [blake2], [with_builtin_blake2=yes] ) @@ -7197,11 +7196,8 @@ dnl By default we always compile these even when OpenSSL is available dnl (issue #14693). The modules are small. PY_STDLIB_MOD([_md5], [test "$with_builtin_md5" = yes]) PY_STDLIB_MOD([_sha1], [test "$with_builtin_sha1" = yes]) -PY_STDLIB_MOD([_sha256], - [test "$with_builtin_sha256" = yes], [], - [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE]) -PY_STDLIB_MOD([_sha512], - [test "$with_builtin_sha512" = yes], [], +PY_STDLIB_MOD([_sha2], + [test "$with_builtin_sha2" = yes], [], [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE]) PY_STDLIB_MOD([_sha3], [test "$with_builtin_sha3" = yes]) PY_STDLIB_MOD([_blake2], From df7ccf6138b1a2ce0b82ff06aa3497ca4d38c90d Mon Sep 17 00:00:00 2001 From: penguin_wwy <940375606@qq.com> Date: Thu, 16 Feb 2023 19:31:41 +0800 Subject: [PATCH 19/76] gh-101928: fix crash in compiler on multi-line lambda in function call (#101933) --- Lib/test/test_compile.py | 11 +++++++++++ Python/compile.c | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 90b067bcf30912..a77742c0cfa6fc 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1155,6 +1155,17 @@ def test_if_expression_expression_empty_block(self): with self.subTest(expr=expr): compile(expr, "", "exec") + def test_multi_line_lambda_as_argument(self): + # See gh-101928 + compile(""" +def foo(param, lambda_exp): + pass + +foo(param=0, + lambda_exp=lambda: + 1) + """, "", "exec") + @requires_debug_ranges() class TestSourcePositions(unittest.TestCase): diff --git a/Python/compile.c b/Python/compile.c index 0534b536e3d12e..c3b344c7af2a7f 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -9085,8 +9085,8 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) Py_DECREF(cnt); break; case RETURN_VALUE: - INSTR_SET_OP1(inst, RETURN_CONST, oparg); - INSTR_SET_OP0(&bb->b_instr[i + 1], NOP); + INSTR_SET_OP0(inst, NOP); + INSTR_SET_OP1(&bb->b_instr[++i], RETURN_CONST, oparg); break; } break; From 36b139af638cdeb671cb6b8b0315b254148688f7 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 16 Feb 2023 12:31:59 +0000 Subject: [PATCH 20/76] gh-101951: use textwrap.dedent in compiler tests to make them more readable (GH-101950) Fixes #101951. Automerge-Triggered-By: GH:iritkatriel --- Lib/test/test_compile.py | 288 ++++++++++++++++++++------------------- 1 file changed, 145 insertions(+), 143 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index a77742c0cfa6fc..fe775779c50f50 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -115,24 +115,24 @@ def test_extended_arg(self): repeat = 2000 longexpr = 'x = x or ' + '-x' * repeat g = {} - code = ''' -def f(x): - %s - %s - %s - %s - %s - %s - %s - %s - %s - %s - # the expressions above have no effect, x == argument - while x: - x -= 1 - # EXTENDED_ARG/JUMP_ABSOLUTE here - return x -''' % ((longexpr,)*10) + code = textwrap.dedent(''' + def f(x): + %s + %s + %s + %s + %s + %s + %s + %s + %s + %s + # the expressions above have no effect, x == argument + while x: + x -= 1 + # EXTENDED_ARG/JUMP_ABSOLUTE here + return x + ''' % ((longexpr,)*10)) exec(code, g) self.assertEqual(g['f'](5), 0) @@ -148,10 +148,11 @@ def test_float_literals(self): def test_indentation(self): # testing compile() of indented block w/o trailing newline" - s = """ -if 1: - if 2: - pass""" + s = textwrap.dedent(""" + if 1: + if 2: + pass + """) compile(s, "", "exec") # This test is probably specific to CPython and may not generalize @@ -1157,14 +1158,15 @@ def test_if_expression_expression_empty_block(self): def test_multi_line_lambda_as_argument(self): # See gh-101928 - compile(""" -def foo(param, lambda_exp): - pass + code = textwrap.dedent(""" + def foo(param, lambda_exp): + pass -foo(param=0, - lambda_exp=lambda: - 1) - """, "", "exec") + foo(param=0, + lambda_exp=lambda: + 1) + """) + compile(code, "", "exec") @requires_debug_ranges() @@ -1252,24 +1254,24 @@ def test_compiles_to_extended_op_arg(self): column=2, end_column=9, occurrence=2) def test_multiline_expression(self): - snippet = """\ -f( - 1, 2, 3, 4 -) -""" + snippet = textwrap.dedent("""\ + f( + 1, 2, 3, 4 + ) + """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'CALL', line=1, end_line=3, column=0, end_column=1) @requires_specialization def test_multiline_boolean_expression(self): - snippet = """\ -if (a or - (b and not c) or - not ( - d > 0)): - x = 42 -""" + snippet = textwrap.dedent("""\ + if (a or + (b and not c) or + not ( + d > 0)): + x = 42 + """) compiled_code, _ = self.check_positions_against_ast(snippet) # jump if a is true: self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_TRUE', @@ -1288,11 +1290,11 @@ def test_multiline_boolean_expression(self): line=4, end_line=4, column=8, end_column=13, occurrence=2) def test_multiline_assert(self): - snippet = """\ -assert (a > 0 and - bb > 0 and - ccc == 4), "error msg" -""" + snippet = textwrap.dedent("""\ + assert (a > 0 and + bb > 0 and + ccc == 4), "error msg" + """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_ASSERTION_ERROR', line=1, end_line=3, column=0, end_column=30, occurrence=1) @@ -1305,14 +1307,14 @@ def test_multiline_assert(self): line=1, end_line=3, column=0, end_column=30, occurrence=1) def test_multiline_generator_expression(self): - snippet = """\ -((x, - 2*x) - for x - in [1,2,3] if (x > 0 - and x < 100 - and x != 50)) -""" + snippet = textwrap.dedent("""\ + ((x, + 2*x) + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)) + """) compiled_code, _ = self.check_positions_against_ast(snippet) compiled_code = compiled_code.co_consts[0] self.assertIsInstance(compiled_code, types.CodeType) @@ -1324,14 +1326,14 @@ def test_multiline_generator_expression(self): line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_async_generator_expression(self): - snippet = """\ -((x, - 2*x) - async for x - in [1,2,3] if (x > 0 - and x < 100 - and x != 50)) -""" + snippet = textwrap.dedent("""\ + ((x, + 2*x) + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)) + """) compiled_code, _ = self.check_positions_against_ast(snippet) compiled_code = compiled_code.co_consts[0] self.assertIsInstance(compiled_code, types.CodeType) @@ -1341,14 +1343,14 @@ def test_multiline_async_generator_expression(self): line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_list_comprehension(self): - snippet = """\ -[(x, - 2*x) - for x - in [1,2,3] if (x > 0 - and x < 100 - and x != 50)] -""" + snippet = textwrap.dedent("""\ + [(x, + 2*x) + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)] + """) compiled_code, _ = self.check_positions_against_ast(snippet) compiled_code = compiled_code.co_consts[0] self.assertIsInstance(compiled_code, types.CodeType) @@ -1360,15 +1362,15 @@ def test_multiline_list_comprehension(self): line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_async_list_comprehension(self): - snippet = """\ -async def f(): - [(x, - 2*x) - async for x - in [1,2,3] if (x > 0 - and x < 100 - and x != 50)] -""" + snippet = textwrap.dedent("""\ + async def f(): + [(x, + 2*x) + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)] + """) compiled_code, _ = self.check_positions_against_ast(snippet) g = {} eval(compiled_code, g) @@ -1382,14 +1384,14 @@ async def f(): line=2, end_line=7, column=4, end_column=36, occurrence=1) def test_multiline_set_comprehension(self): - snippet = """\ -{(x, - 2*x) - for x - in [1,2,3] if (x > 0 - and x < 100 - and x != 50)} -""" + snippet = textwrap.dedent("""\ + {(x, + 2*x) + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) compiled_code, _ = self.check_positions_against_ast(snippet) compiled_code = compiled_code.co_consts[0] self.assertIsInstance(compiled_code, types.CodeType) @@ -1401,15 +1403,15 @@ def test_multiline_set_comprehension(self): line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_async_set_comprehension(self): - snippet = """\ -async def f(): - {(x, - 2*x) - async for x - in [1,2,3] if (x > 0 - and x < 100 - and x != 50)} -""" + snippet = textwrap.dedent("""\ + async def f(): + {(x, + 2*x) + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) compiled_code, _ = self.check_positions_against_ast(snippet) g = {} eval(compiled_code, g) @@ -1423,14 +1425,14 @@ async def f(): line=2, end_line=7, column=4, end_column=36, occurrence=1) def test_multiline_dict_comprehension(self): - snippet = """\ -{x: - 2*x - for x - in [1,2,3] if (x > 0 - and x < 100 - and x != 50)} -""" + snippet = textwrap.dedent("""\ + {x: + 2*x + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) compiled_code, _ = self.check_positions_against_ast(snippet) compiled_code = compiled_code.co_consts[0] self.assertIsInstance(compiled_code, types.CodeType) @@ -1442,15 +1444,15 @@ def test_multiline_dict_comprehension(self): line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_async_dict_comprehension(self): - snippet = """\ -async def f(): - {x: - 2*x - async for x - in [1,2,3] if (x > 0 - and x < 100 - and x != 50)} -""" + snippet = textwrap.dedent("""\ + async def f(): + {x: + 2*x + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) compiled_code, _ = self.check_positions_against_ast(snippet) g = {} eval(compiled_code, g) @@ -1464,11 +1466,11 @@ async def f(): line=2, end_line=7, column=4, end_column=36, occurrence=1) def test_matchcase_sequence(self): - snippet = """\ -match x: - case a, b: - pass -""" + snippet = textwrap.dedent("""\ + match x: + case a, b: + pass + """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_SEQUENCE', line=2, end_line=2, column=9, end_column=13, occurrence=1) @@ -1480,11 +1482,11 @@ def test_matchcase_sequence(self): line=2, end_line=2, column=9, end_column=13, occurrence=2) def test_matchcase_sequence_wildcard(self): - snippet = """\ -match x: - case a, *b, c: - pass -""" + snippet = textwrap.dedent("""\ + match x: + case a, *b, c: + pass + """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_SEQUENCE', line=2, end_line=2, column=9, end_column=17, occurrence=1) @@ -1498,11 +1500,11 @@ def test_matchcase_sequence_wildcard(self): line=2, end_line=2, column=9, end_column=17, occurrence=3) def test_matchcase_mapping(self): - snippet = """\ -match x: - case {"a" : a, "b": b}: - pass -""" + snippet = textwrap.dedent("""\ + match x: + case {"a" : a, "b": b}: + pass + """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_MAPPING', line=2, end_line=2, column=9, end_column=26, occurrence=1) @@ -1514,11 +1516,11 @@ def test_matchcase_mapping(self): line=2, end_line=2, column=9, end_column=26, occurrence=2) def test_matchcase_mapping_wildcard(self): - snippet = """\ -match x: - case {"a" : a, "b": b, **c}: - pass -""" + snippet = textwrap.dedent("""\ + match x: + case {"a" : a, "b": b, **c}: + pass + """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_MAPPING', line=2, end_line=2, column=9, end_column=31, occurrence=1) @@ -1530,11 +1532,11 @@ def test_matchcase_mapping_wildcard(self): line=2, end_line=2, column=9, end_column=31, occurrence=2) def test_matchcase_class(self): - snippet = """\ -match x: - case C(a, b): - pass -""" + snippet = textwrap.dedent("""\ + match x: + case C(a, b): + pass + """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_CLASS', line=2, end_line=2, column=9, end_column=16, occurrence=1) @@ -1546,11 +1548,11 @@ def test_matchcase_class(self): line=2, end_line=2, column=9, end_column=16, occurrence=2) def test_matchcase_or(self): - snippet = """\ -match x: - case C(1) | C(2): - pass -""" + snippet = textwrap.dedent("""\ + match x: + case C(1) | C(2): + pass + """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_CLASS', line=2, end_line=2, column=9, end_column=13, occurrence=1) From 739c026f4488bd2e37d500a2c3d948aaf929b641 Mon Sep 17 00:00:00 2001 From: Rayyan Ansari Date: Thu, 16 Feb 2023 14:52:24 +0000 Subject: [PATCH 21/76] gh-101881: Support (non-)blocking read/write functions on Windows pipes (GH-101882) * fileutils: handle non-blocking pipe IO on Windows Handle erroring operations on non-blocking pipes by reading the _doserrno code. Limit writes on non-blocking pipes that are too large. * Support blocking functions on Windows Use the GetNamedPipeHandleState and SetNamedPipeHandleState Win32 API functions to add support for os.get_blocking and os.set_blocking. --- Doc/library/os.rst | 12 ++- Include/internal/pycore_fileutils.h | 4 +- Lib/test/test_os.py | 1 + ...-02-13-18-05-49.gh-issue-101881._TnHzN.rst | 1 + ...-02-15-11-08-10.gh-issue-101881.fScr3m.rst | 1 + Modules/clinic/posixmodule.c.h | 18 +--- Modules/posixmodule.c | 2 - Python/fileutils.c | 93 ++++++++++++++++++- 8 files changed, 107 insertions(+), 25 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst create mode 100644 Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index fb091176767f7a..85924d0e48366b 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1091,13 +1091,17 @@ as internal buffering of data. See also :func:`set_blocking` and :meth:`socket.socket.setblocking`. - .. availability:: Unix. + .. availability:: Unix, Windows. The function is limited on Emscripten and WASI, see :ref:`wasm-availability` for more information. + On Windows, this function is limited to pipes. + .. versionadded:: 3.5 + .. versionchanged:: 3.12 + Added support for pipes on Windows. .. function:: isatty(fd, /) @@ -1565,13 +1569,17 @@ or `the MSDN `_ on Windo See also :func:`get_blocking` and :meth:`socket.socket.setblocking`. - .. availability:: Unix. + .. availability:: Unix, Windows. The function is limited on Emscripten and WASI, see :ref:`wasm-availability` for more information. + On Windows, this function is limited to pipes. + .. versionadded:: 3.5 + .. versionchanged:: 3.12 + Added support for pipes on Windows. .. data:: SF_NODISKIO SF_MNOWAIT diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index ac89c43d569c07..f8e2bf22590888 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -160,11 +160,11 @@ PyAPI_FUNC(int) _Py_set_inheritable_async_safe(int fd, int inheritable, PyAPI_FUNC(int) _Py_dup(int fd); -#ifndef MS_WINDOWS PyAPI_FUNC(int) _Py_get_blocking(int fd); PyAPI_FUNC(int) _Py_set_blocking(int fd, int blocking); -#else /* MS_WINDOWS */ + +#ifdef MS_WINDOWS PyAPI_FUNC(void*) _Py_get_osfhandle_noraise(int fd); PyAPI_FUNC(void*) _Py_get_osfhandle(int fd); diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 387d2581c06fc6..deea207bfdadd9 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -4099,6 +4099,7 @@ def test_path_t_converter_and_custom_class(self): @unittest.skipUnless(hasattr(os, 'get_blocking'), 'needs os.get_blocking() and os.set_blocking()') @unittest.skipIf(support.is_emscripten, "Cannot unset blocking flag") +@unittest.skipIf(sys.platform == 'win32', 'Windows only supports blocking on pipes') class BlockingTests(unittest.TestCase): def test_blocking(self): fd = os.open(__file__, os.O_RDONLY) diff --git a/Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst b/Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst new file mode 100644 index 00000000000000..ba58dd4f5cb450 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst @@ -0,0 +1 @@ +Add support for the os.get_blocking() and os.set_blocking() functions on Windows. diff --git a/Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst b/Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst new file mode 100644 index 00000000000000..099b2c1c07a665 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst @@ -0,0 +1 @@ +Handle read and write operations on non-blocking pipes properly on Windows. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 5e04507ddd6917..dcd25c28370c93 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -10402,8 +10402,6 @@ os_set_handle_inheritable(PyObject *module, PyObject *const *args, Py_ssize_t na #endif /* defined(MS_WINDOWS) */ -#if !defined(MS_WINDOWS) - PyDoc_STRVAR(os_get_blocking__doc__, "get_blocking($module, fd, /)\n" "--\n" @@ -10439,10 +10437,6 @@ os_get_blocking(PyObject *module, PyObject *arg) return return_value; } -#endif /* !defined(MS_WINDOWS) */ - -#if !defined(MS_WINDOWS) - PyDoc_STRVAR(os_set_blocking__doc__, "set_blocking($module, fd, blocking, /)\n" "--\n" @@ -10482,8 +10476,6 @@ os_set_blocking(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -#endif /* !defined(MS_WINDOWS) */ - PyDoc_STRVAR(os_DirEntry_is_symlink__doc__, "is_symlink($self, /)\n" "--\n" @@ -11789,14 +11781,6 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #define OS_SET_HANDLE_INHERITABLE_METHODDEF #endif /* !defined(OS_SET_HANDLE_INHERITABLE_METHODDEF) */ -#ifndef OS_GET_BLOCKING_METHODDEF - #define OS_GET_BLOCKING_METHODDEF -#endif /* !defined(OS_GET_BLOCKING_METHODDEF) */ - -#ifndef OS_SET_BLOCKING_METHODDEF - #define OS_SET_BLOCKING_METHODDEF -#endif /* !defined(OS_SET_BLOCKING_METHODDEF) */ - #ifndef OS_GETRANDOM_METHODDEF #define OS_GETRANDOM_METHODDEF #endif /* !defined(OS_GETRANDOM_METHODDEF) */ @@ -11812,4 +11796,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=a3f76228b549e8ec input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1b0eb6a76b1a0e28 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index d9e93473aeadaa..524dc7eb1ccc97 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -13930,7 +13930,6 @@ os_set_handle_inheritable_impl(PyObject *module, intptr_t handle, } #endif /* MS_WINDOWS */ -#ifndef MS_WINDOWS /*[clinic input] os.get_blocking -> bool fd: int @@ -13978,7 +13977,6 @@ os_set_blocking_impl(PyObject *module, int fd, int blocking) return NULL; Py_RETURN_NONE; } -#endif /* !MS_WINDOWS */ /*[clinic input] diff --git a/Python/fileutils.c b/Python/fileutils.c index 22b2257a56d0ec..897c2f9f4ea160 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1750,7 +1750,15 @@ _Py_read(int fd, void *buf, size_t count) Py_BEGIN_ALLOW_THREADS errno = 0; #ifdef MS_WINDOWS + _doserrno = 0; n = read(fd, buf, (int)count); + // read() on a non-blocking empty pipe fails with EINVAL, which is + // mapped from the Windows error code ERROR_NO_DATA. + if (n < 0 && errno == EINVAL) { + if (_doserrno == ERROR_NO_DATA) { + errno = EAGAIN; + } + } #else n = read(fd, buf, count); #endif @@ -1804,6 +1812,7 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held) } } } + #endif if (count > _PY_WRITE_MAX) { count = _PY_WRITE_MAX; @@ -1814,7 +1823,18 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held) Py_BEGIN_ALLOW_THREADS errno = 0; #ifdef MS_WINDOWS - n = write(fd, buf, (int)count); + // write() on a non-blocking pipe fails with ENOSPC on Windows if + // the pipe lacks available space for the entire buffer. + int c = (int)count; + do { + _doserrno = 0; + n = write(fd, buf, c); + if (n >= 0 || errno != ENOSPC || _doserrno != 0) { + break; + } + errno = EAGAIN; + c /= 2; + } while (c > 0); #else n = write(fd, buf, count); #endif @@ -1829,7 +1849,18 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held) do { errno = 0; #ifdef MS_WINDOWS - n = write(fd, buf, (int)count); + // write() on a non-blocking pipe fails with ENOSPC on Windows if + // the pipe lacks available space for the entire buffer. + int c = (int)count; + do { + _doserrno = 0; + n = write(fd, buf, c); + if (n >= 0 || errno != ENOSPC || _doserrno != 0) { + break; + } + errno = EAGAIN; + c /= 2; + } while (c > 0); #else n = write(fd, buf, count); #endif @@ -2450,6 +2481,64 @@ _Py_set_blocking(int fd, int blocking) return -1; } #else /* MS_WINDOWS */ +int +_Py_get_blocking(int fd) +{ + HANDLE handle; + DWORD mode; + BOOL success; + + handle = _Py_get_osfhandle(fd); + if (handle == INVALID_HANDLE_VALUE) { + return -1; + } + + Py_BEGIN_ALLOW_THREADS + success = GetNamedPipeHandleStateW(handle, &mode, + NULL, NULL, NULL, NULL, 0); + Py_END_ALLOW_THREADS + + if (!success) { + PyErr_SetFromWindowsErr(0); + return -1; + } + + return !(mode & PIPE_NOWAIT); +} + +int +_Py_set_blocking(int fd, int blocking) +{ + HANDLE handle; + DWORD mode; + BOOL success; + + handle = _Py_get_osfhandle(fd); + if (handle == INVALID_HANDLE_VALUE) { + return -1; + } + + Py_BEGIN_ALLOW_THREADS + success = GetNamedPipeHandleStateW(handle, &mode, + NULL, NULL, NULL, NULL, 0); + if (success) { + if (blocking) { + mode &= ~PIPE_NOWAIT; + } + else { + mode |= PIPE_NOWAIT; + } + success = SetNamedPipeHandleState(handle, &mode, NULL, NULL); + } + Py_END_ALLOW_THREADS + + if (!success) { + PyErr_SetFromWindowsErr(0); + return -1; + } + return 0; +} + void* _Py_get_osfhandle_noraise(int fd) { From 924a3bfa28578802eb9ca77a66fb5d4762a62f14 Mon Sep 17 00:00:00 2001 From: sblondon Date: Thu, 16 Feb 2023 16:13:21 +0100 Subject: [PATCH 22/76] gh-93573: Replace wrong example domains in configparser doc (GH-93574) * Replace bitbucket.org domain by forge.example * Update example to python.org * Use explicitly invalid domain topsecret.server.com domain is not controled by PSF. It's replaced by invalid topsecret.server.example domain. It follows RFC 2606, which advise .example as TLD for documentation. --- Doc/library/configparser.rst | 46 ++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index a925a3dd4fb9c2..a7f75fd6e84f4c 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -69,10 +69,10 @@ Let's take a very basic configuration file that looks like this: CompressionLevel = 9 ForwardX11 = yes - [bitbucket.org] + [forge.example] User = hg - [topsecret.server.com] + [topsecret.server.example] Port = 50022 ForwardX11 = no @@ -89,10 +89,10 @@ creating the above configuration file programmatically. >>> config['DEFAULT'] = {'ServerAliveInterval': '45', ... 'Compression': 'yes', ... 'CompressionLevel': '9'} - >>> config['bitbucket.org'] = {} - >>> config['bitbucket.org']['User'] = 'hg' - >>> config['topsecret.server.com'] = {} - >>> topsecret = config['topsecret.server.com'] + >>> config['forge.example'] = {} + >>> config['forge.example']['User'] = 'hg' + >>> config['topsecret.server.example'] = {} + >>> topsecret = config['topsecret.server.example'] >>> topsecret['Port'] = '50022' # mutates the parser >>> topsecret['ForwardX11'] = 'no' # same here >>> config['DEFAULT']['ForwardX11'] = 'yes' @@ -115,28 +115,28 @@ back and explore the data it holds. >>> config.read('example.ini') ['example.ini'] >>> config.sections() - ['bitbucket.org', 'topsecret.server.com'] - >>> 'bitbucket.org' in config + ['forge.example', 'topsecret.server.example'] + >>> 'forge.example' in config True - >>> 'bytebong.com' in config + >>> 'python.org' in config False - >>> config['bitbucket.org']['User'] + >>> config['forge.example']['User'] 'hg' >>> config['DEFAULT']['Compression'] 'yes' - >>> topsecret = config['topsecret.server.com'] + >>> topsecret = config['topsecret.server.example'] >>> topsecret['ForwardX11'] 'no' >>> topsecret['Port'] '50022' - >>> for key in config['bitbucket.org']: # doctest: +SKIP + >>> for key in config['forge.example']: # doctest: +SKIP ... print(key) user compressionlevel serveraliveinterval compression forwardx11 - >>> config['bitbucket.org']['ForwardX11'] + >>> config['forge.example']['ForwardX11'] 'yes' As we can see above, the API is pretty straightforward. The only bit of magic @@ -154,15 +154,15 @@ configuration while the previously existing keys are retained. >>> another_config = configparser.ConfigParser() >>> another_config.read('example.ini') ['example.ini'] - >>> another_config['topsecret.server.com']['Port'] + >>> another_config['topsecret.server.example']['Port'] '50022' - >>> another_config.read_string("[topsecret.server.com]\nPort=48484") - >>> another_config['topsecret.server.com']['Port'] + >>> another_config.read_string("[topsecret.server.example]\nPort=48484") + >>> another_config['topsecret.server.example']['Port'] '48484' - >>> another_config.read_dict({"topsecret.server.com": {"Port": 21212}}) - >>> another_config['topsecret.server.com']['Port'] + >>> another_config.read_dict({"topsecret.server.example": {"Port": 21212}}) + >>> another_config['topsecret.server.example']['Port'] '21212' - >>> another_config['topsecret.server.com']['ForwardX11'] + >>> another_config['topsecret.server.example']['ForwardX11'] 'no' This behaviour is equivalent to a :meth:`ConfigParser.read` call with several @@ -195,9 +195,9 @@ recognizes Boolean values from ``'yes'``/``'no'``, ``'on'``/``'off'``, >>> topsecret.getboolean('ForwardX11') False - >>> config['bitbucket.org'].getboolean('ForwardX11') + >>> config['forge.example'].getboolean('ForwardX11') True - >>> config.getboolean('bitbucket.org', 'Compression') + >>> config.getboolean('forge.example', 'Compression') True Apart from :meth:`~ConfigParser.getboolean`, config parsers also @@ -224,7 +224,7 @@ provide fallback values: Please note that default values have precedence over fallback values. For instance, in our example the ``'CompressionLevel'`` key was specified only in the ``'DEFAULT'`` section. If we try to get it from -the section ``'topsecret.server.com'``, we will always get the default, +the section ``'topsecret.server.example'``, we will always get the default, even if we specify a fallback: .. doctest:: @@ -239,7 +239,7 @@ the ``fallback`` keyword-only argument: .. doctest:: - >>> config.get('bitbucket.org', 'monster', + >>> config.get('forge.example', 'monster', ... fallback='No such things as monsters') 'No such things as monsters' From 68bd8c5e2efab64ff9d38a214775164182179431 Mon Sep 17 00:00:00 2001 From: Eclips4 <80244920+Eclips4@users.noreply.github.com> Date: Thu, 16 Feb 2023 20:46:43 +0300 Subject: [PATCH 23/76] gh-101952: Fix possible segfault in `BUILD_SET` opcode (#101958) --- .../2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst | 1 + Python/bytecodes.c | 2 ++ Python/generated_cases.c.h | 2 ++ 3 files changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst new file mode 100644 index 00000000000000..3902c988c8bf9f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst @@ -0,0 +1 @@ +Fix possible segfault in ``BUILD_SET`` opcode, when new set created. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d5d5034cbfbf74..84747f1758e06c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1302,6 +1302,8 @@ dummy_func( inst(BUILD_SET, (values[oparg] -- set)) { set = PySet_New(NULL); + if (set == NULL) + goto error; int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 8b8a7161ad898e..730dfb7426acbf 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1649,6 +1649,8 @@ PyObject **values = &PEEK(oparg); PyObject *set; set = PySet_New(NULL); + if (set == NULL) + goto error; int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; From 226484e47599a93f5bf033ac47198e68ff401432 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Thu, 16 Feb 2023 12:57:59 -0500 Subject: [PATCH 24/76] gh-99942: correct the pkg-config/python-config flags for cygwin/android --- .../next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst | 2 ++ configure | 2 +- configure.ac | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst diff --git a/Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst b/Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst new file mode 100644 index 00000000000000..5b692c3cc458c5 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst @@ -0,0 +1,2 @@ +On Android, in a static build, python-config in embed mode no longer +incorrectly reports a library to link to. diff --git a/configure b/configure index 7c4254f3cb176f..17dc62fb63de3b 100755 --- a/configure +++ b/configure @@ -21496,7 +21496,7 @@ $as_echo "$LDVERSION" >&6; } # On Android and Cygwin the shared libraries must be linked with libpython. -if test -n "$ANDROID_API_LEVEL" -o "$MACHDEP" = "cygwin"; then +if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then LIBPYTHON="-lpython${VERSION}${ABIFLAGS}" else LIBPYTHON='' diff --git a/configure.ac b/configure.ac index 370bbe07c57634..bc288b86cfa590 100644 --- a/configure.ac +++ b/configure.ac @@ -5759,7 +5759,7 @@ AC_MSG_RESULT($LDVERSION) # On Android and Cygwin the shared libraries must be linked with libpython. AC_SUBST(LIBPYTHON) -if test -n "$ANDROID_API_LEVEL" -o "$MACHDEP" = "cygwin"; then +if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then LIBPYTHON="-lpython${VERSION}${ABIFLAGS}" else LIBPYTHON='' From a5024a261a75dafa4fb6613298dcb64a9603d9c7 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Fri, 17 Feb 2023 00:18:21 +0530 Subject: [PATCH 25/76] GH-96764: rewrite `asyncio.wait_for` to use `asyncio.timeout` (#98518) Changes `asyncio.wait_for` to use `asyncio.timeout` as its underlying implementation. --- Lib/asyncio/tasks.py | 77 ++++------- Lib/test/test_asyncio/test_futures2.py | 7 +- Lib/test/test_asyncio/test_waitfor.py | 127 ++++++++++++++---- ...2-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst | 1 + 4 files changed, 133 insertions(+), 79 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index e78719de216fd0..a2e06d5ef72f42 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -24,6 +24,7 @@ from . import events from . import exceptions from . import futures +from . import timeouts from .coroutines import _is_coroutine # Helper to generate new task names @@ -437,65 +438,44 @@ async def wait_for(fut, timeout): If the wait is cancelled, the task is also cancelled. + If the task supresses the cancellation and returns a value instead, + that value is returned. + This function is a coroutine. """ - loop = events.get_running_loop() + # The special case for timeout <= 0 is for the following case: + # + # async def test_waitfor(): + # func_started = False + # + # async def func(): + # nonlocal func_started + # func_started = True + # + # try: + # await asyncio.wait_for(func(), 0) + # except asyncio.TimeoutError: + # assert not func_started + # else: + # assert False + # + # asyncio.run(test_waitfor()) - if timeout is None: - return await fut - if timeout <= 0: - fut = ensure_future(fut, loop=loop) + if timeout is not None and timeout <= 0: + fut = ensure_future(fut) if fut.done(): return fut.result() - await _cancel_and_wait(fut, loop=loop) + await _cancel_and_wait(fut) try: return fut.result() except exceptions.CancelledError as exc: - raise exceptions.TimeoutError() from exc - - waiter = loop.create_future() - timeout_handle = loop.call_later(timeout, _release_waiter, waiter) - cb = functools.partial(_release_waiter, waiter) - - fut = ensure_future(fut, loop=loop) - fut.add_done_callback(cb) - - try: - # wait until the future completes or the timeout - try: - await waiter - except exceptions.CancelledError: - if fut.done(): - return fut.result() - else: - fut.remove_done_callback(cb) - # We must ensure that the task is not running - # after wait_for() returns. - # See https://bugs.python.org/issue32751 - await _cancel_and_wait(fut, loop=loop) - raise - - if fut.done(): - return fut.result() - else: - fut.remove_done_callback(cb) - # We must ensure that the task is not running - # after wait_for() returns. - # See https://bugs.python.org/issue32751 - await _cancel_and_wait(fut, loop=loop) - # In case task cancellation failed with some - # exception, we should re-raise it - # See https://bugs.python.org/issue40607 - try: - return fut.result() - except exceptions.CancelledError as exc: - raise exceptions.TimeoutError() from exc - finally: - timeout_handle.cancel() + raise TimeoutError from exc + async with timeouts.timeout(timeout): + return await fut async def _wait(fs, timeout, return_when, loop): """Internal helper for wait(). @@ -541,9 +521,10 @@ def _on_completion(f): return done, pending -async def _cancel_and_wait(fut, loop): +async def _cancel_and_wait(fut): """Cancel the *fut* future or task and wait until it completes.""" + loop = events.get_running_loop() waiter = loop.create_future() cb = functools.partial(_release_waiter, waiter) fut.add_done_callback(cb) diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index 9e7a5775a70383..b7cfffb76bd8f1 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -86,10 +86,9 @@ async def test_recursive_repr_for_pending_tasks(self): async def func(): return asyncio.all_tasks() - # The repr() call should not raise RecursiveError at first. - # The check for returned string is not very reliable but - # exact comparison for the whole string is even weaker. - self.assertIn('...', repr(await asyncio.wait_for(func(), timeout=10))) + # The repr() call should not raise RecursionError at first. + waiter = await asyncio.wait_for(asyncio.Task(func()),timeout=10) + self.assertIn('...', repr(waiter)) if __name__ == '__main__': diff --git a/Lib/test/test_asyncio/test_waitfor.py b/Lib/test/test_asyncio/test_waitfor.py index 45498fa097f6bc..ed80540b2b3852 100644 --- a/Lib/test/test_asyncio/test_waitfor.py +++ b/Lib/test/test_asyncio/test_waitfor.py @@ -237,33 +237,6 @@ async def inner(): with self.assertRaises(FooException): await foo() - async def test_wait_for_self_cancellation(self): - async def inner(): - try: - await asyncio.sleep(0.3) - except asyncio.CancelledError: - try: - await asyncio.sleep(0.3) - except asyncio.CancelledError: - await asyncio.sleep(0.3) - - return 42 - - inner_task = asyncio.create_task(inner()) - - wait = asyncio.wait_for(inner_task, timeout=0.1) - - # Test that wait_for itself is properly cancellable - # even when the initial task holds up the initial cancellation. - task = asyncio.create_task(wait) - await asyncio.sleep(0.2) - task.cancel() - - with self.assertRaises(asyncio.CancelledError): - await task - - self.assertEqual(await inner_task, 42) - async def _test_cancel_wait_for(self, timeout): loop = asyncio.get_running_loop() @@ -289,6 +262,106 @@ async def test_cancel_blocking_wait_for(self): async def test_cancel_wait_for(self): await self._test_cancel_wait_for(60.0) + async def test_wait_for_cancel_suppressed(self): + # GH-86296: Supressing CancelledError is discouraged + # but if a task subpresses CancelledError and returns a value, + # `wait_for` should return the value instead of raising CancelledError. + # This is the same behavior as `asyncio.timeout`. + + async def return_42(): + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + return 42 + + res = await asyncio.wait_for(return_42(), timeout=0.1) + self.assertEqual(res, 42) + + + async def test_wait_for_issue86296(self): + # GH-86296: The task should get cancelled and not run to completion. + # inner completes in one cycle of the event loop so it + # completes before the task is cancelled. + + async def inner(): + return 'done' + + inner_task = asyncio.create_task(inner()) + reached_end = False + + async def wait_for_coro(): + await asyncio.wait_for(inner_task, timeout=100) + await asyncio.sleep(1) + nonlocal reached_end + reached_end = True + + task = asyncio.create_task(wait_for_coro()) + self.assertFalse(task.done()) + # Run the task + await asyncio.sleep(0) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + self.assertTrue(inner_task.done()) + self.assertEqual(await inner_task, 'done') + self.assertFalse(reached_end) + + +class WaitForShieldTests(unittest.IsolatedAsyncioTestCase): + + async def test_zero_timeout(self): + # `asyncio.shield` creates a new task which wraps the passed in + # awaitable and shields it from cancellation so with timeout=0 + # the task returned by `asyncio.shield` aka shielded_task gets + # cancelled immediately and the task wrapped by it is scheduled + # to run. + + async def coro(): + await asyncio.sleep(0.01) + return 'done' + + task = asyncio.create_task(coro()) + with self.assertRaises(asyncio.TimeoutError): + shielded_task = asyncio.shield(task) + await asyncio.wait_for(shielded_task, timeout=0) + + # Task is running in background + self.assertFalse(task.done()) + self.assertFalse(task.cancelled()) + self.assertTrue(shielded_task.cancelled()) + + # Wait for the task to complete + await asyncio.sleep(0.1) + self.assertTrue(task.done()) + + + async def test_none_timeout(self): + # With timeout=None the timeout is disabled so it + # runs till completion. + async def coro(): + await asyncio.sleep(0.1) + return 'done' + + task = asyncio.create_task(coro()) + await asyncio.wait_for(asyncio.shield(task), timeout=None) + + self.assertTrue(task.done()) + self.assertEqual(await task, "done") + + async def test_shielded_timeout(self): + # shield prevents the task from being cancelled. + async def coro(): + await asyncio.sleep(0.1) + return 'done' + + task = asyncio.create_task(coro()) + with self.assertRaises(asyncio.TimeoutError): + await asyncio.wait_for(asyncio.shield(task), timeout=0.01) + + self.assertFalse(task.done()) + self.assertFalse(task.cancelled()) + self.assertEqual(await task, "done") + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst b/Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst new file mode 100644 index 00000000000000..a0174291cbc311 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst @@ -0,0 +1 @@ +:func:`asyncio.wait_for` now uses :func:`asyncio.timeout` as its underlying implementation. Patch by Kumar Aditya. From 4d8959b73ac194ca9a2f623dcb5c23680f7d8536 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 16 Feb 2023 14:05:31 -0700 Subject: [PATCH 26/76] gh-101758: Add _PyState_AddModule() Back for the Stable ABI (gh-101956) We're adding the function back, only for the stable ABI symbol and not as any form of API. I had removed it yesterday. This undocumented "private" function was added with the implementation for PEP 3121 (3.0, 2007) for internal use and later moved out of the limited API (3.6, 2016) and then into the internal API (3.9, 2019). I removed it completely yesterday, including from the stable ABI manifest (where it was added because the symbol happened to be exported). It's unlikely that anyone is using _PyState_AddModule(), especially any stable ABI extensions built against 3.2-3.5, but we're playing it safe. https://github.com/python/cpython/issues/101758 --- Include/internal/pycore_pystate.h | 6 ++++++ Lib/test/test_stable_abi_ctypes.py | 1 + Misc/stable_abi.toml | 3 +++ PC/python3dll.c | 1 + Python/import.c | 20 ++++++++++++++++++++ 5 files changed, 31 insertions(+) diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 638b86253879ea..7046ec8d9adaaf 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -152,6 +152,12 @@ extern void _PySignal_AfterFork(void); #endif +PyAPI_FUNC(int) _PyState_AddModule( + PyThreadState *tstate, + PyObject* module, + PyModuleDef* def); + + PyAPI_FUNC(int) _PyOS_InterruptOccurred(PyThreadState *tstate); #define HEAD_LOCK(runtime) \ diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 7e50fbda2c07cb..e77c1c8409880d 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -864,6 +864,7 @@ def test_windows_feature_macros(self): "_PyObject_GC_Resize", "_PyObject_New", "_PyObject_NewVar", + "_PyState_AddModule", "_PyThreadState_Init", "_PyThreadState_Prealloc", "_PyWeakref_CallableProxyType", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index c04a3a228caf56..21ff9616133445 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -1684,6 +1684,9 @@ [function._PyObject_NewVar] added = '3.2' abi_only = true +[function._PyState_AddModule] + added = '3.2' + abi_only = true [function._PyThreadState_Init] added = '3.2' abi_only = true diff --git a/PC/python3dll.c b/PC/python3dll.c index 79f09037282f54..e300819365756e 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -34,6 +34,7 @@ EXPORT_FUNC(_PyObject_GC_NewVar) EXPORT_FUNC(_PyObject_GC_Resize) EXPORT_FUNC(_PyObject_New) EXPORT_FUNC(_PyObject_NewVar) +EXPORT_FUNC(_PyState_AddModule) EXPORT_FUNC(_PyThreadState_Init) EXPORT_FUNC(_PyThreadState_Prealloc) EXPORT_FUNC(Py_AddPendingCall) diff --git a/Python/import.c b/Python/import.c index ec126f28b85816..fabf03b1c5d698 100644 --- a/Python/import.c +++ b/Python/import.c @@ -487,6 +487,26 @@ PyState_FindModule(PyModuleDef* module) return _modules_by_index_get(interp, module); } +/* _PyState_AddModule() has been completely removed from the C-API + (and was removed from the limited API in 3.6). However, we're + playing it safe and keeping it around for any stable ABI extensions + built against 3.2-3.5. */ +int +_PyState_AddModule(PyThreadState *tstate, PyObject* module, PyModuleDef* def) +{ + if (!def) { + assert(_PyErr_Occurred(tstate)); + return -1; + } + if (def->m_slots) { + _PyErr_SetString(tstate, + PyExc_SystemError, + "PyState_AddModule called on module with slots"); + return -1; + } + return _modules_by_index_set(tstate->interp, def, module); +} + int PyState_AddModule(PyObject* module, PyModuleDef* def) { From 984f8ab018f847fe8d66768af962f69ec0e81849 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 16 Feb 2023 17:21:22 -0700 Subject: [PATCH 27/76] gh-101758: Fix Refleak-Related Failures in test_singlephase_variants (gh-101969) gh-101891 is causing failures under `$> ./python -m test test_imp -R 3:3`. Furthermore, with that fixed, "test_singlephase_variants" is leaking references. This change addresses the first part, but skips the leaking tests until we can follow up with a fix. https://github.com/python/cpython/issues/101758 --- Lib/test/test_imp.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 5997ffad8e1232..2292bb20939599 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -263,6 +263,7 @@ def test_issue16421_multiple_modules_in_one_dll(self): with self.assertRaises(ImportError): imp.load_dynamic('nonexistent', pathname) + @unittest.skip('known refleak (temporarily skipping)') @requires_subinterpreters @requires_load_dynamic def test_singlephase_multiple_interpreters(self): @@ -329,9 +330,10 @@ def clean_up(): # However, globals are still shared. _interpreters.run_string(interp2, script % 2) + @unittest.skip('known refleak (temporarily skipping)') @requires_load_dynamic def test_singlephase_variants(self): - '''Exercise the most meaningful variants described in Python/import.c.''' + # Exercise the most meaningful variants described in Python/import.c. self.maxDiff = None basename = '_testsinglephase' @@ -343,6 +345,11 @@ def clean_up(): _testsinglephase._clear_globals() self.addCleanup(clean_up) + def add_ext_cleanup(name): + def clean_up(): + _testinternalcapi.clear_extension(name, pathname) + self.addCleanup(clean_up) + modules = {} def load(name): assert name not in modules @@ -440,6 +447,7 @@ def check_with_reinit_reloaded(module, lookedup, initialized, # Check the "basic" module. name = basename + add_ext_cleanup(name) expected_init_count = 1 with self.subTest(name): mod = load(name) @@ -457,6 +465,7 @@ def check_with_reinit_reloaded(module, lookedup, initialized, # Check its indirect variants. name = f'{basename}_basic_wrapper' + add_ext_cleanup(name) expected_init_count += 1 with self.subTest(name): mod = load(name) @@ -480,6 +489,7 @@ def check_with_reinit_reloaded(module, lookedup, initialized, # Check its direct variant. name = f'{basename}_basic_copy' + add_ext_cleanup(name) expected_init_count += 1 with self.subTest(name): mod = load(name) @@ -500,6 +510,7 @@ def check_with_reinit_reloaded(module, lookedup, initialized, # Check the non-basic variant that has no state. name = f'{basename}_with_reinit' + add_ext_cleanup(name) with self.subTest(name): mod = load(name) lookedup, initialized, cached = check_common(name, mod) @@ -518,6 +529,7 @@ def check_with_reinit_reloaded(module, lookedup, initialized, # Check the basic variant that has state. name = f'{basename}_with_state' + add_ext_cleanup(name) with self.subTest(name): mod = load(name) lookedup, initialized, cached = check_common(name, mod) From a3bb7fbe7eecfae6bf7b2f0912f9b2b12fac8db1 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Fri, 17 Feb 2023 12:43:07 +0400 Subject: [PATCH 28/76] gh-101973: Fix parameter reference for PyModule_FromDefAndSpec (#101976) --- Doc/c-api/module.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index e2ba157b32c7d9..c0351c8a6c72aa 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -388,7 +388,7 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and .. c:function:: PyObject * PyModule_FromDefAndSpec(PyModuleDef *def, PyObject *spec) - Create a new module object, given the definition in *module* and the + Create a new module object, given the definition in *def* and the ModuleSpec *spec*. This behaves like :c:func:`PyModule_FromDefAndSpec2` with *module_api_version* set to :const:`PYTHON_API_VERSION`. @@ -396,7 +396,7 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and .. c:function:: PyObject * PyModule_FromDefAndSpec2(PyModuleDef *def, PyObject *spec, int module_api_version) - Create a new module object, given the definition in *module* and the + Create a new module object, given the definition in *def* and the ModuleSpec *spec*, assuming the API version *module_api_version*. If that version does not match the version of the running interpreter, a :exc:`RuntimeWarning` is emitted. From 3c0a31cbfd1258bd96153a007dd44a96f2947dbf Mon Sep 17 00:00:00 2001 From: Yeojin Kim Date: Fri, 17 Feb 2023 17:47:02 +0900 Subject: [PATCH 29/76] Docs: fix typos in PyFunction_WatchCallback docs and in 3.12 NEWS (GH-101980) - possitibility => possibility - disaallowed => disallowed --- Doc/c-api/function.rst | 2 +- Misc/NEWS.d/3.12.0a2.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 3cce18bdde3057..bc7569d0add97d 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -169,7 +169,7 @@ There are a few functions specific to Python functions. before the modification to *func* takes place, so the prior state of *func* can be inspected. The runtime is permitted to optimize away the creation of function objects when possible. In such cases no event will be emitted. - Although this creates the possitibility of an observable difference of + Although this creates the possibility of an observable difference of runtime behavior depending on optimization decisions, it does not change the semantics of the Python code being executed. diff --git a/Misc/NEWS.d/3.12.0a2.rst b/Misc/NEWS.d/3.12.0a2.rst index 318f3f71f11546..41ad8cd22b5d89 100644 --- a/Misc/NEWS.d/3.12.0a2.rst +++ b/Misc/NEWS.d/3.12.0a2.rst @@ -1060,7 +1060,7 @@ Add ``getbufferproc`` and ``releasebufferproc`` to the stable API. Some configurable capabilities of sub-interpreters have changed. They always allow subprocesses (:mod:`subprocess`) now, whereas before subprocesses -could be optionally disaallowed for a sub-interpreter. Instead +could be optionally disallowed for a sub-interpreter. Instead :func:`os.exec` can now be disallowed. Disallowing daemon threads is now supported. Disallowing all threads is still allowed, but is never done by default. Note that the optional restrictions are only available through From 775f8819e319127f9bfb046773b74bcc62c68b6a Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Fri, 17 Feb 2023 19:14:07 +0900 Subject: [PATCH 30/76] gh-101766: Fix refleak for _BlockingOnManager resources (gh-101942) --- Lib/importlib/_bootstrap.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index bebe7e15cbce67..1ef7b6adb04434 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -85,6 +85,11 @@ def __enter__(self): def __exit__(self, *args, **kwargs): """Remove self.lock from this thread's _blocking_on list.""" self.blocked_on.remove(self.lock) + if len(self.blocked_on) == 0: + # gh-101766: glboal cache should be cleaned-up + # if there is no more _blocking_on for this thread. + del _blocking_on[self.thread_id] + del self.blocked_on class _DeadlockError(RuntimeError): From d401b20630965c0e1d2a5a0d60d5fc51aa5a8d80 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 17 Feb 2023 14:05:38 +0000 Subject: [PATCH 31/76] gh-101360: Fix anchor matching in pathlib.PureWindowsPath.match() (GH-101363) Use `fnmatch` to match path and pattern anchors, just as we do for other path parts. This allows patterns such as `'*:/Users/*'` to be matched. --- Lib/pathlib.py | 5 ----- Lib/test/test_ntpath.py | 4 ++++ Lib/test/test_pathlib.py | 9 ++++++--- .../2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst | 3 +++ 4 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 17659bcd3e2d7f..d7994a331d5eac 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -647,15 +647,10 @@ def match(self, path_pattern): drv, root, pat_parts = self._parse_parts((path_pattern,)) if not pat_parts: raise ValueError("empty pattern") - elif drv and drv != self._flavour.normcase(self._drv): - return False - elif root and root != self._root: - return False parts = self._parts_normcase if drv or root: if len(pat_parts) != len(parts): return False - pat_parts = pat_parts[1:] elif len(pat_parts) > len(parts): return False for part, pat in zip(reversed(parts), reversed(pat_parts)): diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index b32900697874b1..08c8a7a1f94b95 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -200,6 +200,10 @@ def test_splitroot(self): tester('ntpath.splitroot("//x")', ("//x", "", "")) # non-empty server & missing share tester('ntpath.splitroot("//x/")', ("//x/", "", "")) # non-empty server & empty share + # gh-101363: match GetFullPathNameW() drive letter parsing behaviour + tester('ntpath.splitroot(" :/foo")', (" :", "/", "foo")) + tester('ntpath.splitroot("/:/foo")', ("", "/", ":/foo")) + def test_split(self): tester('ntpath.split("c:\\foo\\bar")', ('c:\\foo', 'bar')) tester('ntpath.split("\\\\conky\\mountpoint\\foo\\bar")', diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index a596795b44f0fa..b8683796d9600d 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -852,8 +852,7 @@ def test_as_uri(self): def test_match_common(self): P = self.cls # Absolute patterns. - self.assertTrue(P('c:/b.py').match('/*.py')) - self.assertTrue(P('c:/b.py').match('c:*.py')) + self.assertTrue(P('c:/b.py').match('*:/*.py')) self.assertTrue(P('c:/b.py').match('c:/*.py')) self.assertFalse(P('d:/b.py').match('c:/*.py')) # wrong drive self.assertFalse(P('b.py').match('/*.py')) @@ -864,7 +863,7 @@ def test_match_common(self): self.assertFalse(P('/b.py').match('c:*.py')) self.assertFalse(P('/b.py').match('c:/*.py')) # UNC patterns. - self.assertTrue(P('//some/share/a.py').match('/*.py')) + self.assertTrue(P('//some/share/a.py').match('//*/*/*.py')) self.assertTrue(P('//some/share/a.py').match('//some/share/*.py')) self.assertFalse(P('//other/share/a.py').match('//some/share/*.py')) self.assertFalse(P('//some/share/a/b.py').match('//some/share/*.py')) @@ -872,6 +871,10 @@ def test_match_common(self): self.assertTrue(P('B.py').match('b.PY')) self.assertTrue(P('c:/a/B.Py').match('C:/A/*.pY')) self.assertTrue(P('//Some/Share/B.Py').match('//somE/sharE/*.pY')) + # Path anchor doesn't match pattern anchor + self.assertFalse(P('c:/b.py').match('/*.py')) # 'c:/' vs '/' + self.assertFalse(P('c:/b.py').match('c:*.py')) # 'c:/' vs 'c:' + self.assertFalse(P('//some/share/a.py').match('/*.py')) # '//some/share/' vs '/' def test_ordering_common(self): # Case-insensitivity. diff --git a/Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst b/Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst new file mode 100644 index 00000000000000..4cfb136c5db853 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst @@ -0,0 +1,3 @@ +Fix anchor matching in :meth:`pathlib.PureWindowsPath.match`. Path and +pattern anchors are now matched with :mod:`fnmatch`, just like other path +parts. This allows patterns such as ``"*:/Users/*"`` to be matched. From 072011b3c38f871cdc3ab62630ea2234d09456d1 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 17 Feb 2023 14:08:14 +0000 Subject: [PATCH 32/76] gh-100809: Fix handling of drive-relative paths in pathlib.Path.absolute() (GH-100812) Resolving the drive independently uses the OS API, which ensures it starts from the current directory on that drive. --- Lib/pathlib.py | 7 +++- Lib/test/support/os_helper.py | 35 +++++++++++++++++++ Lib/test/test_pathlib.py | 20 +++++++++++ ...-01-06-21-14-41.gh-issue-100809.I697UT.rst | 3 ++ 4 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index d7994a331d5eac..dde573592fddce 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -816,7 +816,12 @@ def absolute(self): """ if self.is_absolute(): return self - return self._from_parts([os.getcwd()] + self._parts) + elif self._drv: + # There is a CWD on each drive-letter drive. + cwd = self._flavour.abspath(self._drv) + else: + cwd = os.getcwd() + return self._from_parts([cwd] + self._parts) def resolve(self, strict=False): """ diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 2d4356a1191b1e..821a4b1ffd5077 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -4,6 +4,7 @@ import os import re import stat +import string import sys import time import unittest @@ -716,3 +717,37 @@ def __exit__(self, *ignore_exc): else: self._environ[k] = v os.environ = self._environ + + +try: + import ctypes + kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + + ERROR_FILE_NOT_FOUND = 2 + DDD_REMOVE_DEFINITION = 2 + DDD_EXACT_MATCH_ON_REMOVE = 4 + DDD_NO_BROADCAST_SYSTEM = 8 +except (ImportError, AttributeError): + def subst_drive(path): + raise unittest.SkipTest('ctypes or kernel32 is not available') +else: + @contextlib.contextmanager + def subst_drive(path): + """Temporarily yield a substitute drive for a given path.""" + for c in reversed(string.ascii_uppercase): + drive = f'{c}:' + if (not kernel32.QueryDosDeviceW(drive, None, 0) and + ctypes.get_last_error() == ERROR_FILE_NOT_FOUND): + break + else: + raise unittest.SkipTest('no available logical drive') + if not kernel32.DefineDosDeviceW( + DDD_NO_BROADCAST_SYSTEM, drive, path): + raise ctypes.WinError(ctypes.get_last_error()) + try: + yield drive + finally: + if not kernel32.DefineDosDeviceW( + DDD_REMOVE_DEFINITION | DDD_EXACT_MATCH_ON_REMOVE, + drive, path): + raise ctypes.WinError(ctypes.get_last_error()) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index b8683796d9600d..4de91d52c6d10c 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -2973,6 +2973,26 @@ def test_absolute(self): self.assertEqual(str(P('a', 'b', 'c').absolute()), os.path.join(share, 'a', 'b', 'c')) + drive = os.path.splitdrive(BASE)[0] + with os_helper.change_cwd(BASE): + # Relative path with root + self.assertEqual(str(P('\\').absolute()), drive + '\\') + self.assertEqual(str(P('\\foo').absolute()), drive + '\\foo') + + # Relative path on current drive + self.assertEqual(str(P(drive).absolute()), BASE) + self.assertEqual(str(P(drive + 'foo').absolute()), os.path.join(BASE, 'foo')) + + with os_helper.subst_drive(BASE) as other_drive: + # Set the working directory on the substitute drive + saved_cwd = os.getcwd() + other_cwd = f'{other_drive}\\dirA' + os.chdir(other_cwd) + os.chdir(saved_cwd) + + # Relative path on another drive + self.assertEqual(str(P(other_drive).absolute()), other_cwd) + self.assertEqual(str(P(other_drive + 'foo').absolute()), other_cwd + '\\foo') def test_glob(self): P = self.cls diff --git a/Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst b/Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst new file mode 100644 index 00000000000000..54082de88ccf4a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst @@ -0,0 +1,3 @@ +Fix handling of drive-relative paths (like 'C:' and 'C:foo') in +:meth:`pathlib.Path.absolute`. This method now uses the OS API +to retrieve the correct current working directory for the drive. From f482ade4c7887c49dfd8bba3be76f839e562608d Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sat, 18 Feb 2023 00:18:47 +0900 Subject: [PATCH 33/76] gh-101766: Fix refleak for _BlockingOnManager resources from test suite level (gh-101988) --- Lib/importlib/_bootstrap.py | 5 ----- Lib/test/test_importlib/test_locks.py | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 1ef7b6adb04434..bebe7e15cbce67 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -85,11 +85,6 @@ def __enter__(self): def __exit__(self, *args, **kwargs): """Remove self.lock from this thread's _blocking_on list.""" self.blocked_on.remove(self.lock) - if len(self.blocked_on) == 0: - # gh-101766: glboal cache should be cleaned-up - # if there is no more _blocking_on for this thread. - del _blocking_on[self.thread_id] - del self.blocked_on class _DeadlockError(RuntimeError): diff --git a/Lib/test/test_importlib/test_locks.py b/Lib/test/test_importlib/test_locks.py index 56d73c496e6bbb..ba9cf51c261d52 100644 --- a/Lib/test/test_importlib/test_locks.py +++ b/Lib/test/test_importlib/test_locks.py @@ -33,6 +33,11 @@ class ModuleLockAsRLockTests: test_repr = None test_locked_repr = None + def tearDown(self): + for splitinit in init.values(): + splitinit._bootstrap._blocking_on.clear() + + LOCK_TYPES = {kind: splitinit._bootstrap._ModuleLock for kind, splitinit in init.items()} From a1723caabfcdca5d675c4cb04554fb04c7edf601 Mon Sep 17 00:00:00 2001 From: Dustin Rodrigues Date: Fri, 17 Feb 2023 14:30:29 -0500 Subject: [PATCH 34/76] gh-101992: update plistlib examples to be runnable (#101994) * gh-101992: update plistlib examples to be runnable * Update Doc/library/plistlib.rst --------- Co-authored-by: Terry Jan Reedy --- Doc/library/plistlib.rst | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 5ded9661f08014..7aad15ec91a0ac 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -159,6 +159,9 @@ Examples Generating a plist:: + import datetime + import plistlib + pl = dict( aString = "Doodah", aList = ["A", "B", 12, 32.1, [1, 2, 3]], @@ -172,13 +175,19 @@ Generating a plist:: ), someData = b"", someMoreData = b"" * 10, - aDate = datetime.datetime.fromtimestamp(time.mktime(time.gmtime())), + aDate = datetime.datetime.now() ) - with open(fileName, 'wb') as fp: - dump(pl, fp) + print(plistlib.dumps(pl).decode()) Parsing a plist:: - with open(fileName, 'rb') as fp: - pl = load(fp) - print(pl["aKey"]) + import plistlib + + plist = b""" + + foo + bar + + """ + pl = plistlib.loads(plist) + print(pl["foo"]) From 77d95c83733722ada35eb1ef89ae5b84a51ddd32 Mon Sep 17 00:00:00 2001 From: Jan Gosmann Date: Fri, 17 Feb 2023 22:01:26 +0100 Subject: [PATCH 35/76] gh-100226: Clarify StreamReader.read behavior (#101807) --- Doc/library/asyncio-stream.rst | 12 ++++++++++-- Lib/asyncio/streams.py | 17 +++++++++-------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index 3b3c68ab6ef625..bbac1c32b5695f 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -206,12 +206,20 @@ StreamReader .. coroutinemethod:: read(n=-1) - Read up to *n* bytes. If *n* is not provided, or set to ``-1``, - read until EOF and return all read bytes. + Read up to *n* bytes from the stream. + If *n* is not provided or set to ``-1``, + read until EOF, then return all read :class:`bytes`. If EOF was received and the internal buffer is empty, return an empty ``bytes`` object. + If *n* is ``0``, return an empty ``bytes`` object immediately. + + If *n* is positive, return at most *n* available ``bytes`` + as soon as at least 1 byte is available in the internal buffer. + If EOF is received before any byte is read, return an empty + ``bytes`` object. + .. coroutinemethod:: readline() Read one line, where "line" is a sequence of bytes diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 7d13e961bd2de4..bf15f517e50dce 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -649,16 +649,17 @@ async def readuntil(self, separator=b'\n'): async def read(self, n=-1): """Read up to `n` bytes from the stream. - If n is not provided, or set to -1, read until EOF and return all read - bytes. If the EOF was received and the internal buffer is empty, return - an empty bytes object. + If `n` is not provided or set to -1, + read until EOF, then return all read bytes. + If EOF was received and the internal buffer is empty, + return an empty bytes object. - If n is zero, return empty bytes object immediately. + If `n` is 0, return an empty bytes object immediately. - If n is positive, this function try to read `n` bytes, and may return - less or equal bytes than requested, but at least one byte. If EOF was - received before any byte is read, this function returns empty byte - object. + If `n` is positive, return at most `n` available bytes + as soon as at least 1 byte is available in the internal buffer. + If EOF is received before any byte is read, return an empty + bytes object. Returned value is not limited with limit, configured at stream creation. From 7f1c72175600b21c1c840e8988cc6e6b4b244582 Mon Sep 17 00:00:00 2001 From: Owain Davies <116417456+OTheDev@users.noreply.github.com> Date: Sat, 18 Feb 2023 04:36:47 +0700 Subject: [PATCH 36/76] gh-101739: [Enum] update docs - default boundary for Flag is CONFORM (GH-101746) --- Doc/library/enum.rst | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 13591a1bdc7347..24b6dbfe37cd38 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -696,10 +696,9 @@ Data Types .. attribute:: STRICT - Out-of-range values cause a :exc:`ValueError` to be raised. This is the - default for :class:`Flag`:: + Out-of-range values cause a :exc:`ValueError` to be raised:: - >>> from enum import Flag, STRICT + >>> from enum import Flag, STRICT, auto >>> class StrictFlag(Flag, boundary=STRICT): ... RED = auto() ... GREEN = auto() @@ -715,9 +714,9 @@ Data Types .. attribute:: CONFORM Out-of-range values have invalid values removed, leaving a valid *Flag* - value:: + value. This is the default for :class:`Flag`:: - >>> from enum import Flag, CONFORM + >>> from enum import Flag, CONFORM, auto >>> class ConformFlag(Flag, boundary=CONFORM): ... RED = auto() ... GREEN = auto() @@ -731,7 +730,7 @@ Data Types Out-of-range values lose their *Flag* membership and revert to :class:`int`. This is the default for :class:`IntFlag`:: - >>> from enum import Flag, EJECT + >>> from enum import Flag, EJECT, auto >>> class EjectFlag(Flag, boundary=EJECT): ... RED = auto() ... GREEN = auto() @@ -742,10 +741,10 @@ Data Types .. attribute:: KEEP - Out-of-range values are kept, and the *Flag* membership is kept. This is - used for some stdlib flags: + Out-of-range values are kept, and the *Flag* membership is kept. This is + used for some stdlib flags:: - >>> from enum import Flag, KEEP + >>> from enum import Flag, KEEP, auto >>> class KeepFlag(Flag, boundary=KEEP): ... RED = auto() ... GREEN = auto() From 89413bbccb9261b72190e275eefe4b0d49671477 Mon Sep 17 00:00:00 2001 From: Eclips4 <80244920+Eclips4@users.noreply.github.com> Date: Sat, 18 Feb 2023 03:52:23 +0300 Subject: [PATCH 37/76] gh-101967: add a missing error check (#101968) --- .../2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst | 1 + Python/ceval.c | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst new file mode 100644 index 00000000000000..6e681f910f5359 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst @@ -0,0 +1 @@ +Fix possible segfault in ``positional_only_passed_as_keyword`` function, when new list created. diff --git a/Python/ceval.c b/Python/ceval.c index 09fd2f29266c87..308ef52259df3d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1255,7 +1255,9 @@ positional_only_passed_as_keyword(PyThreadState *tstate, PyCodeObject *co, { int posonly_conflicts = 0; PyObject* posonly_names = PyList_New(0); - + if (posonly_names == NULL) { + goto fail; + } for(int k=0; k < co->co_posonlyargcount; k++){ PyObject* posonly_name = PyTuple_GET_ITEM(co->co_localsplusnames, k); From af446bbb76f64e67831444a0ceee6863a1527088 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 18 Feb 2023 18:46:33 +0300 Subject: [PATCH 38/76] gh-101536: [docs] Improve attributes of `urllib.error.HTTPError` (#101612) * gh-101536: [docs] Improve attributes of `urllib.error.HTTPError` * Address review --- Doc/library/urllib.error.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Doc/library/urllib.error.rst b/Doc/library/urllib.error.rst index f7d47ed76aca18..3adbdd26132273 100644 --- a/Doc/library/urllib.error.rst +++ b/Doc/library/urllib.error.rst @@ -31,7 +31,7 @@ The following exceptions are raised by :mod:`urllib.error` as appropriate: of :exc:`IOError`. -.. exception:: HTTPError +.. exception:: HTTPError(url, code, msg, hdrs, fp) Though being an exception (a subclass of :exc:`URLError`), an :exc:`HTTPError` can also function as a non-exceptional file-like return @@ -39,6 +39,11 @@ The following exceptions are raised by :mod:`urllib.error` as appropriate: is useful when handling exotic HTTP errors, such as requests for authentication. + .. attribute:: url + + Contains the request URL. + An alias for *filename* attribute. + .. attribute:: code An HTTP status code as defined in :rfc:`2616`. This numeric value corresponds @@ -48,14 +53,20 @@ The following exceptions are raised by :mod:`urllib.error` as appropriate: .. attribute:: reason This is usually a string explaining the reason for this error. + An alias for *msg* attribute. .. attribute:: headers The HTTP response headers for the HTTP request that caused the :exc:`HTTPError`. + An alias for *hdrs* attribute. .. versionadded:: 3.4 + .. attribute:: fp + + A file-like object where the HTTP error body can be read from. + .. exception:: ContentTooShortError(msg, content) This exception is raised when the :func:`~urllib.request.urlretrieve` From 128379b8cdb88a6d3d7fed24df082c9a654b3fb8 Mon Sep 17 00:00:00 2001 From: Nicko van Someren Date: Sat, 18 Feb 2023 11:44:41 -0700 Subject: [PATCH 39/76] bpo-46978: Correct docstrings for in-place builtin operators (#31802) --- .../Core and Builtins/2022-03-10-21-48-05.bpo-46978.f5QFfw.rst | 1 + Objects/typeobject.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-03-10-21-48-05.bpo-46978.f5QFfw.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-03-10-21-48-05.bpo-46978.f5QFfw.rst b/Misc/NEWS.d/next/Core and Builtins/2022-03-10-21-48-05.bpo-46978.f5QFfw.rst new file mode 100644 index 00000000000000..72291d042a0394 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-03-10-21-48-05.bpo-46978.f5QFfw.rst @@ -0,0 +1 @@ +Fixed docstrings for in-place operators of built-in types. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index bf6ccdb77a90f0..f2e8092aa37eec 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8564,7 +8564,7 @@ an all-zero entry. #NAME "($self, /)\n--\n\n" DOC) #define IBSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ ETSLOT(NAME, as_number.SLOT, FUNCTION, WRAPPER, \ - #NAME "($self, value, /)\n--\n\nReturn self" DOC "value.") + #NAME "($self, value, /)\n--\n\nCompute self " DOC " value.") #define BINSLOT(NAME, SLOT, FUNCTION, DOC) \ ETSLOT(NAME, as_number.SLOT, FUNCTION, wrap_binaryfunc_l, \ #NAME "($self, value, /)\n--\n\nReturn self" DOC "value.") From 5170caf3059fdacc92d7370eecb9fe4f0c5a1c76 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 18 Feb 2023 16:29:22 -0500 Subject: [PATCH 40/76] gh-97930: Apply changes from importlib_resources 5.12. (GH-102010) --- Lib/importlib/resources/_adapters.py | 4 +- Lib/importlib/resources/_itertools.py | 69 ++++++------ Lib/importlib/resources/readers.py | 36 +++++-- Lib/test/test_importlib/resources/_path.py | 18 ++-- .../subdirectory/subsubdir/resource.txt | 1 + .../resources/test_compatibilty_files.py | 6 +- .../test_importlib/resources/test_custom.py | 46 ++++++++ .../test_importlib/resources/test_files.py | 4 +- .../test_importlib/resources/test_open.py | 14 ++- .../test_importlib/resources/test_path.py | 15 ++- .../test_importlib/resources/test_read.py | 12 ++- .../test_importlib/resources/test_reader.py | 11 ++ .../test_importlib/resources/test_resource.py | 100 ++++++++---------- Lib/test/test_importlib/resources/util.py | 28 +++-- ...3-02-17-19-00-58.gh-issue-97930.C_nQjb.rst | 4 + 15 files changed, 241 insertions(+), 127 deletions(-) create mode 100644 Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt create mode 100644 Lib/test/test_importlib/resources/test_custom.py create mode 100644 Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst diff --git a/Lib/importlib/resources/_adapters.py b/Lib/importlib/resources/_adapters.py index f22f6bc509a3d7..50688fbb666658 100644 --- a/Lib/importlib/resources/_adapters.py +++ b/Lib/importlib/resources/_adapters.py @@ -34,9 +34,7 @@ def _io_wrapper(file, mode='r', *args, **kwargs): return TextIOWrapper(file, *args, **kwargs) elif mode == 'rb': return file - raise ValueError( - f"Invalid mode value '{mode}', only 'r' and 'rb' are supported" - ) + raise ValueError(f"Invalid mode value '{mode}', only 'r' and 'rb' are supported") class CompatibilityFiles: diff --git a/Lib/importlib/resources/_itertools.py b/Lib/importlib/resources/_itertools.py index cce05582ffc6fe..7b775ef5ae893f 100644 --- a/Lib/importlib/resources/_itertools.py +++ b/Lib/importlib/resources/_itertools.py @@ -1,35 +1,38 @@ -from itertools import filterfalse +# from more_itertools 9.0 +def only(iterable, default=None, too_long=None): + """If *iterable* has only one item, return it. + If it has zero items, return *default*. + If it has more than one item, raise the exception given by *too_long*, + which is ``ValueError`` by default. + >>> only([], default='missing') + 'missing' + >>> only([1]) + 1 + >>> only([1, 2]) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ValueError: Expected exactly one item in iterable, but got 1, 2, + and perhaps more.' + >>> only([1, 2], too_long=TypeError) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + TypeError + Note that :func:`only` attempts to advance *iterable* twice to ensure there + is only one item. See :func:`spy` or :func:`peekable` to check + iterable contents less destructively. + """ + it = iter(iterable) + first_value = next(it, default) -from typing import ( - Callable, - Iterable, - Iterator, - Optional, - Set, - TypeVar, - Union, -) - -# Type and type variable definitions -_T = TypeVar('_T') -_U = TypeVar('_U') - - -def unique_everseen( - iterable: Iterable[_T], key: Optional[Callable[[_T], _U]] = None -) -> Iterator[_T]: - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D - seen: Set[Union[_T, _U]] = set() - seen_add = seen.add - if key is None: - for element in filterfalse(seen.__contains__, iterable): - seen_add(element) - yield element + try: + second_value = next(it) + except StopIteration: + pass else: - for element in iterable: - k = key(element) - if k not in seen: - seen_add(k) - yield element + msg = ( + 'Expected exactly one item in iterable, but got {!r}, {!r}, ' + 'and perhaps more.'.format(first_value, second_value) + ) + raise too_long or ValueError(msg) + + return first_value diff --git a/Lib/importlib/resources/readers.py b/Lib/importlib/resources/readers.py index 80cb320dd8bda0..c3cdf769cbecb0 100644 --- a/Lib/importlib/resources/readers.py +++ b/Lib/importlib/resources/readers.py @@ -1,11 +1,12 @@ import collections -import operator +import itertools import pathlib +import operator import zipfile from . import abc -from ._itertools import unique_everseen +from ._itertools import only def remove_duplicates(items): @@ -41,8 +42,10 @@ def open_resource(self, resource): raise FileNotFoundError(exc.args[0]) def is_resource(self, path): - # workaround for `zipfile.Path.is_file` returning true - # for non-existent paths. + """ + Workaround for `zipfile.Path.is_file` returning true + for non-existent paths. + """ target = self.files().joinpath(path) return target.is_file() and target.exists() @@ -67,8 +70,10 @@ def __init__(self, *paths): raise NotADirectoryError('MultiplexedPath only supports directories') def iterdir(self): - files = (file for path in self._paths for file in path.iterdir()) - return unique_everseen(files, key=operator.attrgetter('name')) + children = (child for path in self._paths for child in path.iterdir()) + by_name = operator.attrgetter('name') + groups = itertools.groupby(sorted(children, key=by_name), key=by_name) + return map(self._follow, (locs for name, locs in groups)) def read_bytes(self): raise FileNotFoundError(f'{self} is not a file') @@ -90,6 +95,25 @@ def joinpath(self, *descendants): # Just return something that will not exist. return self._paths[0].joinpath(*descendants) + @classmethod + def _follow(cls, children): + """ + Construct a MultiplexedPath if needed. + + If children contains a sole element, return it. + Otherwise, return a MultiplexedPath of the items. + Unless one of the items is not a Directory, then return the first. + """ + subdirs, one_dir, one_file = itertools.tee(children, 3) + + try: + return only(one_dir) + except ValueError: + try: + return cls(*subdirs) + except NotADirectoryError: + return next(one_file) + def open(self, *args, **kwargs): raise FileNotFoundError(f'{self} is not a file') diff --git a/Lib/test/test_importlib/resources/_path.py b/Lib/test/test_importlib/resources/_path.py index c630e4d3d3f352..1f97c96146960d 100644 --- a/Lib/test/test_importlib/resources/_path.py +++ b/Lib/test/test_importlib/resources/_path.py @@ -1,12 +1,16 @@ import pathlib import functools +from typing import Dict, Union + #### -# from jaraco.path 3.4 +# from jaraco.path 3.4.1 + +FilesSpec = Dict[str, Union[str, bytes, 'FilesSpec']] # type: ignore -def build(spec, prefix=pathlib.Path()): +def build(spec: FilesSpec, prefix=pathlib.Path()): """ Build a set of files/directories, as described by the spec. @@ -23,15 +27,17 @@ def build(spec, prefix=pathlib.Path()): ... "baz.py": "# Some code", ... } ... } - >>> tmpdir = getfixture('tmpdir') - >>> build(spec, tmpdir) + >>> target = getfixture('tmp_path') + >>> build(spec, target) + >>> target.joinpath('foo/baz.py').read_text(encoding='utf-8') + '# Some code' """ for name, contents in spec.items(): create(contents, pathlib.Path(prefix) / name) @functools.singledispatch -def create(content, path): +def create(content: Union[str, bytes, FilesSpec], path): path.mkdir(exist_ok=True) build(content, prefix=path) # type: ignore @@ -43,7 +49,7 @@ def _(content: bytes, path): @create.register def _(content: str, path): - path.write_text(content) + path.write_text(content, encoding='utf-8') # end from jaraco.path diff --git a/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt b/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt new file mode 100644 index 00000000000000..48f587a2d0ac53 --- /dev/null +++ b/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt @@ -0,0 +1 @@ +a resource \ No newline at end of file diff --git a/Lib/test/test_importlib/resources/test_compatibilty_files.py b/Lib/test/test_importlib/resources/test_compatibilty_files.py index 6fa18a24973f64..bcf608d9e2cbdf 100644 --- a/Lib/test/test_importlib/resources/test_compatibilty_files.py +++ b/Lib/test/test_importlib/resources/test_compatibilty_files.py @@ -64,11 +64,13 @@ def test_orphan_path_name(self): def test_spec_path_open(self): self.assertEqual(self.files.read_bytes(), b'Hello, world!') - self.assertEqual(self.files.read_text(), 'Hello, world!') + self.assertEqual(self.files.read_text(encoding='utf-8'), 'Hello, world!') def test_child_path_open(self): self.assertEqual((self.files / 'a').read_bytes(), b'Hello, world!') - self.assertEqual((self.files / 'a').read_text(), 'Hello, world!') + self.assertEqual( + (self.files / 'a').read_text(encoding='utf-8'), 'Hello, world!' + ) def test_orphan_path_open(self): with self.assertRaises(FileNotFoundError): diff --git a/Lib/test/test_importlib/resources/test_custom.py b/Lib/test/test_importlib/resources/test_custom.py new file mode 100644 index 00000000000000..73127209a2761b --- /dev/null +++ b/Lib/test/test_importlib/resources/test_custom.py @@ -0,0 +1,46 @@ +import unittest +import contextlib +import pathlib + +from test.support import os_helper + +from importlib import resources +from importlib.resources.abc import TraversableResources, ResourceReader +from . import util + + +class SimpleLoader: + """ + A simple loader that only implements a resource reader. + """ + + def __init__(self, reader: ResourceReader): + self.reader = reader + + def get_resource_reader(self, package): + return self.reader + + +class MagicResources(TraversableResources): + """ + Magically returns the resources at path. + """ + + def __init__(self, path: pathlib.Path): + self.path = path + + def files(self): + return self.path + + +class CustomTraversableResourcesTests(unittest.TestCase): + def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + + def test_custom_loader(self): + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + loader = SimpleLoader(MagicResources(temp_dir)) + pkg = util.create_package_from_loader(loader) + files = resources.files(pkg) + assert files is temp_dir diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index fe813ae7d08881..1450cfb310926a 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -85,7 +85,7 @@ def test_module_resources(self): _path.build(spec, self.site_dir) import mod - actual = resources.files(mod).joinpath('res.txt').read_text() + actual = resources.files(mod).joinpath('res.txt').read_text(encoding='utf-8') assert actual == spec['res.txt'] @@ -99,7 +99,7 @@ def test_implicit_files(self): '__init__.py': textwrap.dedent( """ import importlib.resources as res - val = res.files().joinpath('res.txt').read_text() + val = res.files().joinpath('res.txt').read_text(encoding='utf-8') """ ), 'res.txt': 'resources are the best', diff --git a/Lib/test/test_importlib/resources/test_open.py b/Lib/test/test_importlib/resources/test_open.py index 0554c41ba67d0e..86becb4bfaad37 100644 --- a/Lib/test/test_importlib/resources/test_open.py +++ b/Lib/test/test_importlib/resources/test_open.py @@ -15,7 +15,7 @@ def execute(self, package, path): class CommonTextTests(util.CommonTests, unittest.TestCase): def execute(self, package, path): target = resources.files(package).joinpath(path) - with target.open(): + with target.open(encoding='utf-8'): pass @@ -28,7 +28,7 @@ def test_open_binary(self): def test_open_text_default_encoding(self): target = resources.files(self.data) / 'utf-8.file' - with target.open() as fp: + with target.open(encoding='utf-8') as fp: result = fp.read() self.assertEqual(result, 'Hello, UTF-8 world!\n') @@ -39,7 +39,9 @@ def test_open_text_given_encoding(self): self.assertEqual(result, 'Hello, UTF-16 world!\n') def test_open_text_with_errors(self): - # Raises UnicodeError without the 'errors' argument. + """ + Raises UnicodeError without the 'errors' argument. + """ target = resources.files(self.data) / 'utf-16.file' with target.open(encoding='utf-8', errors='strict') as fp: self.assertRaises(UnicodeError, fp.read) @@ -54,11 +56,13 @@ def test_open_text_with_errors(self): def test_open_binary_FileNotFoundError(self): target = resources.files(self.data) / 'does-not-exist' - self.assertRaises(FileNotFoundError, target.open, 'rb') + with self.assertRaises(FileNotFoundError): + target.open('rb') def test_open_text_FileNotFoundError(self): target = resources.files(self.data) / 'does-not-exist' - self.assertRaises(FileNotFoundError, target.open) + with self.assertRaises(FileNotFoundError): + target.open(encoding='utf-8') class OpenDiskTests(OpenTests, unittest.TestCase): diff --git a/Lib/test/test_importlib/resources/test_path.py b/Lib/test/test_importlib/resources/test_path.py index adcf75feea78ec..34a6bdd2d58b91 100644 --- a/Lib/test/test_importlib/resources/test_path.py +++ b/Lib/test/test_importlib/resources/test_path.py @@ -14,9 +14,12 @@ def execute(self, package, path): class PathTests: def test_reading(self): - # Path should be readable. - # Test also implicitly verifies the returned object is a pathlib.Path - # instance. + """ + Path should be readable. + + Test also implicitly verifies the returned object is a pathlib.Path + instance. + """ target = resources.files(self.data) / 'utf-8.file' with resources.as_file(target) as path: self.assertTrue(path.name.endswith("utf-8.file"), repr(path)) @@ -51,8 +54,10 @@ def setUp(self): class PathZipTests(PathTests, util.ZipSetup, unittest.TestCase): def test_remove_in_context_manager(self): - # It is not an error if the file that was temporarily stashed on the - # file system is removed inside the `with` stanza. + """ + It is not an error if the file that was temporarily stashed on the + file system is removed inside the `with` stanza. + """ target = resources.files(self.data) / 'utf-8.file' with resources.as_file(target) as path: path.unlink() diff --git a/Lib/test/test_importlib/resources/test_read.py b/Lib/test/test_importlib/resources/test_read.py index 0ca8ee9d02856b..088982681e8b0c 100644 --- a/Lib/test/test_importlib/resources/test_read.py +++ b/Lib/test/test_importlib/resources/test_read.py @@ -12,7 +12,7 @@ def execute(self, package, path): class CommonTextTests(util.CommonTests, unittest.TestCase): def execute(self, package, path): - resources.files(package).joinpath(path).read_text() + resources.files(package).joinpath(path).read_text(encoding='utf-8') class ReadTests: @@ -21,7 +21,11 @@ def test_read_bytes(self): self.assertEqual(result, b'\0\1\2\3') def test_read_text_default_encoding(self): - result = resources.files(self.data).joinpath('utf-8.file').read_text() + result = ( + resources.files(self.data) + .joinpath('utf-8.file') + .read_text(encoding='utf-8') + ) self.assertEqual(result, 'Hello, UTF-8 world!\n') def test_read_text_given_encoding(self): @@ -33,7 +37,9 @@ def test_read_text_given_encoding(self): self.assertEqual(result, 'Hello, UTF-16 world!\n') def test_read_text_with_errors(self): - # Raises UnicodeError without the 'errors' argument. + """ + Raises UnicodeError without the 'errors' argument. + """ target = resources.files(self.data) / 'utf-16.file' self.assertRaises(UnicodeError, target.read_text, encoding='utf-8') result = target.read_text(encoding='utf-8', errors='ignore') diff --git a/Lib/test/test_importlib/resources/test_reader.py b/Lib/test/test_importlib/resources/test_reader.py index 4fd9e6bbe4281c..8670f72a334585 100644 --- a/Lib/test/test_importlib/resources/test_reader.py +++ b/Lib/test/test_importlib/resources/test_reader.py @@ -81,6 +81,17 @@ def test_join_path_compound(self): path = MultiplexedPath(self.folder) assert not path.joinpath('imaginary/foo.py').exists() + def test_join_path_common_subdir(self): + prefix = os.path.abspath(os.path.join(__file__, '..')) + data01 = os.path.join(prefix, 'data01') + data02 = os.path.join(prefix, 'data02') + path = MultiplexedPath(data01, data02) + self.assertIsInstance(path.joinpath('subdirectory'), MultiplexedPath) + self.assertEqual( + str(path.joinpath('subdirectory', 'subsubdir'))[len(prefix) + 1 :], + os.path.join('data02', 'subdirectory', 'subsubdir'), + ) + def test_repr(self): self.assertEqual( repr(MultiplexedPath(self.folder)), diff --git a/Lib/test/test_importlib/resources/test_resource.py b/Lib/test/test_importlib/resources/test_resource.py index f7e3abbdc805a7..6f75cf57f03d02 100644 --- a/Lib/test/test_importlib/resources/test_resource.py +++ b/Lib/test/test_importlib/resources/test_resource.py @@ -1,3 +1,4 @@ +import contextlib import sys import unittest import uuid @@ -7,7 +8,7 @@ from . import zipdata01, zipdata02 from . import util from importlib import resources, import_module -from test.support import import_helper +from test.support import import_helper, os_helper from test.support.os_helper import unlink @@ -69,10 +70,12 @@ def test_resource_missing(self): class ResourceCornerCaseTests(unittest.TestCase): def test_package_has_no_reader_fallback(self): - # Test odd ball packages which: + """ + Test odd ball packages which: # 1. Do not have a ResourceReader as a loader # 2. Are not on the file system # 3. Are not in a zip file + """ module = util.create_package( file=data01, path=data01.__file__, contents=['A', 'B', 'C'] ) @@ -138,82 +141,71 @@ def test_unrelated_contents(self): ) +@contextlib.contextmanager +def zip_on_path(dir): + data_path = pathlib.Path(zipdata01.__file__) + source_zip_path = data_path.parent.joinpath('ziptestdata.zip') + zip_path = pathlib.Path(dir) / f'{uuid.uuid4()}.zip' + zip_path.write_bytes(source_zip_path.read_bytes()) + sys.path.append(str(zip_path)) + import_module('ziptestdata') + + try: + yield + finally: + with contextlib.suppress(ValueError): + sys.path.remove(str(zip_path)) + + with contextlib.suppress(KeyError): + del sys.path_importer_cache[str(zip_path)] + del sys.modules['ziptestdata'] + + with contextlib.suppress(OSError): + unlink(zip_path) + + class DeletingZipsTest(unittest.TestCase): """Having accessed resources in a zip file should not keep an open reference to the zip. """ - ZIP_MODULE = zipdata01 - def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + modules = import_helper.modules_setup() self.addCleanup(import_helper.modules_cleanup, *modules) - data_path = pathlib.Path(self.ZIP_MODULE.__file__) - data_dir = data_path.parent - self.source_zip_path = data_dir / 'ziptestdata.zip' - self.zip_path = pathlib.Path(f'{uuid.uuid4()}.zip').absolute() - self.zip_path.write_bytes(self.source_zip_path.read_bytes()) - sys.path.append(str(self.zip_path)) - self.data = import_module('ziptestdata') - - def tearDown(self): - try: - sys.path.remove(str(self.zip_path)) - except ValueError: - pass - - try: - del sys.path_importer_cache[str(self.zip_path)] - del sys.modules[self.data.__name__] - except KeyError: - pass - - try: - unlink(self.zip_path) - except OSError: - # If the test fails, this will probably fail too - pass + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + self.fixtures.enter_context(zip_on_path(temp_dir)) def test_iterdir_does_not_keep_open(self): - c = [item.name for item in resources.files('ziptestdata').iterdir()] - self.zip_path.unlink() - del c + [item.name for item in resources.files('ziptestdata').iterdir()] def test_is_file_does_not_keep_open(self): - c = resources.files('ziptestdata').joinpath('binary.file').is_file() - self.zip_path.unlink() - del c + resources.files('ziptestdata').joinpath('binary.file').is_file() def test_is_file_failure_does_not_keep_open(self): - c = resources.files('ziptestdata').joinpath('not-present').is_file() - self.zip_path.unlink() - del c + resources.files('ziptestdata').joinpath('not-present').is_file() @unittest.skip("Desired but not supported.") def test_as_file_does_not_keep_open(self): # pragma: no cover - c = resources.as_file(resources.files('ziptestdata') / 'binary.file') - self.zip_path.unlink() - del c + resources.as_file(resources.files('ziptestdata') / 'binary.file') def test_entered_path_does_not_keep_open(self): - # This is what certifi does on import to make its bundle - # available for the process duration. - c = resources.as_file( - resources.files('ziptestdata') / 'binary.file' - ).__enter__() - self.zip_path.unlink() - del c + """ + Mimic what certifi does on import to make its bundle + available for the process duration. + """ + resources.as_file(resources.files('ziptestdata') / 'binary.file').__enter__() def test_read_binary_does_not_keep_open(self): - c = resources.files('ziptestdata').joinpath('binary.file').read_bytes() - self.zip_path.unlink() - del c + resources.files('ziptestdata').joinpath('binary.file').read_bytes() def test_read_text_does_not_keep_open(self): - c = resources.files('ziptestdata').joinpath('utf-8.file').read_text() - self.zip_path.unlink() - del c + resources.files('ziptestdata').joinpath('utf-8.file').read_text( + encoding='utf-8' + ) class ResourceFromNamespaceTest01(unittest.TestCase): diff --git a/Lib/test/test_importlib/resources/util.py b/Lib/test/test_importlib/resources/util.py index 1e72b91ff6b31f..dbe6ee81476699 100644 --- a/Lib/test/test_importlib/resources/util.py +++ b/Lib/test/test_importlib/resources/util.py @@ -80,32 +80,44 @@ def execute(self, package, path): """ def test_package_name(self): - # Passing in the package name should succeed. + """ + Passing in the package name should succeed. + """ self.execute(data01.__name__, 'utf-8.file') def test_package_object(self): - # Passing in the package itself should succeed. + """ + Passing in the package itself should succeed. + """ self.execute(data01, 'utf-8.file') def test_string_path(self): - # Passing in a string for the path should succeed. + """ + Passing in a string for the path should succeed. + """ path = 'utf-8.file' self.execute(data01, path) def test_pathlib_path(self): - # Passing in a pathlib.PurePath object for the path should succeed. + """ + Passing in a pathlib.PurePath object for the path should succeed. + """ path = pathlib.PurePath('utf-8.file') self.execute(data01, path) def test_importing_module_as_side_effect(self): - # The anchor package can already be imported. + """ + The anchor package can already be imported. + """ del sys.modules[data01.__name__] self.execute(data01.__name__, 'utf-8.file') def test_missing_path(self): - # Attempting to open or read or request the path for a - # non-existent path should succeed if open_resource - # can return a viable data stream. + """ + Attempting to open or read or request the path for a + non-existent path should succeed if open_resource + can return a viable data stream. + """ bytes_data = io.BytesIO(b'Hello, world!') package = create_package(file=bytes_data, path=FileNotFoundError()) self.execute(package, 'utf-8.file') diff --git a/Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst b/Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst new file mode 100644 index 00000000000000..967e13f752bcd1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst @@ -0,0 +1,4 @@ +Apply changes from `importlib_resources 5.12 +`_, +including fix for ``MultiplexedPath`` to support directories in multiple +namespaces (python/importlib_resources#265). From 61f1e67c6fcbf80eb9be2b75f7d62954e28c89e6 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sun, 19 Feb 2023 00:22:02 +0000 Subject: [PATCH 41/76] GH-84783: Make the slice object hashable (GH-101264) --- Doc/library/functions.rst | 3 ++ Lib/test/test_capi/test_misc.py | 7 +--- Lib/test/test_doctest.py | 2 +- Lib/test/test_slice.py | 12 ++++-- ...3-02-11-23-14-06.gh-issue-84783._P5sMa.rst | 1 + Objects/sliceobject.c | 38 ++++++++++++++++++- Objects/tupleobject.c | 2 +- 7 files changed, 53 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 658d6768457d16..3ff28849025153 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1635,6 +1635,9 @@ are always available. They are listed here in alphabetical order. example: ``a[start:stop:step]`` or ``a[start:stop, i]``. See :func:`itertools.islice` for an alternate version that returns an iterator. + .. versionchanged:: 3.12 + Slice objects are now :term:`hashable` (provided :attr:`~slice.start`, + :attr:`~slice.stop`, and :attr:`~slice.step` are hashable). .. function:: sorted(iterable, /, *, key=None, reverse=False) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index f26b4723d1e68b..ad099c61463b66 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -419,11 +419,6 @@ def __setitem__(self, index, value): with self.assertRaises(TypeError): _testcapi.sequence_set_slice(None, 1, 3, 'xy') - mapping = {1: 'a', 2: 'b', 3: 'c'} - with self.assertRaises(TypeError): - _testcapi.sequence_set_slice(mapping, 1, 3, 'xy') - self.assertEqual(mapping, {1: 'a', 2: 'b', 3: 'c'}) - def test_sequence_del_slice(self): # Correct case: data = [1, 2, 3, 4, 5] @@ -459,7 +454,7 @@ def __delitem__(self, index): _testcapi.sequence_del_slice(None, 1, 3) mapping = {1: 'a', 2: 'b', 3: 'c'} - with self.assertRaises(TypeError): + with self.assertRaises(KeyError): _testcapi.sequence_del_slice(mapping, 1, 3) self.assertEqual(mapping, {1: 'a', 2: 'b', 3: 'c'}) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 65e215f1cdda4a..3491d4cdb1c18b 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -707,7 +707,7 @@ def non_Python_modules(): r""" >>> import builtins >>> tests = doctest.DocTestFinder().find(builtins) - >>> 825 < len(tests) < 845 # approximate number of objects with docstrings + >>> 830 < len(tests) < 850 # approximate number of objects with docstrings True >>> real_tests = [t for t in tests if len(t.examples) > 0] >>> len(real_tests) # objects that actually have doctests diff --git a/Lib/test/test_slice.py b/Lib/test/test_slice.py index 03fde3275e1475..c35a2293f790a2 100644 --- a/Lib/test/test_slice.py +++ b/Lib/test/test_slice.py @@ -80,10 +80,16 @@ def test_repr(self): self.assertEqual(repr(slice(1, 2, 3)), "slice(1, 2, 3)") def test_hash(self): - # Verify clearing of SF bug #800796 - self.assertRaises(TypeError, hash, slice(5)) + self.assertEqual(hash(slice(5)), slice(5).__hash__()) + self.assertEqual(hash(slice(1, 2)), slice(1, 2).__hash__()) + self.assertEqual(hash(slice(1, 2, 3)), slice(1, 2, 3).__hash__()) + self.assertNotEqual(slice(5), slice(6)) + + with self.assertRaises(TypeError): + hash(slice(1, 2, [])) + with self.assertRaises(TypeError): - slice(5).__hash__() + hash(slice(4, {})) def test_cmp(self): s1 = slice(1, 2, 3) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst new file mode 100644 index 00000000000000..e1c851a0825a7f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst @@ -0,0 +1 @@ +Make the slice object hashable. diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 5694bd9c661fa5..5d2e6ad522bcf2 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -628,6 +628,42 @@ slice_traverse(PySliceObject *v, visitproc visit, void *arg) return 0; } +/* code based on tuplehash() of Objects/tupleobject.c */ +#if SIZEOF_PY_UHASH_T > 4 +#define _PyHASH_XXPRIME_1 ((Py_uhash_t)11400714785074694791ULL) +#define _PyHASH_XXPRIME_2 ((Py_uhash_t)14029467366897019727ULL) +#define _PyHASH_XXPRIME_5 ((Py_uhash_t)2870177450012600261ULL) +#define _PyHASH_XXROTATE(x) ((x << 31) | (x >> 33)) /* Rotate left 31 bits */ +#else +#define _PyHASH_XXPRIME_1 ((Py_uhash_t)2654435761UL) +#define _PyHASH_XXPRIME_2 ((Py_uhash_t)2246822519UL) +#define _PyHASH_XXPRIME_5 ((Py_uhash_t)374761393UL) +#define _PyHASH_XXROTATE(x) ((x << 13) | (x >> 19)) /* Rotate left 13 bits */ +#endif + +static Py_hash_t +slicehash(PySliceObject *v) +{ + Py_uhash_t acc = _PyHASH_XXPRIME_5; +#define _PyHASH_SLICE_PART(com) { \ + Py_uhash_t lane = PyObject_Hash(v->com); \ + if(lane == (Py_uhash_t)-1) { \ + return -1; \ + } \ + acc += lane * _PyHASH_XXPRIME_2; \ + acc = _PyHASH_XXROTATE(acc); \ + acc *= _PyHASH_XXPRIME_1; \ +} + _PyHASH_SLICE_PART(start); + _PyHASH_SLICE_PART(stop); + _PyHASH_SLICE_PART(step); +#undef _PyHASH_SLICE_PART + if(acc == (Py_uhash_t)-1) { + return 1546275796; + } + return acc; +} + PyTypeObject PySlice_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "slice", /* Name of this type */ @@ -642,7 +678,7 @@ PyTypeObject PySlice_Type = { 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - PyObject_HashNotImplemented, /* tp_hash */ + (hashfunc)slicehash, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index e1b9953226c0d7..7d6d0e1bea249e 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -288,7 +288,7 @@ tuplerepr(PyTupleObject *v) /* Hash for tuples. This is a slightly simplified version of the xxHash non-cryptographic hash: - - we do not use any parallellism, there is only 1 accumulator. + - we do not use any parallelism, there is only 1 accumulator. - we drop the final mixing since this is just a permutation of the output space: it does not help against collisions. - at the end, we mangle the length with a single constant. From 36b670908b3546f46283aae4dbf311e53289f3d1 Mon Sep 17 00:00:00 2001 From: Reza Rastak Date: Sat, 18 Feb 2023 19:55:43 -0500 Subject: [PATCH 42/76] Fix incorrectly documented attribute in csv docs (#101250) --- Doc/library/csv.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 41f11505aa1030..f1776554d8b9f2 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -458,7 +458,7 @@ Reader objects have the following public attributes: DictReader objects have the following public attribute: -.. attribute:: csvreader.fieldnames +.. attribute:: DictReader.fieldnames If not passed as a parameter when creating the object, this attribute is initialized upon first access or when the first record is read from the From 6aab56f3c2ee331116eba242d2fcdca592577328 Mon Sep 17 00:00:00 2001 From: Patricio Paez Date: Sat, 18 Feb 2023 19:06:03 -0600 Subject: [PATCH 43/76] gh-99735: Use required=True in argparse subparsers example (#100927) --- Doc/library/argparse.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index dbaa5d0d9b995b..34b4c61649b99f 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1867,7 +1867,7 @@ Sub-commands ... >>> # create the top-level parser >>> parser = argparse.ArgumentParser() - >>> subparsers = parser.add_subparsers() + >>> subparsers = parser.add_subparsers(required=True) >>> >>> # create the parser for the "foo" command >>> parser_foo = subparsers.add_parser('foo') From 072935951f7cd44b40ee37fe561478b2e431c2fb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 18 Feb 2023 21:32:50 -0500 Subject: [PATCH 44/76] gh-97930: Also include subdirectory in makefile. (#102030) --- Makefile.pre.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile.pre.in b/Makefile.pre.in index 490483a712014c..b28c6067a535b8 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2075,6 +2075,8 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_importlib/resources/data01/subdirectory \ test/test_importlib/resources/data02 \ test/test_importlib/resources/data02/one \ + test/test_importlib/resources/data02/subdirectory \ + test/test_importlib/resources/data02/subdirectory/subsubdir \ test/test_importlib/resources/data02/two \ test/test_importlib/resources/data03 \ test/test_importlib/resources/data03/namespace \ From 9a07eff628c1cd88b7cdda88a8fd0db3fe7ea552 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Amiel Date: Sun, 19 Feb 2023 11:18:12 +0100 Subject: [PATCH 45/76] gh-100210: Correct the comment link for unescaping HTML (#100212) gh-100210: correct the comment link for unescaping HTML --- Lib/html/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/html/__init__.py b/Lib/html/__init__.py index da0a0a3ce70eed..1543460ca33b0a 100644 --- a/Lib/html/__init__.py +++ b/Lib/html/__init__.py @@ -25,7 +25,7 @@ def escape(s, quote=True): return s -# see http://www.w3.org/TR/html5/syntax.html#tokenizing-character-references +# see https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state _invalid_charrefs = { 0x00: '\ufffd', # REPLACEMENT CHARACTER From 71f614ef2a3d66213b9cae807cbbc1ed03741221 Mon Sep 17 00:00:00 2001 From: Owain Davies <116417456+OTheDev@users.noreply.github.com> Date: Sun, 19 Feb 2023 22:00:59 +0700 Subject: [PATCH 46/76] Add missing 'is' to `cmath.log()` docstring (#102049) Fix missing 'is' in cmath.log() docstring --- Modules/clinic/cmathmodule.c.h | 4 ++-- Modules/cmathmodule.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/clinic/cmathmodule.c.h b/Modules/clinic/cmathmodule.c.h index b1da9452c61db8..941448e76e80de 100644 --- a/Modules/clinic/cmathmodule.c.h +++ b/Modules/clinic/cmathmodule.c.h @@ -644,7 +644,7 @@ PyDoc_STRVAR(cmath_log__doc__, "\n" "log(z[, base]) -> the logarithm of z to the given base.\n" "\n" -"If the base not specified, returns the natural logarithm (base e) of z."); +"If the base is not specified, returns the natural logarithm (base e) of z."); #define CMATH_LOG_METHODDEF \ {"log", _PyCFunction_CAST(cmath_log), METH_FASTCALL, cmath_log__doc__}, @@ -982,4 +982,4 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=0146c656e67f5d5f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=87f609786ef270cd input=a9049054013a1b77]*/ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 2038ac26e65857..53e34061d53773 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -957,12 +957,12 @@ cmath.log log(z[, base]) -> the logarithm of z to the given base. -If the base not specified, returns the natural logarithm (base e) of z. +If the base is not specified, returns the natural logarithm (base e) of z. [clinic start generated code]*/ static PyObject * cmath_log_impl(PyObject *module, Py_complex x, PyObject *y_obj) -/*[clinic end generated code: output=4effdb7d258e0d94 input=230ed3a71ecd000a]*/ +/*[clinic end generated code: output=4effdb7d258e0d94 input=e1f81d4fcfd26497]*/ { Py_complex y; From 32df540635cacce1053ee0ef98ee23f3f6a43c02 Mon Sep 17 00:00:00 2001 From: neuralstring <107343209+neuralstring@users.noreply.github.com> Date: Sun, 19 Feb 2023 16:39:03 +0000 Subject: [PATCH 47/76] gh-100425: Update tutorial docs related to sum() accuracy (FH-101854) --- Doc/tutorial/stdlib2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 0c101c1f207235..33f311db3a24d2 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -394,7 +394,7 @@ point:: >>> sum([Decimal('0.1')]*10) == Decimal('1.0') True - >>> sum([0.1]*10) == 1.0 + >>> 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 == 1.0 False The :mod:`decimal` module provides arithmetic with as much precision as needed:: From b513c46d998344dc07eb6d510782c2e23d2b859e Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Sun, 19 Feb 2023 19:15:44 +0000 Subject: [PATCH 48/76] gh-85417: Clarify behaviour on branch cuts in cmath module (#102046) This PR updates the cmath module documentation to reflect the reality that Python is almost always (and as far as I can tell, that "almost" can be omitted) running on a machine whose C double supports signed zeros. * Removes misleading references to functions being continuous from above / below / the left / the right at branch cuts * Expands the note on branch cuts at the top of the module documentation to explain the double-sided sign-of-zero-based behaviour --- Doc/library/cmath.rst | 66 +++++++++++-------- ...3-02-19-10-33-01.gh-issue-85417.kYO8u3.rst | 1 + 2 files changed, 39 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index 28cd96b0e12da9..5ed7a09b3e9db2 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -15,11 +15,27 @@ the function is then applied to the result of the conversion. .. note:: - On platforms with hardware and system-level support for signed - zeros, functions involving branch cuts are continuous on *both* - sides of the branch cut: the sign of the zero distinguishes one - side of the branch cut from the other. On platforms that do not - support signed zeros the continuity is as specified below. + For functions involving branch cuts, we have the problem of deciding how to + define those functions on the cut itself. Following Kahan's "Branch cuts for + complex elementary functions" paper, as well as Annex G of C99 and later C + standards, we use the sign of zero to distinguish one side of the branch cut + from the other: for a branch cut along (a portion of) the real axis we look + at the sign of the imaginary part, while for a branch cut along the + imaginary axis we look at the sign of the real part. + + For example, the :func:`cmath.sqrt` function has a branch cut along the + negative real axis. An argument of ``complex(-2.0, -0.0)`` is treated as + though it lies *below* the branch cut, and so gives a result on the negative + imaginary axis:: + + >>> cmath.sqrt(complex(-2.0, -0.0)) + -1.4142135623730951j + + But an argument of ``complex(-2.0, 0.0)`` is treated as though it lies above + the branch cut:: + + >>> cmath.sqrt(complex(-2.0, 0.0)) + 1.4142135623730951j Conversions to and from polar coordinates @@ -44,14 +60,11 @@ rectangular coordinates to polar coordinates and back. .. function:: phase(x) - Return the phase of *x* (also known as the *argument* of *x*), as a - float. ``phase(x)`` is equivalent to ``math.atan2(x.imag, - x.real)``. The result lies in the range [-\ *π*, *π*], and the branch - cut for this operation lies along the negative real axis, - continuous from above. On systems with support for signed zeros - (which includes most systems in current use), this means that the - sign of the result is the same as the sign of ``x.imag``, even when - ``x.imag`` is zero:: + Return the phase of *x* (also known as the *argument* of *x*), as a float. + ``phase(x)`` is equivalent to ``math.atan2(x.imag, x.real)``. The result + lies in the range [-\ *π*, *π*], and the branch cut for this operation lies + along the negative real axis. The sign of the result is the same as the + sign of ``x.imag``, even when ``x.imag`` is zero:: >>> phase(complex(-1.0, 0.0)) 3.141592653589793 @@ -92,8 +105,8 @@ Power and logarithmic functions .. function:: log(x[, base]) Returns the logarithm of *x* to the given *base*. If the *base* is not - specified, returns the natural logarithm of *x*. There is one branch cut, from 0 - along the negative real axis to -∞, continuous from above. + specified, returns the natural logarithm of *x*. There is one branch cut, + from 0 along the negative real axis to -∞. .. function:: log10(x) @@ -112,9 +125,9 @@ Trigonometric functions .. function:: acos(x) - Return the arc cosine of *x*. There are two branch cuts: One extends right from - 1 along the real axis to ∞, continuous from below. The other extends left from - -1 along the real axis to -∞, continuous from above. + Return the arc cosine of *x*. There are two branch cuts: One extends right + from 1 along the real axis to ∞. The other extends left from -1 along the + real axis to -∞. .. function:: asin(x) @@ -125,9 +138,8 @@ Trigonometric functions .. function:: atan(x) Return the arc tangent of *x*. There are two branch cuts: One extends from - ``1j`` along the imaginary axis to ``∞j``, continuous from the right. The - other extends from ``-1j`` along the imaginary axis to ``-∞j``, continuous - from the left. + ``1j`` along the imaginary axis to ``∞j``. The other extends from ``-1j`` + along the imaginary axis to ``-∞j``. .. function:: cos(x) @@ -151,23 +163,21 @@ Hyperbolic functions .. function:: acosh(x) Return the inverse hyperbolic cosine of *x*. There is one branch cut, - extending left from 1 along the real axis to -∞, continuous from above. + extending left from 1 along the real axis to -∞. .. function:: asinh(x) Return the inverse hyperbolic sine of *x*. There are two branch cuts: - One extends from ``1j`` along the imaginary axis to ``∞j``, - continuous from the right. The other extends from ``-1j`` along - the imaginary axis to ``-∞j``, continuous from the left. + One extends from ``1j`` along the imaginary axis to ``∞j``. The other + extends from ``-1j`` along the imaginary axis to ``-∞j``. .. function:: atanh(x) Return the inverse hyperbolic tangent of *x*. There are two branch cuts: One - extends from ``1`` along the real axis to ``∞``, continuous from below. The - other extends from ``-1`` along the real axis to ``-∞``, continuous from - above. + extends from ``1`` along the real axis to ``∞``. The other extends from + ``-1`` along the real axis to ``-∞``. .. function:: cosh(x) diff --git a/Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst b/Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst new file mode 100644 index 00000000000000..a5532df14795d2 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst @@ -0,0 +1 @@ +Update :mod:`cmath` documentation to clarify behaviour on branch cuts. From 3b264df470c82d77f5b01c6f9d1d7173d1cb9597 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 19 Feb 2023 13:21:37 -0600 Subject: [PATCH 49/76] Misc improvements to the float tutorial (GH-102052) --- Doc/tutorial/floatingpoint.rst | 139 +++++++++++++++++++++++++-------- 1 file changed, 106 insertions(+), 33 deletions(-) diff --git a/Doc/tutorial/floatingpoint.rst b/Doc/tutorial/floatingpoint.rst index cedade6e336608..306b1eba3c45b8 100644 --- a/Doc/tutorial/floatingpoint.rst +++ b/Doc/tutorial/floatingpoint.rst @@ -1,6 +1,7 @@ .. testsetup:: import math + from fractions import Fraction .. _tut-fp-issues: @@ -9,12 +10,13 @@ Floating Point Arithmetic: Issues and Limitations ************************************************** .. sectionauthor:: Tim Peters +.. sectionauthor:: Raymond Hettinger Floating-point numbers are represented in computer hardware as base 2 (binary) -fractions. For example, the **decimal** fraction ``0.125`` -has value 1/10 + 2/100 + 5/1000, and in the same way the **binary** fraction ``0.001`` -has value 0/2 + 0/4 + 1/8. These two fractions have identical values, the only +fractions. For example, the **decimal** fraction ``0.625`` +has value 6/10 + 2/100 + 5/1000, and in the same way the **binary** fraction ``0.101`` +has value 1/2 + 0/4 + 1/8. These two fractions have identical values, the only real difference being that the first is written in base 10 fractional notation, and the second in base 2. @@ -57,13 +59,15 @@ Many users are not aware of the approximation because of the way values are displayed. Python only prints a decimal approximation to the true decimal value of the binary approximation stored by the machine. On most machines, if Python were to print the true decimal value of the binary approximation stored -for 0.1, it would have to display :: +for 0.1, it would have to display:: >>> 0.1 0.1000000000000000055511151231257827021181583404541015625 That is more digits than most people find useful, so Python keeps the number -of digits manageable by displaying a rounded value instead :: +of digits manageable by displaying a rounded value instead: + +.. doctest:: >>> 1 / 10 0.1 @@ -90,7 +94,10 @@ thing in all languages that support your hardware's floating-point arithmetic (although some languages may not *display* the difference by default, or in all output modes). -For more pleasant output, you may wish to use string formatting to produce a limited number of significant digits:: +For more pleasant output, you may wish to use string formatting to produce a +limited number of significant digits: + +.. doctest:: >>> format(math.pi, '.12g') # give 12 significant digits '3.14159265359' @@ -101,33 +108,49 @@ For more pleasant output, you may wish to use string formatting to produce a lim >>> repr(math.pi) '3.141592653589793' - It's important to realize that this is, in a real sense, an illusion: you're simply rounding the *display* of the true machine value. One illusion may beget another. For example, since 0.1 is not exactly 1/10, -summing three values of 0.1 may not yield exactly 0.3, either:: +summing three values of 0.1 may not yield exactly 0.3, either: + +.. doctest:: - >>> .1 + .1 + .1 == .3 + >>> 0.1 + 0.1 + 0.1 == 0.3 False Also, since the 0.1 cannot get any closer to the exact value of 1/10 and 0.3 cannot get any closer to the exact value of 3/10, then pre-rounding with -:func:`round` function cannot help:: +:func:`round` function cannot help: - >>> round(.1, 1) + round(.1, 1) + round(.1, 1) == round(.3, 1) +.. doctest:: + + >>> round(0.1, 1) + round(0.1, 1) + round(0.1, 1) == round(0.3, 1) False Though the numbers cannot be made closer to their intended exact values, -the :func:`round` function can be useful for post-rounding so that results -with inexact values become comparable to one another:: +the :func:`math.isclose` function can be useful for comparing inexact values: - >>> round(.1 + .1 + .1, 10) == round(.3, 10) - True +.. doctest:: + + >>> math.isclose(0.1 + 0.1 + 0.1, 0.3) + True + +Alternatively, the :func:`round` function can be used to compare rough +approximations:: + +.. doctest:: + + >>> round(math.pi, ndigits=2) == round(22 / 7, ndigits=2) + True Binary floating-point arithmetic holds many surprises like this. The problem with "0.1" is explained in precise detail below, in the "Representation Error" -section. See `The Perils of Floating Point `_ +section. See `Examples of Floating Point Problems +`_ for +a pleasant summary of how binary floating point works and the kinds of +problems commonly encountered in practice. Also see +`The Perils of Floating Point `_ for a more complete account of other common surprises. As that says near the end, "there are no easy answers." Still, don't be unduly @@ -158,26 +181,34 @@ statistical operations supplied by the SciPy project. See . Python provides tools that may help on those rare occasions when you really *do* want to know the exact value of a float. The :meth:`float.as_integer_ratio` method expresses the value of a float as a -fraction:: +fraction: + +.. doctest:: >>> x = 3.14159 >>> x.as_integer_ratio() (3537115888337719, 1125899906842624) Since the ratio is exact, it can be used to losslessly recreate the -original value:: +original value: + +.. doctest:: >>> x == 3537115888337719 / 1125899906842624 True The :meth:`float.hex` method expresses a float in hexadecimal (base -16), again giving the exact value stored by your computer:: +16), again giving the exact value stored by your computer: + +.. doctest:: >>> x.hex() '0x1.921f9f01b866ep+1' This precise hexadecimal representation can be used to reconstruct -the float value exactly:: +the float value exactly: + +.. doctest:: >>> x == float.fromhex('0x1.921f9f01b866ep+1') True @@ -186,17 +217,43 @@ Since the representation is exact, it is useful for reliably porting values across different versions of Python (platform independence) and exchanging data with other languages that support the same format (such as Java and C99). -Another helpful tool is the :func:`math.fsum` function which helps mitigate -loss-of-precision during summation. It tracks "lost digits" as values are -added onto a running total. That can make a difference in overall accuracy -so that the errors do not accumulate to the point where they affect the -final total: +Another helpful tool is the :func:`sum` function which helps mitigate +loss-of-precision during summation. It uses extended precision for +intermediate rounding steps as values are added onto a running total. +That can make a difference in overall accuracy so that the errors do not +accumulate to the point where they affect the final total: + +.. doctest:: >>> 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 == 1.0 False - >>> math.fsum([0.1] * 10) == 1.0 + >>> sum([0.1] * 10) == 1.0 True +The :func:`math.fsum()` goes further and tracks all of the "lost digits" +as values are added onto a running total so that the result has only a +single rounding. This is slower than :func:`sum` but will be more +accurate in uncommon cases where large magnitude inputs mostly cancel +each other out leaving a final sum near zero: + +.. doctest:: + + >>> arr = [-0.10430216751806065, -266310978.67179024, 143401161448607.16, + ... -143401161400469.7, 266262841.31058735, -0.003244936839808227] + >>> float(sum(map(Fraction, arr))) # Exact summation with single rounding + 8.042173697819788e-13 + >>> math.fsum(arr) # Single rounding + 8.042173697819788e-13 + >>> sum(arr) # Multiple roundings in extended precision + 8.042178034628478e-13 + >>> total = 0.0 + >>> for x in arr: + ... total += x # Multiple roundings in standard precision + ... + >>> total # Straight addition has no correct digits! + -0.0051575902860057365 + + .. _tut-fp-error: Representation Error @@ -225,20 +282,28 @@ as :: J ~= 2**N / 10 and recalling that *J* has exactly 53 bits (is ``>= 2**52`` but ``< 2**53``), -the best value for *N* is 56:: +the best value for *N* is 56: + +.. doctest:: >>> 2**52 <= 2**56 // 10 < 2**53 True That is, 56 is the only value for *N* that leaves *J* with exactly 53 bits. The -best possible value for *J* is then that quotient rounded:: +best possible value for *J* is then that quotient rounded: + +.. doctest:: >>> q, r = divmod(2**56, 10) >>> r 6 Since the remainder is more than half of 10, the best approximation is obtained -by rounding up:: +by rounding up: + +.. doctest:: + + >>> q+1 7205759403792794 @@ -256,13 +321,17 @@ if we had not rounded up, the quotient would have been a little bit smaller than 1/10. But in no case can it be *exactly* 1/10! So the computer never "sees" 1/10: what it sees is the exact fraction given -above, the best 754 double approximation it can get:: +above, the best 754 double approximation it can get: + +.. doctest:: >>> 0.1 * 2 ** 55 3602879701896397.0 If we multiply that fraction by 10\*\*55, we can see the value out to -55 decimal digits:: +55 decimal digits: + +.. doctest:: >>> 3602879701896397 * 10 ** 55 // 2 ** 55 1000000000000000055511151231257827021181583404541015625 @@ -270,13 +339,17 @@ If we multiply that fraction by 10\*\*55, we can see the value out to meaning that the exact number stored in the computer is equal to the decimal value 0.1000000000000000055511151231257827021181583404541015625. Instead of displaying the full decimal value, many languages (including -older versions of Python), round the result to 17 significant digits:: +older versions of Python), round the result to 17 significant digits: + +.. doctest:: >>> format(0.1, '.17f') '0.10000000000000001' The :mod:`fractions` and :mod:`decimal` modules make these calculations -easy:: +easy: + +.. doctest:: >>> from decimal import Decimal >>> from fractions import Fraction From 60bbed7f174e481d3fc69984430a4667951678b3 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 19 Feb 2023 21:22:29 +0100 Subject: [PATCH 50/76] gh-101578: Amend PyErr_{Set,Get}RaisedException docs (#101962) Co-authored-by: C.A.M. Gerlach --- Doc/c-api/exceptions.rst | 49 ++++++++++++++-------------------------- Doc/data/refcounts.dat | 3 +++ 2 files changed, 20 insertions(+), 32 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index de9b15edd6859a..e735f8db402363 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -402,51 +402,36 @@ Querying the error indicator .. c:function:: PyObject *PyErr_GetRaisedException(void) - Returns the exception currently being raised, clearing the exception at - the same time. Do not confuse this with the exception currently being - handled which can be accessed with :c:func:`PyErr_GetHandledException`. + Return the exception currently being raised, clearing the error indicator at + the same time. - .. note:: + This function is used by code that needs to catch exceptions, + or code that needs to save and restore the error indicator temporarily. - This function is normally only used by code that needs to catch exceptions or - by code that needs to save and restore the error indicator temporarily, e.g.:: + For example:: - { - PyObject *exc = PyErr_GetRaisedException(); + { + PyObject *exc = PyErr_GetRaisedException(); - /* ... code that might produce other errors ... */ + /* ... code that might produce other errors ... */ - PyErr_SetRaisedException(exc); - } + PyErr_SetRaisedException(exc); + } + + .. seealso:: :c:func:`PyErr_GetHandledException`, + to save the exception currently being handled. .. versionadded:: 3.12 .. c:function:: void PyErr_SetRaisedException(PyObject *exc) - Sets the exception currently being raised ``exc``. - If the exception is already set, it is cleared first. - - ``exc`` must be a valid exception. - (Violating this rules will cause subtle problems later.) - This call consumes a reference to the ``exc`` object: you must own a - reference to that object before the call and after the call you no longer own - that reference. - (If you don't understand this, don't use this function. I warned you.) + Set *exc* as the exception currently being raised, + clearing the existing exception if one is set. - .. note:: - - This function is normally only used by code that needs to save and restore the - error indicator temporarily. Use :c:func:`PyErr_GetRaisedException` to save - the current exception, e.g.:: - - { - PyObject *exc = PyErr_GetRaisedException(); - - /* ... code that might produce other errors ... */ + .. warning:: - PyErr_SetRaisedException(exc); - } + This call steals a reference to *exc*, which must be a valid exception. .. versionadded:: 3.12 diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 349c4dd5be3d81..ed2958f8cd2205 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -606,6 +606,9 @@ PyErr_GetExcInfo:PyObject**:ptype:+1: PyErr_GetExcInfo:PyObject**:pvalue:+1: PyErr_GetExcInfo:PyObject**:ptraceback:+1: +PyErr_GetRaisedException:PyObject*::+1: +PyErr_SetRaisedException:::: + PyErr_GivenExceptionMatches:int::: PyErr_GivenExceptionMatches:PyObject*:given:0: PyErr_GivenExceptionMatches:PyObject*:exc:0: From b1b375e2670a58fc37cb4c2629ed73b045159918 Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Mon, 20 Feb 2023 01:16:11 +0000 Subject: [PATCH 51/76] gh-97786: Fix compiler warnings in pytime.c (#101826) Fixes compiler warnings in pytime.c. --- ...3-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst | 2 ++ Python/pytime.c | 35 +++++++++++++++---- 2 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst diff --git a/Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst b/Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst new file mode 100644 index 00000000000000..df194b67590d67 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst @@ -0,0 +1,2 @@ +Fix potential undefined behaviour in corner cases of floating-point-to-time +conversions. diff --git a/Python/pytime.c b/Python/pytime.c index 01c07da074757e..acd1842056af43 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -1,5 +1,4 @@ #include "Python.h" -#include "pycore_pymath.h" // _Py_InIntegralTypeRange() #ifdef MS_WINDOWS # include // struct timeval #endif @@ -41,6 +40,14 @@ # error "unsupported time_t size" #endif +#if PY_TIME_T_MAX + PY_TIME_T_MIN != -1 +# error "time_t is not a two's complement integer type" +#endif + +#if _PyTime_MIN + _PyTime_MAX != -1 +# error "_PyTime_t is not a two's complement integer type" +#endif + static void pytime_time_t_overflow(void) @@ -294,7 +301,21 @@ pytime_double_to_denominator(double d, time_t *sec, long *numerator, } assert(0.0 <= floatpart && floatpart < denominator); - if (!_Py_InIntegralTypeRange(time_t, intpart)) { + /* + Conversion of an out-of-range value to time_t gives undefined behaviour + (C99 §6.3.1.4p1), so we must guard against it. However, checking that + `intpart` is in range is delicate: the obvious expression `intpart <= + PY_TIME_T_MAX` will first convert the value `PY_TIME_T_MAX` to a double, + potentially changing its value and leading to us failing to catch some + UB-inducing values. The code below works correctly under the mild + assumption that time_t is a two's complement integer type with no trap + representation, and that `PY_TIME_T_MIN` is within the representable + range of a C double. + + Note: we want the `if` condition below to be true for NaNs; therefore, + resist any temptation to simplify by applying De Morgan's laws. + */ + if (!((double)PY_TIME_T_MIN <= intpart && intpart < -(double)PY_TIME_T_MIN)) { pytime_time_t_overflow(); return -1; } @@ -349,7 +370,8 @@ _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) d = pytime_round(d, round); (void)modf(d, &intpart); - if (!_Py_InIntegralTypeRange(time_t, intpart)) { + /* See comments in pytime_double_to_denominator */ + if (!((double)PY_TIME_T_MIN <= intpart && intpart < -(double)PY_TIME_T_MIN)) { pytime_time_t_overflow(); return -1; } @@ -515,8 +537,9 @@ pytime_from_double(_PyTime_t *tp, double value, _PyTime_round_t round, d *= (double)unit_to_ns; d = pytime_round(d, round); - if (!_Py_InIntegralTypeRange(_PyTime_t, d)) { - pytime_overflow(); + /* See comments in pytime_double_to_denominator */ + if (!((double)_PyTime_MIN <= d && d < -(double)_PyTime_MIN)) { + pytime_time_t_overflow(); return -1; } _PyTime_t ns = (_PyTime_t)d; @@ -910,7 +933,7 @@ py_get_system_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) info->monotonic = 0; info->adjustable = 1; if (clock_getres(CLOCK_REALTIME, &res) == 0) { - info->resolution = res.tv_sec + res.tv_nsec * 1e-9; + info->resolution = (double)res.tv_sec + (double)res.tv_nsec * 1e-9; } else { info->resolution = 1e-9; From 27136310414965a3ea7f835e416cf74b91cefb48 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 20 Feb 2023 14:07:25 +0100 Subject: [PATCH 52/76] gh-101981: Build macOS as recommended by the devguide (GH-102070) Automerge-Triggered-By: GH:erlend-aasland --- .github/workflows/build.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 97ea2d94598e2c..acc8d936774af5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -157,12 +157,19 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v3 - - name: Prepare homebrew environment variables + - name: Install Homebrew dependencies + run: brew install pkg-config openssl@1.1 xz gdbm tcl-tk + - name: Prepare Homebrew environment variables run: | - echo "LDFLAGS=-L$(brew --prefix tcl-tk)/lib" >> $GITHUB_ENV + echo "CFLAGS=-I$(brew --prefix gdbm)/include -I$(brew --prefix xz)/include" >> $GITHUB_ENV + echo "LDFLAGS=-L$(brew --prefix gdbm)/lib -I$(brew --prefix xz)/lib" >> $GITHUB_ENV echo "PKG_CONFIG_PATH=$(brew --prefix openssl@1.1)/lib/pkgconfig:$(brew --prefix tcl-tk)/lib/pkgconfig" >> $GITHUB_ENV - name: Configure CPython - run: ./configure --with-pydebug --prefix=/opt/python-dev + run: | + ./configure \ + --with-pydebug \ + --prefix=/opt/python-dev \ + --with-openssl="$(brew --prefix openssl@1.1)" - name: Build CPython run: make -j4 - name: Display build info From c00faf79438cc7f0d98af2679c695f747e4369a3 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 20 Feb 2023 14:46:20 +0100 Subject: [PATCH 53/76] gh-101819: Adapt _io types to heap types, batch 1 (GH-101949) Adapt StringIO, TextIOWrapper, FileIO, Buffered*, and BytesIO types. Automerge-Triggered-By: GH:erlend-aasland --- Lib/test/test_io.py | 91 +++++- Modules/_io/_iomodule.c | 99 +++--- Modules/_io/_iomodule.h | 47 ++- Modules/_io/bufferedio.c | 341 +++++++------------- Modules/_io/bytesio.c | 77 ++--- Modules/_io/clinic/bufferedio.c.h | 4 +- Modules/_io/fileio.c | 90 ++---- Modules/_io/stringio.c | 85 +++-- Modules/_io/textio.c | 99 +++--- Tools/c-analyzer/cpython/globals-to-fix.tsv | 8 - 10 files changed, 463 insertions(+), 478 deletions(-) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index c5f2e5060a546d..9bcc92a40c61df 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -1042,6 +1042,95 @@ def close(self): support.gc_collect() self.assertIsNone(wr(), wr) +@support.cpython_only +class TestIOCTypes(unittest.TestCase): + def setUp(self): + _io = import_helper.import_module("_io") + self.types = [ + _io.BufferedRWPair, + _io.BufferedRandom, + _io.BufferedReader, + _io.BufferedWriter, + _io.BytesIO, + _io.FileIO, + _io.IncrementalNewlineDecoder, + _io.StringIO, + _io.TextIOWrapper, + _io._BufferedIOBase, + _io._BytesIOBuffer, + _io._IOBase, + _io._RawIOBase, + _io._TextIOBase, + ] + if sys.platform == "win32": + self.types.append(_io._WindowsConsoleIO) + self._io = _io + + def test_immutable_types(self): + for tp in self.types: + with self.subTest(tp=tp): + with self.assertRaisesRegex(TypeError, "immutable"): + tp.foo = "bar" + + def test_class_hierarchy(self): + def check_subs(types, base): + for tp in types: + with self.subTest(tp=tp, base=base): + self.assertTrue(issubclass(tp, base)) + + def recursive_check(d): + for k, v in d.items(): + if isinstance(v, dict): + recursive_check(v) + elif isinstance(v, set): + check_subs(v, k) + else: + self.fail("corrupt test dataset") + + _io = self._io + hierarchy = { + _io._IOBase: { + _io._BufferedIOBase: { + _io.BufferedRWPair, + _io.BufferedRandom, + _io.BufferedReader, + _io.BufferedWriter, + _io.BytesIO, + }, + _io._RawIOBase: { + _io.FileIO, + }, + _io._TextIOBase: { + _io.StringIO, + _io.TextIOWrapper, + }, + }, + } + if sys.platform == "win32": + hierarchy[_io._IOBase][_io._RawIOBase].add(_io._WindowsConsoleIO) + + recursive_check(hierarchy) + + def test_subclassing(self): + _io = self._io + dataset = {k: True for k in self.types} + dataset[_io._BytesIOBuffer] = False + + for tp, is_basetype in dataset.items(): + with self.subTest(tp=tp, is_basetype=is_basetype): + name = f"{tp.__name__}_subclass" + bases = (tp,) + if is_basetype: + _ = type(name, bases, {}) + else: + msg = "not an acceptable base type" + with self.assertRaisesRegex(TypeError, msg): + _ = type(name, bases, {}) + + def test_disallow_instantiation(self): + _io = self._io + support.check_disallow_instantiation(self, _io._BytesIOBuffer) + class PyIOTest(IOTest): pass @@ -4671,7 +4760,7 @@ def load_tests(loader, tests, pattern): CIncrementalNewlineDecoderTest, PyIncrementalNewlineDecoderTest, CTextIOWrapperTest, PyTextIOWrapperTest, CMiscIOTest, PyMiscIOTest, - CSignalsTest, PySignalsTest, + CSignalsTest, PySignalsTest, TestIOCTypes, ) # Put the namespaces of the IO module we are testing and some useful mock diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 811b1d221a0122..55b6535eb34b66 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -10,7 +10,6 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" #include "_iomodule.h" -#include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pystate.h" // _PyInterpreterState_GET() #ifdef HAVE_SYS_TYPES_H @@ -315,8 +314,9 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, } /* Create the Raw file stream */ + _PyIO_State *state = get_io_state(module); { - PyObject *RawIO_class = (PyObject *)&PyFileIO_Type; + PyObject *RawIO_class = (PyObject *)state->PyFileIO_Type; #ifdef MS_WINDOWS const PyConfig *config = _Py_GetConfig(); if (!config->legacy_windows_stdio && _PyIO_get_console_type(path_or_fd) != '\0') { @@ -390,12 +390,15 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, { PyObject *Buffered_class; - if (updating) - Buffered_class = (PyObject *)&PyBufferedRandom_Type; - else if (creating || writing || appending) - Buffered_class = (PyObject *)&PyBufferedWriter_Type; - else if (reading) - Buffered_class = (PyObject *)&PyBufferedReader_Type; + if (updating) { + Buffered_class = (PyObject *)state->PyBufferedRandom_Type; + } + else if (creating || writing || appending) { + Buffered_class = (PyObject *)state->PyBufferedWriter_Type; + } + else if (reading) { + Buffered_class = (PyObject *)state->PyBufferedReader_Type; + } else { PyErr_Format(PyExc_ValueError, "unknown mode: '%s'", mode); @@ -417,7 +420,7 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, } /* wraps into a TextIOWrapper */ - wrapper = PyObject_CallFunction((PyObject *)&PyTextIOWrapper_Type, + wrapper = PyObject_CallFunction((PyObject *)state->PyTextIOWrapper_Type, "OsssO", buffer, encoding, errors, newline, @@ -558,14 +561,6 @@ PyNumber_AsOff_t(PyObject *item, PyObject *err) return result; } -static inline _PyIO_State* -get_io_state(PyObject *module) -{ - void *state = _PyModule_GetState(module); - assert(state != NULL); - return (_PyIO_State *)state; -} - _PyIO_State * _PyIO_get_module_state(void) { @@ -587,6 +582,15 @@ iomodule_traverse(PyObject *mod, visitproc visit, void *arg) { return 0; Py_VISIT(state->locale_module); Py_VISIT(state->unsupported_operation); + + Py_VISIT(state->PyBufferedRWPair_Type); + Py_VISIT(state->PyBufferedRandom_Type); + Py_VISIT(state->PyBufferedReader_Type); + Py_VISIT(state->PyBufferedWriter_Type); + Py_VISIT(state->PyBytesIO_Type); + Py_VISIT(state->PyFileIO_Type); + Py_VISIT(state->PyStringIO_Type); + Py_VISIT(state->PyTextIOWrapper_Type); return 0; } @@ -599,6 +603,15 @@ iomodule_clear(PyObject *mod) { if (state->locale_module != NULL) Py_CLEAR(state->locale_module); Py_CLEAR(state->unsupported_operation); + + Py_CLEAR(state->PyBufferedRWPair_Type); + Py_CLEAR(state->PyBufferedRandom_Type); + Py_CLEAR(state->PyBufferedReader_Type); + Py_CLEAR(state->PyBufferedWriter_Type); + Py_CLEAR(state->PyBytesIO_Type); + Py_CLEAR(state->PyFileIO_Type); + Py_CLEAR(state->PyStringIO_Type); + Py_CLEAR(state->PyTextIOWrapper_Type); return 0; } @@ -612,7 +625,9 @@ iomodule_free(PyObject *mod) { * Module definition */ +#define clinic_state() (get_io_state(module)) #include "clinic/_iomodule.c.h" +#undef clinic_state static PyMethodDef module_methods[] = { _IO_OPEN_METHODDEF @@ -644,23 +659,11 @@ static PyTypeObject* static_types[] = { &PyRawIOBase_Type, &PyTextIOBase_Type, - // PyBufferedIOBase_Type(PyIOBase_Type) subclasses - &PyBytesIO_Type, - &PyBufferedReader_Type, - &PyBufferedWriter_Type, - &PyBufferedRWPair_Type, - &PyBufferedRandom_Type, - // PyRawIOBase_Type(PyIOBase_Type) subclasses - &PyFileIO_Type, &_PyBytesIOBuffer_Type, #ifdef MS_WINDOWS &PyWindowsConsoleIO_Type, #endif - - // PyTextIOBase_Type(PyIOBase_Type) subclasses - &PyStringIO_Type, - &PyTextIOWrapper_Type, }; @@ -673,6 +676,17 @@ _PyIO_Fini(void) } } +#define ADD_TYPE(module, type, spec, base) \ +do { \ + type = (PyTypeObject *)PyType_FromModuleAndSpec(module, spec, \ + (PyObject *)base); \ + if (type == NULL) { \ + goto fail; \ + } \ + if (PyModule_AddType(module, type) < 0) { \ + goto fail; \ + } \ +} while (0) PyMODINIT_FUNC PyInit__io(void) @@ -705,17 +719,9 @@ PyInit__io(void) } // Set type base classes - PyFileIO_Type.tp_base = &PyRawIOBase_Type; - PyBytesIO_Type.tp_base = &PyBufferedIOBase_Type; - PyStringIO_Type.tp_base = &PyTextIOBase_Type; #ifdef MS_WINDOWS PyWindowsConsoleIO_Type.tp_base = &PyRawIOBase_Type; #endif - PyBufferedReader_Type.tp_base = &PyBufferedIOBase_Type; - PyBufferedWriter_Type.tp_base = &PyBufferedIOBase_Type; - PyBufferedRWPair_Type.tp_base = &PyBufferedIOBase_Type; - PyBufferedRandom_Type.tp_base = &PyBufferedIOBase_Type; - PyTextIOWrapper_Type.tp_base = &PyTextIOBase_Type; // Add types for (size_t i=0; i < Py_ARRAY_LENGTH(static_types); i++) { @@ -725,6 +731,25 @@ PyInit__io(void) } } + // PyBufferedIOBase_Type(PyIOBase_Type) subclasses + ADD_TYPE(m, state->PyBytesIO_Type, &bytesio_spec, &PyBufferedIOBase_Type); + ADD_TYPE(m, state->PyBufferedWriter_Type, &bufferedwriter_spec, + &PyBufferedIOBase_Type); + ADD_TYPE(m, state->PyBufferedReader_Type, &bufferedreader_spec, + &PyBufferedIOBase_Type); + ADD_TYPE(m, state->PyBufferedRWPair_Type, &bufferedrwpair_spec, + &PyBufferedIOBase_Type); + ADD_TYPE(m, state->PyBufferedRandom_Type, &bufferedrandom_spec, + &PyBufferedIOBase_Type); + + // PyRawIOBase_Type(PyIOBase_Type) subclasses + ADD_TYPE(m, state->PyFileIO_Type, &fileio_spec, &PyRawIOBase_Type); + + // PyTextIOBase_Type(PyIOBase_Type) subclasses + ADD_TYPE(m, state->PyStringIO_Type, &stringio_spec, &PyTextIOBase_Type); + ADD_TYPE(m, state->PyTextIOWrapper_Type, &textiowrapper_spec, + &PyTextIOBase_Type); + state->initialized = 1; return m; diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index 7617cb8fb70e43..02daef9e85677e 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -4,6 +4,9 @@ #include "exports.h" +#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "structmember.h" + /* ABCs */ extern PyTypeObject PyIOBase_Type; extern PyTypeObject PyRawIOBase_Type; @@ -11,16 +14,18 @@ extern PyTypeObject PyBufferedIOBase_Type; extern PyTypeObject PyTextIOBase_Type; /* Concrete classes */ -extern PyTypeObject PyFileIO_Type; -extern PyTypeObject PyBytesIO_Type; -extern PyTypeObject PyStringIO_Type; -extern PyTypeObject PyBufferedReader_Type; -extern PyTypeObject PyBufferedWriter_Type; -extern PyTypeObject PyBufferedRWPair_Type; -extern PyTypeObject PyBufferedRandom_Type; -extern PyTypeObject PyTextIOWrapper_Type; extern PyTypeObject PyIncrementalNewlineDecoder_Type; +/* Type specs */ +extern PyType_Spec bufferedrandom_spec; +extern PyType_Spec bufferedreader_spec; +extern PyType_Spec bufferedrwpair_spec; +extern PyType_Spec bufferedwriter_spec; +extern PyType_Spec bytesio_spec; +extern PyType_Spec fileio_spec; +extern PyType_Spec stringio_spec; +extern PyType_Spec textiowrapper_spec; + #ifdef MS_WINDOWS extern PyTypeObject PyWindowsConsoleIO_Type; #endif /* MS_WINDOWS */ @@ -140,11 +145,37 @@ typedef struct { PyObject *locale_module; PyObject *unsupported_operation; + + /* Types */ + PyTypeObject *PyBufferedRWPair_Type; + PyTypeObject *PyBufferedRandom_Type; + PyTypeObject *PyBufferedReader_Type; + PyTypeObject *PyBufferedWriter_Type; + PyTypeObject *PyBytesIO_Type; + PyTypeObject *PyFileIO_Type; + PyTypeObject *PyStringIO_Type; + PyTypeObject *PyTextIOWrapper_Type; } _PyIO_State; #define IO_MOD_STATE(mod) ((_PyIO_State *)PyModule_GetState(mod)) #define IO_STATE() _PyIO_get_module_state() +static inline _PyIO_State * +get_io_state(PyObject *module) +{ + void *state = _PyModule_GetState(module); + assert(state != NULL); + return (_PyIO_State *)state; +} + +static inline _PyIO_State * +find_io_state_by_def(PyTypeObject *type) +{ + PyObject *mod = PyType_GetModuleByDef(type, &_PyIO_Module); + assert(mod != NULL); + return get_io_state(mod); +} + extern _PyIO_State *_PyIO_get_module_state(void); #ifdef MS_WINDOWS diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index ba8969f0bcd100..56491f097100c0 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -18,12 +18,12 @@ module _io class _io._BufferedIOBase "PyObject *" "&PyBufferedIOBase_Type" class _io._Buffered "buffered *" "&PyBufferedIOBase_Type" -class _io.BufferedReader "buffered *" "&PyBufferedReader_Type" -class _io.BufferedWriter "buffered *" "&PyBufferedWriter_Type" -class _io.BufferedRWPair "rwpair *" "&PyBufferedRWPair_Type" -class _io.BufferedRandom "buffered *" "&PyBufferedRandom_Type" +class _io.BufferedReader "buffered *" "clinic_state()->PyBufferedReader_Type" +class _io.BufferedWriter "buffered *" "clinic_state()->PyBufferedWriter_Type" +class _io.BufferedRWPair "rwpair *" "clinic_state()->PyBufferedRWPair_Type" +class _io.BufferedRandom "buffered *" "clinic_state()->PyBufferedRandom_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=59460b9c5639984d]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=abd685b9d94b9888]*/ /* * BufferedIOBase class, inherits from IOBase. @@ -366,6 +366,7 @@ _enter_buffered_busy(buffered *self) static void buffered_dealloc(buffered *self) { + PyTypeObject *tp = Py_TYPE(self); self->finalizing = 1; if (_PyIOBase_finalize((PyObject *) self) < 0) return; @@ -383,7 +384,8 @@ buffered_dealloc(buffered *self) self->lock = NULL; } Py_CLEAR(self->dict); - Py_TYPE(self)->tp_free((PyObject *)self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); } static PyObject * @@ -399,6 +401,7 @@ buffered_sizeof(buffered *self, PyObject *Py_UNUSED(ignored)) static int buffered_traverse(buffered *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); Py_VISIT(self->raw); Py_VISIT(self->dict); return 0; @@ -1328,9 +1331,11 @@ buffered_iternext(buffered *self) CHECK_INITIALIZED(self); + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); tp = Py_TYPE(self); - if (tp == &PyBufferedReader_Type || - tp == &PyBufferedRandom_Type) { + if (Py_IS_TYPE(tp, state->PyBufferedReader_Type) || + Py_IS_TYPE(tp, state->PyBufferedRandom_Type)) + { /* Skip method call overhead for speed */ line = _buffered_readline(self, -1); } @@ -1428,8 +1433,11 @@ _io_BufferedReader___init___impl(buffered *self, PyObject *raw, return -1; _bufferedreader_reset_buf(self); - self->fast_closed_checks = (Py_IS_TYPE(self, &PyBufferedReader_Type) && - Py_IS_TYPE(raw, &PyFileIO_Type)); + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); + self->fast_closed_checks = ( + Py_IS_TYPE(self, state->PyBufferedReader_Type) && + Py_IS_TYPE(raw, state->PyFileIO_Type) + ); self->ok = 1; return 0; @@ -1783,8 +1791,11 @@ _io_BufferedWriter___init___impl(buffered *self, PyObject *raw, _bufferedwriter_reset_buf(self); self->pos = 0; - self->fast_closed_checks = (Py_IS_TYPE(self, &PyBufferedWriter_Type) && - Py_IS_TYPE(raw, &PyFileIO_Type)); + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); + self->fast_closed_checks = ( + Py_IS_TYPE(self, state->PyBufferedWriter_Type) && + Py_IS_TYPE(raw, state->PyFileIO_Type) + ); self->ok = 1; return 0; @@ -2090,13 +2101,16 @@ _io_BufferedRWPair___init___impl(rwpair *self, PyObject *reader, if (_PyIOBase_check_writable(writer, Py_True) == NULL) return -1; + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); self->reader = (buffered *) PyObject_CallFunction( - (PyObject *) &PyBufferedReader_Type, "On", reader, buffer_size); + (PyObject *)state->PyBufferedReader_Type, + "On", reader, buffer_size); if (self->reader == NULL) return -1; self->writer = (buffered *) PyObject_CallFunction( - (PyObject *) &PyBufferedWriter_Type, "On", writer, buffer_size); + (PyObject *)state->PyBufferedWriter_Type, + "On", writer, buffer_size); if (self->writer == NULL) { Py_CLEAR(self->reader); return -1; @@ -2108,6 +2122,7 @@ _io_BufferedRWPair___init___impl(rwpair *self, PyObject *reader, static int bufferedrwpair_traverse(rwpair *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); return 0; } @@ -2124,13 +2139,15 @@ bufferedrwpair_clear(rwpair *self) static void bufferedrwpair_dealloc(rwpair *self) { + PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *)self); Py_CLEAR(self->reader); Py_CLEAR(self->writer); Py_CLEAR(self->dict); - Py_TYPE(self)->tp_free((PyObject *) self); + tp->tp_free((PyObject *) self); + Py_DECREF(tp); } static PyObject * @@ -2295,14 +2312,17 @@ _io_BufferedRandom___init___impl(buffered *self, PyObject *raw, _bufferedwriter_reset_buf(self); self->pos = 0; - self->fast_closed_checks = (Py_IS_TYPE(self, &PyBufferedRandom_Type) && - Py_IS_TYPE(raw, &PyFileIO_Type)); + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); + self->fast_closed_checks = (Py_IS_TYPE(self, state->PyBufferedRandom_Type) && + Py_IS_TYPE(raw, state->PyFileIO_Type)); self->ok = 1; return 0; } +#define clinic_state() (find_io_state_by_def(Py_TYPE(self))) #include "clinic/bufferedio.c.h" +#undef clinic_state static PyMethodDef bufferediobase_methods[] = { @@ -2394,6 +2414,8 @@ static PyMethodDef bufferedreader_methods[] = { static PyMemberDef bufferedreader_members[] = { {"raw", T_OBJECT, offsetof(buffered, raw), READONLY}, {"_finalizing", T_BOOL, offsetof(buffered, finalizing), 0}, + {"__weaklistoffset__", T_PYSSIZET, offsetof(buffered, weakreflist), READONLY}, + {"__dictoffset__", T_PYSSIZET, offsetof(buffered, dict), READONLY}, {NULL} }; @@ -2405,58 +2427,27 @@ static PyGetSetDef bufferedreader_getset[] = { }; -PyTypeObject PyBufferedReader_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.BufferedReader", /*tp_name*/ - sizeof(buffered), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)buffered_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - (reprfunc)buffered_repr, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - _io_BufferedReader___init____doc__, /* tp_doc */ - (traverseproc)buffered_traverse, /* tp_traverse */ - (inquiry)buffered_clear, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(buffered, weakreflist), /*tp_weaklistoffset*/ - 0, /* tp_iter */ - (iternextfunc)buffered_iternext, /* tp_iternext */ - bufferedreader_methods, /* tp_methods */ - bufferedreader_members, /* tp_members */ - bufferedreader_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(buffered, dict), /* tp_dictoffset */ - _io_BufferedReader___init__, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0, /* tp_finalize */ +static PyType_Slot bufferedreader_slots[] = { + {Py_tp_dealloc, buffered_dealloc}, + {Py_tp_repr, buffered_repr}, + {Py_tp_doc, (void *)_io_BufferedReader___init____doc__}, + {Py_tp_traverse, buffered_traverse}, + {Py_tp_clear, buffered_clear}, + {Py_tp_iternext, buffered_iternext}, + {Py_tp_methods, bufferedreader_methods}, + {Py_tp_members, bufferedreader_members}, + {Py_tp_getset, bufferedreader_getset}, + {Py_tp_init, _io_BufferedReader___init__}, + {0, NULL}, }; +PyType_Spec bufferedreader_spec = { + .name = "_io.BufferedReader", + .basicsize = sizeof(buffered), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = bufferedreader_slots, +}; static PyMethodDef bufferedwriter_methods[] = { /* BufferedIOMixin methods */ @@ -2480,6 +2471,8 @@ static PyMethodDef bufferedwriter_methods[] = { static PyMemberDef bufferedwriter_members[] = { {"raw", T_OBJECT, offsetof(buffered, raw), READONLY}, {"_finalizing", T_BOOL, offsetof(buffered, finalizing), 0}, + {"__weaklistoffset__", T_PYSSIZET, offsetof(buffered, weakreflist), READONLY}, + {"__dictoffset__", T_PYSSIZET, offsetof(buffered, dict), READONLY}, {NULL} }; @@ -2491,58 +2484,26 @@ static PyGetSetDef bufferedwriter_getset[] = { }; -PyTypeObject PyBufferedWriter_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.BufferedWriter", /*tp_name*/ - sizeof(buffered), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)buffered_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - (reprfunc)buffered_repr, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - _io_BufferedWriter___init____doc__, /* tp_doc */ - (traverseproc)buffered_traverse, /* tp_traverse */ - (inquiry)buffered_clear, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(buffered, weakreflist), /*tp_weaklistoffset*/ - 0, /* tp_iter */ - 0, /* tp_iternext */ - bufferedwriter_methods, /* tp_methods */ - bufferedwriter_members, /* tp_members */ - bufferedwriter_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(buffered, dict), /* tp_dictoffset */ - _io_BufferedWriter___init__, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0, /* tp_finalize */ +static PyType_Slot bufferedwriter_slots[] = { + {Py_tp_dealloc, buffered_dealloc}, + {Py_tp_repr, buffered_repr}, + {Py_tp_doc, (void *)_io_BufferedWriter___init____doc__}, + {Py_tp_traverse, buffered_traverse}, + {Py_tp_clear, buffered_clear}, + {Py_tp_methods, bufferedwriter_methods}, + {Py_tp_members, bufferedwriter_members}, + {Py_tp_getset, bufferedwriter_getset}, + {Py_tp_init, _io_BufferedWriter___init__}, + {0, NULL}, }; +PyType_Spec bufferedwriter_spec = { + .name = "_io.BufferedWriter", + .basicsize = sizeof(buffered), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = bufferedwriter_slots, +}; static PyMethodDef bufferedrwpair_methods[] = { {"read", (PyCFunction)bufferedrwpair_read, METH_VARARGS}, @@ -2563,61 +2524,35 @@ static PyMethodDef bufferedrwpair_methods[] = { {NULL, NULL} }; +static PyMemberDef bufferedrwpair_members[] = { + {"__weaklistoffset__", T_PYSSIZET, offsetof(rwpair, weakreflist), READONLY}, + {"__dictoffset__", T_PYSSIZET, offsetof(rwpair, dict), READONLY}, + {NULL} +}; + static PyGetSetDef bufferedrwpair_getset[] = { {"closed", (getter)bufferedrwpair_closed_get, NULL, NULL}, {NULL} }; -PyTypeObject PyBufferedRWPair_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.BufferedRWPair", /*tp_name*/ - sizeof(rwpair), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)bufferedrwpair_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - _io_BufferedRWPair___init____doc__, /* tp_doc */ - (traverseproc)bufferedrwpair_traverse, /* tp_traverse */ - (inquiry)bufferedrwpair_clear, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(rwpair, weakreflist), /*tp_weaklistoffset*/ - 0, /* tp_iter */ - 0, /* tp_iternext */ - bufferedrwpair_methods, /* tp_methods */ - 0, /* tp_members */ - bufferedrwpair_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(rwpair, dict), /* tp_dictoffset */ - _io_BufferedRWPair___init__, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0, /* tp_finalize */ +static PyType_Slot bufferedrwpair_slots[] = { + {Py_tp_dealloc, bufferedrwpair_dealloc}, + {Py_tp_doc, (void *)_io_BufferedRWPair___init____doc__}, + {Py_tp_traverse, bufferedrwpair_traverse}, + {Py_tp_clear, bufferedrwpair_clear}, + {Py_tp_methods, bufferedrwpair_methods}, + {Py_tp_members, bufferedrwpair_members}, + {Py_tp_getset, bufferedrwpair_getset}, + {Py_tp_init, _io_BufferedRWPair___init__}, + {0, NULL}, +}; + +PyType_Spec bufferedrwpair_spec = { + .name = "_io.BufferedRWPair", + .basicsize = sizeof(rwpair), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = bufferedrwpair_slots, }; @@ -2651,6 +2586,8 @@ static PyMethodDef bufferedrandom_methods[] = { static PyMemberDef bufferedrandom_members[] = { {"raw", T_OBJECT, offsetof(buffered, raw), READONLY}, {"_finalizing", T_BOOL, offsetof(buffered, finalizing), 0}, + {"__weaklistoffset__", T_PYSSIZET, offsetof(buffered, weakreflist), READONLY}, + {"__dictoffset__", T_PYSSIZET, offsetof(buffered, dict), READONLY}, {NULL} }; @@ -2662,54 +2599,24 @@ static PyGetSetDef bufferedrandom_getset[] = { }; -PyTypeObject PyBufferedRandom_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.BufferedRandom", /*tp_name*/ - sizeof(buffered), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)buffered_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - (reprfunc)buffered_repr, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - _io_BufferedRandom___init____doc__, /* tp_doc */ - (traverseproc)buffered_traverse, /* tp_traverse */ - (inquiry)buffered_clear, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(buffered, weakreflist), /*tp_weaklistoffset*/ - 0, /* tp_iter */ - (iternextfunc)buffered_iternext, /* tp_iternext */ - bufferedrandom_methods, /* tp_methods */ - bufferedrandom_members, /* tp_members */ - bufferedrandom_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /*tp_dict*/ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(buffered, dict), /*tp_dictoffset*/ - _io_BufferedRandom___init__, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0, /* tp_finalize */ +static PyType_Slot bufferedrandom_slots[] = { + {Py_tp_dealloc, buffered_dealloc}, + {Py_tp_repr, buffered_repr}, + {Py_tp_doc, (void *)_io_BufferedRandom___init____doc__}, + {Py_tp_traverse, buffered_traverse}, + {Py_tp_clear, buffered_clear}, + {Py_tp_iternext, buffered_iternext}, + {Py_tp_methods, bufferedrandom_methods}, + {Py_tp_members, bufferedrandom_members}, + {Py_tp_getset, bufferedrandom_getset}, + {Py_tp_init, _io_BufferedRandom___init__}, + {0, NULL}, +}; + +PyType_Spec bufferedrandom_spec = { + .name = "_io.BufferedRandom", + .basicsize = sizeof(buffered), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = bufferedrandom_slots, }; diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 6698c60355fcc5..7e9d28b3b9655c 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -5,9 +5,9 @@ /*[clinic input] module _io -class _io.BytesIO "bytesio *" "&PyBytesIO_Type" +class _io.BytesIO "bytesio *" "clinic_state()->PyBytesIO_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7f50ec034f5c0b26]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=48ede2f330f847c3]*/ typedef struct { PyObject_HEAD @@ -881,6 +881,7 @@ bytesio_setstate(bytesio *self, PyObject *state) static void bytesio_dealloc(bytesio *self) { + PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); if (self->exports > 0) { PyErr_SetString(PyExc_SystemError, @@ -891,7 +892,8 @@ bytesio_dealloc(bytesio *self) Py_CLEAR(self->dict); if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); - Py_TYPE(self)->tp_free(self); + tp->tp_free(self); + Py_DECREF(tp); } static PyObject * @@ -971,6 +973,7 @@ bytesio_sizeof(bytesio *self, void *unused) static int bytesio_traverse(bytesio *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); return 0; } @@ -983,7 +986,9 @@ bytesio_clear(bytesio *self) } +#define clinic_state() (find_io_state_by_def(Py_TYPE(self))) #include "clinic/bytesio.c.h" +#undef clinic_state static PyGetSetDef bytesio_getsetlist[] = { {"closed", (getter)bytesio_get_closed, NULL, @@ -1016,48 +1021,34 @@ static struct PyMethodDef bytesio_methods[] = { {NULL, NULL} /* sentinel */ }; -PyTypeObject PyBytesIO_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.BytesIO", /*tp_name*/ - sizeof(bytesio), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)bytesio_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - _io_BytesIO___init____doc__, /*tp_doc*/ - (traverseproc)bytesio_traverse, /*tp_traverse*/ - (inquiry)bytesio_clear, /*tp_clear*/ - 0, /*tp_richcompare*/ - offsetof(bytesio, weakreflist), /*tp_weaklistoffset*/ - PyObject_SelfIter, /*tp_iter*/ - (iternextfunc)bytesio_iternext, /*tp_iternext*/ - bytesio_methods, /*tp_methods*/ - 0, /*tp_members*/ - bytesio_getsetlist, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - offsetof(bytesio, dict), /*tp_dictoffset*/ - _io_BytesIO___init__, /*tp_init*/ - 0, /*tp_alloc*/ - bytesio_new, /*tp_new*/ +static PyMemberDef bytesio_members[] = { + {"__weaklistoffset__", T_PYSSIZET, offsetof(bytesio, weakreflist), READONLY}, + {"__dictoffset__", T_PYSSIZET, offsetof(bytesio, dict), READONLY}, + {NULL} }; +static PyType_Slot bytesio_slots[] = { + {Py_tp_dealloc, bytesio_dealloc}, + {Py_tp_doc, (void *)_io_BytesIO___init____doc__}, + {Py_tp_traverse, bytesio_traverse}, + {Py_tp_clear, bytesio_clear}, + {Py_tp_iter, PyObject_SelfIter}, + {Py_tp_iternext, bytesio_iternext}, + {Py_tp_methods, bytesio_methods}, + {Py_tp_members, bytesio_members}, + {Py_tp_getset, bytesio_getsetlist}, + {Py_tp_init, _io_BytesIO___init__}, + {Py_tp_new, bytesio_new}, + {0, NULL}, +}; + +PyType_Spec bytesio_spec = { + .name = "_io.BytesIO", + .basicsize = sizeof(bytesio), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = bytesio_slots, +}; /* * Implementation of the small intermediate object used by getbuffer(). diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index 38ea756879c122..d44321bb8b960e 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -601,7 +601,7 @@ static int _io_BufferedRWPair___init__(PyObject *self, PyObject *args, PyObject *kwargs) { int return_value = -1; - PyTypeObject *base_tp = &PyBufferedRWPair_Type; + PyTypeObject *base_tp = clinic_state()->PyBufferedRWPair_Type; PyObject *reader; PyObject *writer; Py_ssize_t buffer_size = DEFAULT_BUFFER_SIZE; @@ -714,4 +714,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=953f1577e96e8d86 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8412b10c04259bb8 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index d1a183cedac53a..f424fb8439d7a8 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -51,9 +51,9 @@ /*[clinic input] module _io -class _io.FileIO "fileio *" "&PyFileIO_Type" +class _io.FileIO "fileio *" "clinic_state()->PyFileIO_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=1c77708b41fda70c]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=ac25ec278f4d6703]*/ typedef struct { PyObject_HEAD @@ -70,9 +70,7 @@ typedef struct { PyObject *dict; } fileio; -PyTypeObject PyFileIO_Type; - -#define PyFileIO_Check(op) (PyObject_TypeCheck((op), &PyFileIO_Type)) +#define PyFileIO_Check(state, op) (PyObject_TypeCheck((op), state->PyFileIO_Type)) /* Forward declarations */ static PyObject* portable_lseek(fileio *self, PyObject *posobj, int whence, bool suppress_pipe_error); @@ -242,7 +240,10 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, int fstat_result; int async_err = 0; - assert(PyFileIO_Check(self)); +#ifdef Py_DEBUG + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); + assert(PyFileIO_Check(state, self)); +#endif if (self->fd >= 0) { if (self->closefd) { /* Have to close the existing file first. */ @@ -503,6 +504,7 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, static int fileio_traverse(fileio *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); return 0; } @@ -517,6 +519,7 @@ fileio_clear(fileio *self) static void fileio_dealloc(fileio *self) { + PyTypeObject *tp = Py_TYPE(self); self->finalizing = 1; if (_PyIOBase_finalize((PyObject *) self) < 0) return; @@ -524,7 +527,8 @@ fileio_dealloc(fileio *self) if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); Py_CLEAR(self->dict); - Py_TYPE(self)->tp_free((PyObject *)self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); } static PyObject * @@ -1177,57 +1181,29 @@ static PyGetSetDef fileio_getsetlist[] = { static PyMemberDef fileio_members[] = { {"_blksize", T_UINT, offsetof(fileio, blksize), 0}, {"_finalizing", T_BOOL, offsetof(fileio, finalizing), 0}, + {"__weaklistoffset__", T_PYSSIZET, offsetof(fileio, weakreflist), READONLY}, + {"__dictoffset__", T_PYSSIZET, offsetof(fileio, dict), READONLY}, {NULL} }; -PyTypeObject PyFileIO_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.FileIO", - sizeof(fileio), - 0, - (destructor)fileio_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)fileio_repr, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - _io_FileIO___init____doc__, /* tp_doc */ - (traverseproc)fileio_traverse, /* tp_traverse */ - (inquiry)fileio_clear, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(fileio, weakreflist), /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - fileio_methods, /* tp_methods */ - fileio_members, /* tp_members */ - fileio_getsetlist, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(fileio, dict), /* tp_dictoffset */ - _io_FileIO___init__, /* tp_init */ - PyType_GenericAlloc, /* tp_alloc */ - fileio_new, /* tp_new */ - PyObject_GC_Del, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0, /* tp_finalize */ +static PyType_Slot fileio_slots[] = { + {Py_tp_dealloc, fileio_dealloc}, + {Py_tp_repr, fileio_repr}, + {Py_tp_doc, (void *)_io_FileIO___init____doc__}, + {Py_tp_traverse, fileio_traverse}, + {Py_tp_clear, fileio_clear}, + {Py_tp_methods, fileio_methods}, + {Py_tp_members, fileio_members}, + {Py_tp_getset, fileio_getsetlist}, + {Py_tp_init, _io_FileIO___init__}, + {Py_tp_new, fileio_new}, + {0, NULL}, +}; + +PyType_Spec fileio_spec = { + .name = "_io.FileIO", + .basicsize = sizeof(fileio), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = fileio_slots, }; diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index ae6c3125a2d9da..54c050f0be4688 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -13,9 +13,9 @@ /*[clinic input] module _io -class _io.StringIO "stringio *" "&PyStringIO_Type" +class _io.StringIO "stringio *" "clinic_state()->PyStringIO_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c17bc0f42165cd7d]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=2693eada0658d470]*/ typedef struct { PyObject_HEAD @@ -43,6 +43,7 @@ typedef struct { PyObject *dict; PyObject *weakreflist; + _PyIO_State *module_state; } stringio; static int _io_StringIO___init__(PyObject *self, PyObject *args, PyObject *kwargs); @@ -401,7 +402,7 @@ stringio_iternext(stringio *self) CHECK_CLOSED(self); ENSURE_REALIZED(self); - if (Py_IS_TYPE(self, &PyStringIO_Type)) { + if (Py_IS_TYPE(self, self->module_state->PyStringIO_Type)) { /* Skip method call overhead for speed */ line = _stringio_readline(self, -1); } @@ -581,6 +582,7 @@ _io_StringIO_close_impl(stringio *self) static int stringio_traverse(stringio *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); return 0; } @@ -595,6 +597,7 @@ stringio_clear(stringio *self) static void stringio_dealloc(stringio *self) { + PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); self->ok = 0; if (self->buf) { @@ -606,9 +609,11 @@ stringio_dealloc(stringio *self) Py_CLEAR(self->writenl); Py_CLEAR(self->decoder); Py_CLEAR(self->dict); - if (self->weakreflist != NULL) + if (self->weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *) self); - Py_TYPE(self)->tp_free(self); + } + tp->tp_free(self); + Py_DECREF(tp); } static PyObject * @@ -745,7 +750,7 @@ _io_StringIO___init___impl(stringio *self, PyObject *value, self->state = STATE_ACCUMULATING; } self->pos = 0; - + self->module_state = find_io_state_by_def(Py_TYPE(self)); self->closed = 0; self->ok = 1; return 0; @@ -963,7 +968,9 @@ stringio_newlines(stringio *self, void *context) return PyObject_GetAttr(self->decoder, &_Py_ID(newlines)); } +#define clinic_state() (find_io_state_by_def(Py_TYPE(self))) #include "clinic/stringio.c.h" +#undef clinic_state static struct PyMethodDef stringio_methods[] = { _IO_STRINGIO_CLOSE_METHODDEF @@ -997,44 +1004,30 @@ static PyGetSetDef stringio_getset[] = { {NULL} }; -PyTypeObject PyStringIO_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.StringIO", /*tp_name*/ - sizeof(stringio), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)stringio_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - _io_StringIO___init____doc__, /*tp_doc*/ - (traverseproc)stringio_traverse, /*tp_traverse*/ - (inquiry)stringio_clear, /*tp_clear*/ - 0, /*tp_richcompare*/ - offsetof(stringio, weakreflist), /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - (iternextfunc)stringio_iternext, /*tp_iternext*/ - stringio_methods, /*tp_methods*/ - 0, /*tp_members*/ - stringio_getset, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - offsetof(stringio, dict), /*tp_dictoffset*/ - _io_StringIO___init__, /*tp_init*/ - 0, /*tp_alloc*/ - stringio_new, /*tp_new*/ +static struct PyMemberDef stringio_members[] = { + {"__weaklistoffset__", T_PYSSIZET, offsetof(stringio, weakreflist), READONLY}, + {"__dictoffset__", T_PYSSIZET, offsetof(stringio, dict), READONLY}, + {NULL}, +}; + +static PyType_Slot stringio_slots[] = { + {Py_tp_dealloc, stringio_dealloc}, + {Py_tp_doc, (void *)_io_StringIO___init____doc__}, + {Py_tp_traverse, stringio_traverse}, + {Py_tp_clear, stringio_clear}, + {Py_tp_iternext, stringio_iternext}, + {Py_tp_methods, stringio_methods}, + {Py_tp_members, stringio_members}, + {Py_tp_getset, stringio_getset}, + {Py_tp_init, _io_StringIO___init__}, + {Py_tp_new, stringio_new}, + {0, NULL}, +}; + +PyType_Spec stringio_spec = { + .name = "_io.StringIO", + .basicsize = sizeof(stringio), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = stringio_slots, }; diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index ea2ea32c336954..fbf0bf46840374 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -19,9 +19,9 @@ /*[clinic input] module _io class _io.IncrementalNewlineDecoder "nldecoder_object *" "&PyIncrementalNewlineDecoder_Type" -class _io.TextIOWrapper "textio *" "&TextIOWrapper_Type" +class _io.TextIOWrapper "textio *" "clinic_state()->TextIOWrapper_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=ed072384f8aada2c]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=d3f032e90f74c8f2]*/ /* TextIOBase */ @@ -682,6 +682,8 @@ typedef struct PyObject *weakreflist; PyObject *dict; + + _PyIO_State *state; } textio; static void @@ -1175,15 +1177,16 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, /* Finished sorting out the codec details */ Py_CLEAR(codec_info); - if (Py_IS_TYPE(buffer, &PyBufferedReader_Type) || - Py_IS_TYPE(buffer, &PyBufferedWriter_Type) || - Py_IS_TYPE(buffer, &PyBufferedRandom_Type)) + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); + if (Py_IS_TYPE(buffer, state->PyBufferedReader_Type) || + Py_IS_TYPE(buffer, state->PyBufferedWriter_Type) || + Py_IS_TYPE(buffer, state->PyBufferedRandom_Type)) { if (_PyObject_LookupAttr(buffer, &_Py_ID(raw), &raw) < 0) goto error; /* Cache the raw FileIO object to speed up 'closed' checks */ if (raw != NULL) { - if (Py_IS_TYPE(raw, &PyFileIO_Type)) + if (Py_IS_TYPE(raw, state->PyFileIO_Type)) self->raw = raw; else Py_DECREF(raw); @@ -1211,6 +1214,7 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, goto error; } + self->state = state; self->ok = 1; return 0; @@ -1387,6 +1391,7 @@ textiowrapper_clear(textio *self) static void textiowrapper_dealloc(textio *self) { + PyTypeObject *tp = Py_TYPE(self); self->finalizing = 1; if (_PyIOBase_finalize((PyObject *) self) < 0) return; @@ -1395,12 +1400,14 @@ textiowrapper_dealloc(textio *self) if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *)self); textiowrapper_clear(self); - Py_TYPE(self)->tp_free((PyObject *)self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); } static int textiowrapper_traverse(textio *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); Py_VISIT(self->buffer); Py_VISIT(self->encoding); Py_VISIT(self->encoder); @@ -1424,7 +1431,7 @@ textiowrapper_closed_get(textio *self, void *context); do { \ int r; \ PyObject *_res; \ - if (Py_IS_TYPE(self, &PyTextIOWrapper_Type)) { \ + if (Py_IS_TYPE(self, self->state->PyTextIOWrapper_Type)) { \ if (self->raw != NULL) \ r = _PyFileIO_closed(self->raw); \ else { \ @@ -3053,7 +3060,7 @@ textiowrapper_iternext(textio *self) CHECK_ATTACHED(self); self->telling = 0; - if (Py_IS_TYPE(self, &PyTextIOWrapper_Type)) { + if (Py_IS_TYPE(self, self->state->PyTextIOWrapper_Type)) { /* Skip method call overhead for speed */ line = _textiowrapper_readline(self, -1); } @@ -3145,7 +3152,9 @@ textiowrapper_chunk_size_set(textio *self, PyObject *arg, void *context) return 0; } +#define clinic_state() (find_io_state_by_def(Py_TYPE(self))) #include "clinic/textio.c.h" +#undef clinic_state static PyMethodDef incrementalnewlinedecoder_methods[] = { _IO_INCREMENTALNEWLINEDECODER_DECODE_METHODDEF @@ -3229,6 +3238,8 @@ static PyMemberDef textiowrapper_members[] = { {"line_buffering", T_BOOL, offsetof(textio, line_buffering), READONLY}, {"write_through", T_BOOL, offsetof(textio, write_through), READONLY}, {"_finalizing", T_BOOL, offsetof(textio, finalizing), 0}, + {"__weaklistoffset__", T_PYSSIZET, offsetof(textio, weakreflist), READONLY}, + {"__dictoffset__", T_PYSSIZET, offsetof(textio, dict), READONLY}, {NULL} }; @@ -3244,54 +3255,24 @@ static PyGetSetDef textiowrapper_getset[] = { {NULL} }; -PyTypeObject PyTextIOWrapper_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.TextIOWrapper", /*tp_name*/ - sizeof(textio), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)textiowrapper_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tps_etattr*/ - 0, /*tp_as_async*/ - (reprfunc)textiowrapper_repr,/*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - _io_TextIOWrapper___init____doc__, /* tp_doc */ - (traverseproc)textiowrapper_traverse, /* tp_traverse */ - (inquiry)textiowrapper_clear, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(textio, weakreflist), /*tp_weaklistoffset*/ - 0, /* tp_iter */ - (iternextfunc)textiowrapper_iternext, /* tp_iternext */ - textiowrapper_methods, /* tp_methods */ - textiowrapper_members, /* tp_members */ - textiowrapper_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(textio, dict), /*tp_dictoffset*/ - _io_TextIOWrapper___init__, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0, /* tp_finalize */ +PyType_Slot textiowrapper_slots[] = { + {Py_tp_dealloc, textiowrapper_dealloc}, + {Py_tp_repr, textiowrapper_repr}, + {Py_tp_doc, (void *)_io_TextIOWrapper___init____doc__}, + {Py_tp_traverse, textiowrapper_traverse}, + {Py_tp_clear, textiowrapper_clear}, + {Py_tp_iternext, textiowrapper_iternext}, + {Py_tp_methods, textiowrapper_methods}, + {Py_tp_members, textiowrapper_members}, + {Py_tp_getset, textiowrapper_getset}, + {Py_tp_init, _io_TextIOWrapper___init__}, + {0, NULL}, +}; + +PyType_Spec textiowrapper_spec = { + .name = "_io.TextIOWrapper", + .basicsize = sizeof(textio), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = textiowrapper_slots, }; diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 6011b1604508af..2e28c50c6ff69a 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -317,19 +317,11 @@ Modules/_collectionsmodule.c - dequeiter_type - Modules/_collectionsmodule.c - dequereviter_type - Modules/_collectionsmodule.c - tuplegetter_type - Modules/_io/bufferedio.c - PyBufferedIOBase_Type - -Modules/_io/bufferedio.c - PyBufferedRWPair_Type - -Modules/_io/bufferedio.c - PyBufferedRandom_Type - -Modules/_io/bufferedio.c - PyBufferedReader_Type - -Modules/_io/bufferedio.c - PyBufferedWriter_Type - -Modules/_io/bytesio.c - PyBytesIO_Type - Modules/_io/bytesio.c - _PyBytesIOBuffer_Type - -Modules/_io/fileio.c - PyFileIO_Type - Modules/_io/iobase.c - PyIOBase_Type - Modules/_io/iobase.c - PyRawIOBase_Type - -Modules/_io/stringio.c - PyStringIO_Type - Modules/_io/textio.c - PyIncrementalNewlineDecoder_Type - Modules/_io/textio.c - PyTextIOBase_Type - -Modules/_io/textio.c - PyTextIOWrapper_Type - Modules/_io/winconsoleio.c - PyWindowsConsoleIO_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorBase_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorDerived_Type - From a99eb5cd9947629a6745a4ad99cb07af1c287b5d Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 20 Feb 2023 14:56:48 +0000 Subject: [PATCH 54/76] gh-101907: Stop using `_Py_OPCODE` and `_Py_OPARG` macros (GH-101912) * gh-101907: Removes use of non-standard C++ extension from Include/cpython/code.h * Make cases_generator correct on Windows --- Include/cpython/code.h | 28 ++- ...-02-14-15-53-01.gh-issue-101907.HgF1N2.rst | 1 + Objects/codeobject.c | 20 +- Objects/frameobject.c | 24 +-- Objects/genobject.c | 10 +- Objects/typeobject.c | 4 +- Python/bytecodes.c | 34 ++-- Python/ceval.c | 12 +- Python/ceval_macros.h | 18 +- Python/compile.c | 20 +- Python/generated_cases.c.h | 34 ++-- Python/specialize.c | 172 +++++++++--------- Tools/cases_generator/generate_cases.py | 7 +- 13 files changed, 200 insertions(+), 184 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-02-14-15-53-01.gh-issue-101907.HgF1N2.rst diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 0cf49f06c87732..fba9296aedc99d 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -19,21 +19,35 @@ extern "C" { typedef union { uint16_t cache; struct { - uint8_t opcode; - uint8_t oparg; - }; + uint8_t code; + uint8_t arg; + } op; } _Py_CODEUNIT; -#define _Py_OPCODE(word) ((word).opcode) -#define _Py_OPARG(word) ((word).oparg) + +/* These macros only remain defined for compatibility. */ +#define _Py_OPCODE(word) ((word).op.code) +#define _Py_OPARG(word) ((word).op.arg) + +static inline _Py_CODEUNIT +_py_make_codeunit(uint8_t opcode, uint8_t oparg) +{ + // No designated initialisers because of C++ compat + _Py_CODEUNIT word; + word.op.code = opcode; + word.op.arg = oparg; + return word; +} static inline void _py_set_opcode(_Py_CODEUNIT *word, uint8_t opcode) { - word->opcode = opcode; + word->op.code = opcode; } -#define _Py_SET_OPCODE(word, opcode) _py_set_opocde(&(word), opcode) +#define _Py_MAKE_CODEUNIT(opcode, oparg) _py_make_codeunit((opcode), (oparg)) +#define _Py_SET_OPCODE(word, opcode) _py_set_opcode(&(word), (opcode)) + typedef struct { PyObject *_co_code; diff --git a/Misc/NEWS.d/next/C API/2023-02-14-15-53-01.gh-issue-101907.HgF1N2.rst b/Misc/NEWS.d/next/C API/2023-02-14-15-53-01.gh-issue-101907.HgF1N2.rst new file mode 100644 index 00000000000000..cfc0d72cdbca00 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-02-14-15-53-01.gh-issue-101907.HgF1N2.rst @@ -0,0 +1 @@ +Removes use of non-standard C++ extension in public header files. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index ab31b6582cdaae..a03b14edea8d4c 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -413,7 +413,7 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) PyBytes_GET_SIZE(con->code)); int entry_point = 0; while (entry_point < Py_SIZE(co) && - _Py_OPCODE(_PyCode_CODE(co)[entry_point]) != RESUME) { + _PyCode_CODE(co)[entry_point].op.code != RESUME) { entry_point++; } co->_co_firsttraceable = entry_point; @@ -1505,12 +1505,12 @@ deopt_code(_Py_CODEUNIT *instructions, Py_ssize_t len) { for (int i = 0; i < len; i++) { _Py_CODEUNIT instruction = instructions[i]; - int opcode = _PyOpcode_Deopt[_Py_OPCODE(instruction)]; + int opcode = _PyOpcode_Deopt[instruction.op.code]; int caches = _PyOpcode_Caches[opcode]; - instructions[i].opcode = opcode; + instructions[i].op.code = opcode; while (caches--) { - instructions[++i].opcode = CACHE; - instructions[i].oparg = 0; + instructions[++i].op.code = CACHE; + instructions[i].op.arg = 0; } } } @@ -1763,13 +1763,13 @@ code_richcompare(PyObject *self, PyObject *other, int op) for (int i = 0; i < Py_SIZE(co); i++) { _Py_CODEUNIT co_instr = _PyCode_CODE(co)[i]; _Py_CODEUNIT cp_instr = _PyCode_CODE(cp)[i]; - co_instr.opcode = _PyOpcode_Deopt[_Py_OPCODE(co_instr)]; - cp_instr.opcode =_PyOpcode_Deopt[_Py_OPCODE(cp_instr)]; + co_instr.op.code = _PyOpcode_Deopt[co_instr.op.code]; + cp_instr.op.code = _PyOpcode_Deopt[cp_instr.op.code]; eq = co_instr.cache == cp_instr.cache; if (!eq) { goto unequal; } - i += _PyOpcode_Caches[_Py_OPCODE(co_instr)]; + i += _PyOpcode_Caches[co_instr.op.code]; } /* compare constants */ @@ -1848,9 +1848,9 @@ code_hash(PyCodeObject *co) SCRAMBLE_IN(co->co_firstlineno); SCRAMBLE_IN(Py_SIZE(co)); for (int i = 0; i < Py_SIZE(co); i++) { - int deop = _PyOpcode_Deopt[_Py_OPCODE(_PyCode_CODE(co)[i])]; + int deop = _PyOpcode_Deopt[_PyCode_CODE(co)[i].op.code]; SCRAMBLE_IN(deop); - SCRAMBLE_IN(_Py_OPARG(_PyCode_CODE(co)[i])); + SCRAMBLE_IN(_PyCode_CODE(co)[i].op.arg); i += _PyOpcode_Caches[deop]; } if ((Py_hash_t)uhash == -1) { diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 581ed2d214c4d9..34143c9a40b293 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -111,13 +111,13 @@ static unsigned int get_arg(const _Py_CODEUNIT *codestr, Py_ssize_t i) { _Py_CODEUNIT word; - unsigned int oparg = _Py_OPARG(codestr[i]); - if (i >= 1 && _Py_OPCODE(word = codestr[i-1]) == EXTENDED_ARG) { - oparg |= _Py_OPARG(word) << 8; - if (i >= 2 && _Py_OPCODE(word = codestr[i-2]) == EXTENDED_ARG) { - oparg |= _Py_OPARG(word) << 16; - if (i >= 3 && _Py_OPCODE(word = codestr[i-3]) == EXTENDED_ARG) { - oparg |= _Py_OPARG(word) << 24; + unsigned int oparg = codestr[i].op.arg; + if (i >= 1 && (word = codestr[i-1]).op.code == EXTENDED_ARG) { + oparg |= word.op.arg << 8; + if (i >= 2 && (word = codestr[i-2]).op.code == EXTENDED_ARG) { + oparg |= word.op.arg << 16; + if (i >= 3 && (word = codestr[i-3]).op.code == EXTENDED_ARG) { + oparg |= word.op.arg << 24; } } } @@ -304,7 +304,7 @@ mark_stacks(PyCodeObject *code_obj, int len) if (next_stack == UNINITIALIZED) { continue; } - opcode = _Py_OPCODE(code[i]); + opcode = code[i].op.code; switch (opcode) { case JUMP_IF_FALSE_OR_POP: case JUMP_IF_TRUE_OR_POP: @@ -610,7 +610,7 @@ _PyFrame_GetState(PyFrameObject *frame) if (_PyInterpreterFrame_LASTI(frame->f_frame) < 0) { return FRAME_CREATED; } - switch (_Py_OPCODE(*frame->f_frame->prev_instr)) + switch (frame->f_frame->prev_instr->op.code) { case COPY_FREE_VARS: case MAKE_CELL: @@ -1092,8 +1092,8 @@ _PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame, int opcode, int oparg) for (_Py_CODEUNIT *instruction = _PyCode_CODE(frame->f_code); instruction < frame->prev_instr; instruction++) { - int check_opcode = _PyOpcode_Deopt[_Py_OPCODE(*instruction)]; - check_oparg |= _Py_OPARG(*instruction); + int check_opcode = _PyOpcode_Deopt[instruction->op.code]; + check_oparg |= instruction->op.arg; if (check_opcode == opcode && check_oparg == oparg) { return 1; } @@ -1117,7 +1117,7 @@ frame_init_get_vars(_PyInterpreterFrame *frame) // here: PyCodeObject *co = frame->f_code; int lasti = _PyInterpreterFrame_LASTI(frame); - if (!(lasti < 0 && _Py_OPCODE(_PyCode_CODE(co)[0]) == COPY_FREE_VARS + if (!(lasti < 0 && _PyCode_CODE(co)->op.code == COPY_FREE_VARS && PyFunction_Check(frame->f_funcobj))) { /* Free vars are initialized */ diff --git a/Objects/genobject.c b/Objects/genobject.c index 35246653c45348..aec64ca7004e11 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -332,11 +332,11 @@ _PyGen_yf(PyGenObject *gen) /* Return immediately if the frame didn't start yet. SEND always come after LOAD_CONST: a code object should not start with SEND */ - assert(_Py_OPCODE(_PyCode_CODE(gen->gi_code)[0]) != SEND); + assert(_PyCode_CODE(gen->gi_code)[0].op.code != SEND); return NULL; } _Py_CODEUNIT next = frame->prev_instr[1]; - if (_Py_OPCODE(next) != RESUME || _Py_OPARG(next) < 2) + if (next.op.code != RESUME || next.op.arg < 2) { /* Not in a yield from */ return NULL; @@ -371,9 +371,9 @@ gen_close(PyGenObject *gen, PyObject *args) _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; /* It is possible for the previous instruction to not be a * YIELD_VALUE if the debugger has changed the lineno. */ - if (err == 0 && frame->prev_instr->opcode == YIELD_VALUE) { - assert(frame->prev_instr[1].opcode == RESUME); - int exception_handler_depth = frame->prev_instr->oparg; + if (err == 0 && frame->prev_instr[0].op.code == YIELD_VALUE) { + assert(frame->prev_instr[1].op.code == RESUME); + int exception_handler_depth = frame->prev_instr[0].op.code; assert(exception_handler_depth > 0); /* We can safely ignore the outermost try block * as it automatically generated to handle diff --git a/Objects/typeobject.c b/Objects/typeobject.c index f2e8092aa37eec..2d1220a0695036 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -9507,8 +9507,8 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyCodeObject *co, if (_PyInterpreterFrame_LASTI(cframe) >= 0) { // MAKE_CELL and COPY_FREE_VARS have no quickened forms, so no need // to use _PyOpcode_Deopt here: - assert(_Py_OPCODE(_PyCode_CODE(co)[0]) == MAKE_CELL || - _Py_OPCODE(_PyCode_CODE(co)[0]) == COPY_FREE_VARS); + assert(_PyCode_CODE(co)[0].op.code == MAKE_CELL || + _PyCode_CODE(co)[0].op.code == COPY_FREE_VARS); assert(PyCell_Check(firstarg)); firstarg = PyCell_GET(firstarg); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 84747f1758e06c..c5959f2f994fdc 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -246,9 +246,9 @@ dummy_func( DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; - assert(_Py_OPCODE(true_next) == STORE_FAST || - _Py_OPCODE(true_next) == STORE_FAST__LOAD_FAST); - PyObject **target_local = &GETLOCAL(_Py_OPARG(true_next)); + assert(true_next.op.code == STORE_FAST || + true_next.op.code == STORE_FAST__LOAD_FAST); + PyObject **target_local = &GETLOCAL(true_next.op.arg); DEOPT_IF(*target_local != left, BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. @@ -1748,10 +1748,10 @@ dummy_func( Py_DECREF(left); Py_DECREF(right); ERROR_IF(cond == NULL, error); - assert(_Py_OPCODE(next_instr[1]) == POP_JUMP_IF_FALSE || - _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE); - bool jump_on_true = _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE; - int offset = _Py_OPARG(next_instr[1]); + assert(next_instr[1].op.code == POP_JUMP_IF_FALSE || + next_instr[1].op.code == POP_JUMP_IF_TRUE); + bool jump_on_true = next_instr[1].op.code == POP_JUMP_IF_TRUE; + int offset = next_instr[1].op.arg; int err = PyObject_IsTrue(cond); Py_DECREF(cond); if (err < 0) { @@ -1774,7 +1774,7 @@ dummy_func( _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); if (sign_ish & oparg) { - int offset = _Py_OPARG(next_instr[1]); + int offset = next_instr[1].op.arg; JUMPBY(offset); } } @@ -1795,7 +1795,7 @@ dummy_func( _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); if (sign_ish & oparg) { - int offset = _Py_OPARG(next_instr[1]); + int offset = next_instr[1].op.arg; JUMPBY(offset); } } @@ -1814,7 +1814,7 @@ dummy_func( assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); if ((res + COMPARISON_NOT_EQUALS) & oparg) { - int offset = _Py_OPARG(next_instr[1]); + int offset = next_instr[1].op.arg; JUMPBY(offset); } } @@ -2122,7 +2122,7 @@ dummy_func( _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(_Py_OPCODE(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg]) == END_FOR); + assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR); Py_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR instruction */ @@ -2186,7 +2186,7 @@ dummy_func( DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); _Py_CODEUNIT next = next_instr[INLINE_CACHE_ENTRIES_FOR_ITER]; - assert(_PyOpcode_Deopt[_Py_OPCODE(next)] == STORE_FAST); + assert(_PyOpcode_Deopt[next.op.code] == STORE_FAST); if (r->len <= 0) { STACK_SHRINK(1); Py_DECREF(r); @@ -2197,7 +2197,7 @@ dummy_func( long value = r->start; r->start = value + r->step; r->len--; - if (_PyLong_AssignValue(&GETLOCAL(_Py_OPARG(next)), value) < 0) { + if (_PyLong_AssignValue(&GETLOCAL(next.op.arg), value) < 0) { goto error; } // The STORE_FAST is already done. @@ -2220,7 +2220,7 @@ dummy_func( gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg); - assert(_Py_OPCODE(*next_instr) == END_FOR); + assert(next_instr->op.code == END_FOR); DISPATCH_INLINED(gen_frame); } @@ -2809,7 +2809,7 @@ dummy_func( STACK_SHRINK(3); // CALL + POP_TOP JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); - assert(_Py_OPCODE(next_instr[-1]) == POP_TOP); + assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); } @@ -3118,8 +3118,8 @@ dummy_func( inst(EXTENDED_ARG, (--)) { assert(oparg); assert(cframe.use_tracing == 0); - opcode = _Py_OPCODE(*next_instr); - oparg = oparg << 8 | _Py_OPARG(*next_instr); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); } diff --git a/Python/ceval.c b/Python/ceval.c index 308ef52259df3d..b85231a1df8a95 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -132,8 +132,8 @@ lltrace_instruction(_PyInterpreterFrame *frame, objects enters the interpreter recursively. It is also slow. So you might want to comment it out. */ dump_stack(frame, stack_pointer); - int oparg = _Py_OPARG(*next_instr); - int opcode = _Py_OPCODE(*next_instr); + int oparg = next_instr->op.arg; + int opcode = next_instr->op.code; const char *opname = _PyOpcode_OpName[opcode]; assert(opname != NULL); int offset = (int)(next_instr - _PyCode_CODE(frame->f_code)); @@ -920,8 +920,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int // CPython hasn't ever traced the instruction after an EXTENDED_ARG. // Inline the EXTENDED_ARG here, so we can avoid branching there: INSTRUCTION_START(EXTENDED_ARG); - opcode = _Py_OPCODE(*next_instr); - oparg = oparg << 8 | _Py_OPARG(*next_instr); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; // Make sure the next instruction isn't a RESUME, since that needs // to trace properly (and shouldn't have an EXTENDED_ARG, anyways): assert(opcode != RESUME); @@ -946,7 +946,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif /* Tell C compilers not to hold the opcode variable in the loop. next_instr points the current instruction without TARGET(). */ - opcode = _Py_OPCODE(*next_instr); + opcode = next_instr->op.code; _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", frame->f_code->co_filename, @@ -2196,7 +2196,7 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj, (_PyInterpreterFrame_LASTI(frame) < instr_prev && // SEND has no quickened forms, so no need to use _PyOpcode_Deopt // here: - _Py_OPCODE(*frame->prev_instr) != SEND); + frame->prev_instr->op.code != SEND); if (trace) { result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 691bf8e1caae95..ac1fec77daca8a 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -100,7 +100,7 @@ #define DISPATCH_SAME_OPARG() \ { \ - opcode = _Py_OPCODE(*next_instr); \ + opcode = next_instr->op.code; \ PRE_DISPATCH_GOTO(); \ opcode |= cframe.use_tracing OR_DTRACE_LINE; \ DISPATCH_GOTO(); \ @@ -143,8 +143,8 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define INSTR_OFFSET() ((int)(next_instr - _PyCode_CODE(frame->f_code))) #define NEXTOPARG() do { \ _Py_CODEUNIT word = *next_instr; \ - opcode = _Py_OPCODE(word); \ - oparg = _Py_OPARG(word); \ + opcode = word.op.code; \ + oparg = word.op.arg; \ } while (0) #define JUMPTO(x) (next_instr = _PyCode_CODE(frame->f_code) + (x)) #define JUMPBY(x) (next_instr += (x)) @@ -180,14 +180,14 @@ GETITEM(PyObject *v, Py_ssize_t i) { #if USE_COMPUTED_GOTOS #define PREDICT(op) if (0) goto PREDICT_ID(op) #else -#define PREDICT(op) \ +#define PREDICT(next_op) \ do { \ _Py_CODEUNIT word = *next_instr; \ - opcode = _Py_OPCODE(word) | cframe.use_tracing OR_DTRACE_LINE; \ - if (opcode == op) { \ - oparg = _Py_OPARG(word); \ - INSTRUCTION_START(op); \ - goto PREDICT_ID(op); \ + opcode = word.op.code | cframe.use_tracing OR_DTRACE_LINE; \ + if (opcode == next_op) { \ + oparg = word.op.arg; \ + INSTRUCTION_START(next_op); \ + goto PREDICT_ID(next_op); \ } \ } while(0) #endif diff --git a/Python/compile.c b/Python/compile.c index c3b344c7af2a7f..3f620beb0d0205 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -274,31 +274,31 @@ write_instr(_Py_CODEUNIT *codestr, struct instr *instruction, int ilen) int caches = _PyOpcode_Caches[opcode]; switch (ilen - caches) { case 4: - codestr->opcode = EXTENDED_ARG; - codestr->oparg = (oparg >> 24) & 0xFF; + codestr->op.code = EXTENDED_ARG; + codestr->op.arg = (oparg >> 24) & 0xFF; codestr++; /* fall through */ case 3: - codestr->opcode = EXTENDED_ARG; - codestr->oparg = (oparg >> 16) & 0xFF; + codestr->op.code = EXTENDED_ARG; + codestr->op.arg = (oparg >> 16) & 0xFF; codestr++; /* fall through */ case 2: - codestr->opcode = EXTENDED_ARG; - codestr->oparg = (oparg >> 8) & 0xFF; + codestr->op.code = EXTENDED_ARG; + codestr->op.arg = (oparg >> 8) & 0xFF; codestr++; /* fall through */ case 1: - codestr->opcode = opcode; - codestr->oparg = oparg & 0xFF; + codestr->op.code = opcode; + codestr->op.arg = oparg & 0xFF; codestr++; break; default: Py_UNREACHABLE(); } while (caches--) { - codestr->opcode = CACHE; - codestr->oparg = 0; + codestr->op.code = CACHE; + codestr->op.arg = 0; codestr++; } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 730dfb7426acbf..487e63d855d14a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -339,9 +339,9 @@ DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; - assert(_Py_OPCODE(true_next) == STORE_FAST || - _Py_OPCODE(true_next) == STORE_FAST__LOAD_FAST); - PyObject **target_local = &GETLOCAL(_Py_OPARG(true_next)); + assert(true_next.op.code == STORE_FAST || + true_next.op.code == STORE_FAST__LOAD_FAST); + PyObject **target_local = &GETLOCAL(true_next.op.arg); DEOPT_IF(*target_local != left, BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. @@ -2199,10 +2199,10 @@ Py_DECREF(left); Py_DECREF(right); if (cond == NULL) goto pop_2_error; - assert(_Py_OPCODE(next_instr[1]) == POP_JUMP_IF_FALSE || - _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE); - bool jump_on_true = _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE; - int offset = _Py_OPARG(next_instr[1]); + assert(next_instr[1].op.code == POP_JUMP_IF_FALSE || + next_instr[1].op.code == POP_JUMP_IF_TRUE); + bool jump_on_true = next_instr[1].op.code == POP_JUMP_IF_TRUE; + int offset = next_instr[1].op.arg; int err = PyObject_IsTrue(cond); Py_DECREF(cond); if (err < 0) { @@ -2230,7 +2230,7 @@ _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); if (sign_ish & oparg) { - int offset = _Py_OPARG(next_instr[1]); + int offset = next_instr[1].op.arg; JUMPBY(offset); } STACK_SHRINK(2); @@ -2255,7 +2255,7 @@ _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); if (sign_ish & oparg) { - int offset = _Py_OPARG(next_instr[1]); + int offset = next_instr[1].op.arg; JUMPBY(offset); } STACK_SHRINK(2); @@ -2278,7 +2278,7 @@ assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); if ((res + COMPARISON_NOT_EQUALS) & oparg) { - int offset = _Py_OPARG(next_instr[1]); + int offset = next_instr[1].op.arg; JUMPBY(offset); } STACK_SHRINK(2); @@ -2682,7 +2682,7 @@ _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(_Py_OPCODE(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg]) == END_FOR); + assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR); Py_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR instruction */ @@ -2761,7 +2761,7 @@ DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); _Py_CODEUNIT next = next_instr[INLINE_CACHE_ENTRIES_FOR_ITER]; - assert(_PyOpcode_Deopt[_Py_OPCODE(next)] == STORE_FAST); + assert(_PyOpcode_Deopt[next.op.code] == STORE_FAST); if (r->len <= 0) { STACK_SHRINK(1); Py_DECREF(r); @@ -2772,7 +2772,7 @@ long value = r->start; r->start = value + r->step; r->len--; - if (_PyLong_AssignValue(&GETLOCAL(_Py_OPARG(next)), value) < 0) { + if (_PyLong_AssignValue(&GETLOCAL(next.op.arg), value) < 0) { goto error; } // The STORE_FAST is already done. @@ -2795,7 +2795,7 @@ gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg); - assert(_Py_OPCODE(*next_instr) == END_FOR); + assert(next_instr->op.code == END_FOR); DISPATCH_INLINED(gen_frame); } @@ -3516,7 +3516,7 @@ STACK_SHRINK(3); // CALL + POP_TOP JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); - assert(_Py_OPCODE(next_instr[-1]) == POP_TOP); + assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); } @@ -3903,8 +3903,8 @@ TARGET(EXTENDED_ARG) { assert(oparg); assert(cframe.use_tracing == 0); - opcode = _Py_OPCODE(*next_instr); - oparg = oparg << 8 | _Py_OPARG(*next_instr); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); } diff --git a/Python/specialize.c b/Python/specialize.c index 4ede3122d38046..c9555f8ad4dccb 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -282,7 +282,7 @@ _PyCode_Quicken(PyCodeObject *code) _Py_CODEUNIT *instructions = _PyCode_CODE(code); for (int i = 0; i < Py_SIZE(code); i++) { int previous_opcode = opcode; - opcode = _PyOpcode_Deopt[_Py_OPCODE(instructions[i])]; + opcode = _PyOpcode_Deopt[instructions[i].op.code]; int caches = _PyOpcode_Caches[opcode]; if (caches) { instructions[i + 1].cache = adaptive_counter_warmup(); @@ -291,31 +291,31 @@ _PyCode_Quicken(PyCodeObject *code) } switch (previous_opcode << 8 | opcode) { case LOAD_CONST << 8 | LOAD_FAST: - instructions[i - 1].opcode = LOAD_CONST__LOAD_FAST; + instructions[i - 1].op.code = LOAD_CONST__LOAD_FAST; break; case LOAD_FAST << 8 | LOAD_CONST: - instructions[i - 1].opcode = LOAD_FAST__LOAD_CONST; + instructions[i - 1].op.code = LOAD_FAST__LOAD_CONST; break; case LOAD_FAST << 8 | LOAD_FAST: - instructions[i - 1].opcode = LOAD_FAST__LOAD_FAST; + instructions[i - 1].op.code = LOAD_FAST__LOAD_FAST; break; case STORE_FAST << 8 | LOAD_FAST: - instructions[i - 1].opcode = STORE_FAST__LOAD_FAST; + instructions[i - 1].op.code = STORE_FAST__LOAD_FAST; break; case STORE_FAST << 8 | STORE_FAST: - instructions[i - 1].opcode = STORE_FAST__STORE_FAST; + instructions[i - 1].op.code = STORE_FAST__STORE_FAST; break; case COMPARE_OP << 8 | POP_JUMP_IF_TRUE: case COMPARE_OP << 8 | POP_JUMP_IF_FALSE: { - int oparg = instructions[i - 1 - INLINE_CACHE_ENTRIES_COMPARE_OP].oparg; + int oparg = instructions[i - 1 - INLINE_CACHE_ENTRIES_COMPARE_OP].op.arg; assert((oparg >> 4) <= Py_GE); int mask = compare_masks[oparg >> 4]; if (opcode == POP_JUMP_IF_FALSE) { mask = mask ^ 0xf; } - instructions[i - 1 - INLINE_CACHE_ENTRIES_COMPARE_OP].opcode = COMPARE_AND_BRANCH; - instructions[i - 1 - INLINE_CACHE_ENTRIES_COMPARE_OP].oparg = (oparg & 0xf0) | mask; + instructions[i - 1 - INLINE_CACHE_ENTRIES_COMPARE_OP].op.code = COMPARE_AND_BRANCH; + instructions[i - 1 - INLINE_CACHE_ENTRIES_COMPARE_OP].op.arg = (oparg & 0xf0) | mask; break; } } @@ -519,7 +519,7 @@ specialize_module_load_attr( } write_u32(cache->version, keys_version); cache->index = (uint16_t)index; - _py_set_opcode(instr, LOAD_ATTR_MODULE); + instr->op.code = LOAD_ATTR_MODULE; return 0; } @@ -674,7 +674,7 @@ specialize_dict_access( } write_u32(cache->version, type->tp_version_tag); cache->index = (uint16_t)index; - _py_set_opcode(instr, values_op); + instr->op.code = values_op; } else { PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); @@ -694,7 +694,7 @@ specialize_dict_access( } cache->index = (uint16_t)index; write_u32(cache->version, type->tp_version_tag); - _py_set_opcode(instr, hint_op); + instr->op.code = hint_op; } return 1; } @@ -739,7 +739,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) goto fail; case METHOD: { - int oparg = _Py_OPARG(*instr); + int oparg = instr->op.arg; if (oparg & 1) { if (specialize_attr_loadmethod(owner, instr, name, descr, kind)) { goto success; @@ -775,7 +775,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) write_u32(lm_cache->type_version, type->tp_version_tag); /* borrowed */ write_obj(lm_cache->descr, fget); - _py_set_opcode(instr, LOAD_ATTR_PROPERTY); + instr->op.code = LOAD_ATTR_PROPERTY; goto success; } case OBJECT_SLOT: @@ -799,7 +799,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) assert(offset > 0); cache->index = (uint16_t)offset; write_u32(cache->version, type->tp_version_tag); - _py_set_opcode(instr, LOAD_ATTR_SLOT); + instr->op.code = LOAD_ATTR_SLOT; goto success; } case DUNDER_CLASS: @@ -808,7 +808,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) assert(offset == (uint16_t)offset); cache->index = (uint16_t)offset; write_u32(cache->version, type->tp_version_tag); - _py_set_opcode(instr, LOAD_ATTR_SLOT); + instr->op.code = LOAD_ATTR_SLOT; goto success; } case OTHER_SLOT: @@ -836,7 +836,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) /* borrowed */ write_obj(lm_cache->descr, descr); write_u32(lm_cache->type_version, type->tp_version_tag); - _py_set_opcode(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + instr->op.code = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; goto success; } case BUILTIN_CLASSMETHOD: @@ -867,7 +867,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) fail: STAT_INC(LOAD_ATTR, failure); assert(!PyErr_Occurred()); - _py_set_opcode(instr, LOAD_ATTR); + instr->op.code = LOAD_ATTR; cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -927,7 +927,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) assert(offset > 0); cache->index = (uint16_t)offset; write_u32(cache->version, type->tp_version_tag); - _py_set_opcode(instr, STORE_ATTR_SLOT); + instr->op.code = STORE_ATTR_SLOT; goto success; } case DUNDER_CLASS: @@ -963,7 +963,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) fail: STAT_INC(STORE_ATTR, failure); assert(!PyErr_Occurred()); - _py_set_opcode(instr, STORE_ATTR); + instr->op.code = STORE_ATTR; cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1027,7 +1027,7 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, case NON_DESCRIPTOR: write_u32(cache->type_version, ((PyTypeObject *)owner)->tp_version_tag); write_obj(cache->descr, descr); - _py_set_opcode(instr, LOAD_ATTR_CLASS); + instr->op.code = LOAD_ATTR_CLASS; return 0; #ifdef Py_STATS case ABSENT: @@ -1069,7 +1069,7 @@ PyObject *descr, DescriptorClassification kind) return 0; } write_u32(cache->keys_version, keys_version); - _py_set_opcode(instr, LOAD_ATTR_METHOD_WITH_VALUES); + instr->op.code = LOAD_ATTR_METHOD_WITH_VALUES; } else { Py_ssize_t dictoffset = owner_cls->tp_dictoffset; @@ -1078,7 +1078,7 @@ PyObject *descr, DescriptorClassification kind) return 0; } if (dictoffset == 0) { - _py_set_opcode(instr, LOAD_ATTR_METHOD_NO_DICT); + instr->op.code = LOAD_ATTR_METHOD_NO_DICT; } else { PyObject *dict = *(PyObject **) ((char *)owner + dictoffset); @@ -1088,7 +1088,7 @@ PyObject *descr, DescriptorClassification kind) } assert(owner_cls->tp_dictoffset > 0); assert(owner_cls->tp_dictoffset <= INT16_MAX); - _py_set_opcode(instr, LOAD_ATTR_METHOD_LAZY_DICT); + instr->op.code = LOAD_ATTR_METHOD_LAZY_DICT; } } /* `descr` is borrowed. This is safe for methods (even inherited ones from @@ -1146,7 +1146,7 @@ _Py_Specialize_LoadGlobal( } cache->index = (uint16_t)index; write_u32(cache->module_keys_version, keys_version); - _py_set_opcode(instr, LOAD_GLOBAL_MODULE); + instr->op.code = LOAD_GLOBAL_MODULE; goto success; } if (!PyDict_CheckExact(builtins)) { @@ -1184,12 +1184,12 @@ _Py_Specialize_LoadGlobal( cache->index = (uint16_t)index; write_u32(cache->module_keys_version, globals_version); cache->builtin_keys_version = (uint16_t)builtins_version; - _py_set_opcode(instr, LOAD_GLOBAL_BUILTIN); + instr->op.code = LOAD_GLOBAL_BUILTIN; goto success; fail: STAT_INC(LOAD_GLOBAL, failure); assert(!PyErr_Occurred()); - _py_set_opcode(instr, LOAD_GLOBAL); + instr->op.code = LOAD_GLOBAL; cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1295,7 +1295,7 @@ _Py_Specialize_BinarySubscr( if (container_type == &PyList_Type) { if (PyLong_CheckExact(sub)) { if (Py_SIZE(sub) == 0 || Py_SIZE(sub) == 1) { - _py_set_opcode(instr, BINARY_SUBSCR_LIST_INT); + instr->op.code = BINARY_SUBSCR_LIST_INT; goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); @@ -1308,7 +1308,7 @@ _Py_Specialize_BinarySubscr( if (container_type == &PyTuple_Type) { if (PyLong_CheckExact(sub)) { if (Py_SIZE(sub) == 0 || Py_SIZE(sub) == 1) { - _py_set_opcode(instr, BINARY_SUBSCR_TUPLE_INT); + instr->op.code = BINARY_SUBSCR_TUPLE_INT; goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); @@ -1319,7 +1319,7 @@ _Py_Specialize_BinarySubscr( goto fail; } if (container_type == &PyDict_Type) { - _py_set_opcode(instr, BINARY_SUBSCR_DICT); + instr->op.code = BINARY_SUBSCR_DICT; goto success; } PyTypeObject *cls = Py_TYPE(container); @@ -1350,7 +1350,7 @@ _Py_Specialize_BinarySubscr( } cache->func_version = version; ((PyHeapTypeObject *)container_type)->_spec_cache.getitem = descriptor; - _py_set_opcode(instr, BINARY_SUBSCR_GETITEM); + instr->op.code = BINARY_SUBSCR_GETITEM; goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, @@ -1358,7 +1358,7 @@ _Py_Specialize_BinarySubscr( fail: STAT_INC(BINARY_SUBSCR, failure); assert(!PyErr_Occurred()); - _py_set_opcode(instr, BINARY_SUBSCR); + instr->op.code = BINARY_SUBSCR; cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1378,7 +1378,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins if ((Py_SIZE(sub) == 0 || Py_SIZE(sub) == 1) && ((PyLongObject *)sub)->long_value.ob_digit[0] < (size_t)PyList_GET_SIZE(container)) { - _py_set_opcode(instr, STORE_SUBSCR_LIST_INT); + instr->op.code = STORE_SUBSCR_LIST_INT; goto success; } else { @@ -1396,8 +1396,8 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins } } if (container_type == &PyDict_Type) { - _py_set_opcode(instr, STORE_SUBSCR_DICT); - goto success; + instr->op.code = STORE_SUBSCR_DICT; + goto success; } #ifdef Py_STATS PyMappingMethods *as_mapping = container_type->tp_as_mapping; @@ -1463,7 +1463,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins fail: STAT_INC(STORE_SUBSCR, failure); assert(!PyErr_Occurred()); - _py_set_opcode(instr, STORE_SUBSCR); + instr->op.code = STORE_SUBSCR; cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1482,23 +1482,23 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, return -1; } if (tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { - int oparg = _Py_OPARG(*instr); + int oparg = instr->op.arg; if (nargs == 1 && kwnames == NULL && oparg == 1) { if (tp == &PyUnicode_Type) { - _py_set_opcode(instr, CALL_NO_KW_STR_1); + instr->op.code = CALL_NO_KW_STR_1; return 0; } else if (tp == &PyType_Type) { - _py_set_opcode(instr, CALL_NO_KW_TYPE_1); + instr->op.code = CALL_NO_KW_TYPE_1; return 0; } else if (tp == &PyTuple_Type) { - _py_set_opcode(instr, CALL_NO_KW_TUPLE_1); + instr->op.code = CALL_NO_KW_TUPLE_1; return 0; } } if (tp->tp_vectorcall != NULL) { - _py_set_opcode(instr, CALL_BUILTIN_CLASS); + instr->op.code = CALL_BUILTIN_CLASS; return 0; } SPECIALIZATION_FAIL(CALL, tp == &PyUnicode_Type ? @@ -1573,7 +1573,7 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); return -1; } - _py_set_opcode(instr, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS); + instr->op.code = CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS; return 0; } case METH_O: { @@ -1584,21 +1584,21 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, PyInterpreterState *interp = _PyInterpreterState_GET(); PyObject *list_append = interp->callable_cache.list_append; _Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_CALL + 1]; - bool pop = (_Py_OPCODE(next) == POP_TOP); - int oparg = _Py_OPARG(*instr); + bool pop = (next.op.code == POP_TOP); + int oparg = instr->op.arg; if ((PyObject *)descr == list_append && oparg == 1 && pop) { - _py_set_opcode(instr, CALL_NO_KW_LIST_APPEND); + instr->op.code = CALL_NO_KW_LIST_APPEND; return 0; } - _py_set_opcode(instr, CALL_NO_KW_METHOD_DESCRIPTOR_O); + instr->op.code = CALL_NO_KW_METHOD_DESCRIPTOR_O; return 0; } case METH_FASTCALL: { - _py_set_opcode(instr, CALL_NO_KW_METHOD_DESCRIPTOR_FAST); + instr->op.code = CALL_NO_KW_METHOD_DESCRIPTOR_FAST; return 0; } case METH_FASTCALL | METH_KEYWORDS: { - _py_set_opcode(instr, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + instr->op.code = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; return 0; } } @@ -1649,14 +1649,14 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, write_u32(cache->func_version, version); cache->min_args = min_args; if (argcount == nargs) { - _py_set_opcode(instr, bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS); + instr->op.code = bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS; } else if (bound_method) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD); return -1; } else { - _py_set_opcode(instr, CALL_PY_WITH_DEFAULTS); + instr->op.code = CALL_PY_WITH_DEFAULTS; } return 0; } @@ -1683,10 +1683,10 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, /* len(o) */ PyInterpreterState *interp = _PyInterpreterState_GET(); if (callable == interp->callable_cache.len) { - _py_set_opcode(instr, CALL_NO_KW_LEN); + instr->op.code = CALL_NO_KW_LEN; return 0; } - _py_set_opcode(instr, CALL_NO_KW_BUILTIN_O); + instr->op.code = CALL_NO_KW_BUILTIN_O; return 0; } case METH_FASTCALL: { @@ -1698,15 +1698,15 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, /* isinstance(o1, o2) */ PyInterpreterState *interp = _PyInterpreterState_GET(); if (callable == interp->callable_cache.isinstance) { - _py_set_opcode(instr, CALL_NO_KW_ISINSTANCE); + instr->op.code = CALL_NO_KW_ISINSTANCE; return 0; } } - _py_set_opcode(instr, CALL_NO_KW_BUILTIN_FAST); + instr->op.code = CALL_NO_KW_BUILTIN_FAST; return 0; } case METH_FASTCALL | METH_KEYWORDS: { - _py_set_opcode(instr, CALL_BUILTIN_FAST_WITH_KEYWORDS); + instr->op.code = CALL_BUILTIN_FAST_WITH_KEYWORDS; return 0; } default: @@ -1785,7 +1785,7 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, if (fail) { STAT_INC(CALL, failure); assert(!PyErr_Occurred()); - _py_set_opcode(instr, CALL); + instr->op.code = CALL; cache->counter = adaptive_counter_backoff(cache->counter); } else { @@ -1880,21 +1880,21 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, } if (PyUnicode_CheckExact(lhs)) { _Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_BINARY_OP + 1]; - bool to_store = (_Py_OPCODE(next) == STORE_FAST || - _Py_OPCODE(next) == STORE_FAST__LOAD_FAST); - if (to_store && locals[_Py_OPARG(next)] == lhs) { - _py_set_opcode(instr, BINARY_OP_INPLACE_ADD_UNICODE); + bool to_store = (next.op.code == STORE_FAST || + next.op.code == STORE_FAST__LOAD_FAST); + if (to_store && locals[next.op.arg] == lhs) { + instr->op.code = BINARY_OP_INPLACE_ADD_UNICODE; goto success; } - _py_set_opcode(instr, BINARY_OP_ADD_UNICODE); + instr->op.code = BINARY_OP_ADD_UNICODE; goto success; } if (PyLong_CheckExact(lhs)) { - _py_set_opcode(instr, BINARY_OP_ADD_INT); + instr->op.code = BINARY_OP_ADD_INT; goto success; } if (PyFloat_CheckExact(lhs)) { - _py_set_opcode(instr, BINARY_OP_ADD_FLOAT); + instr->op.code = BINARY_OP_ADD_FLOAT; goto success; } break; @@ -1904,11 +1904,11 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, break; } if (PyLong_CheckExact(lhs)) { - _py_set_opcode(instr, BINARY_OP_MULTIPLY_INT); + instr->op.code = BINARY_OP_MULTIPLY_INT; goto success; } if (PyFloat_CheckExact(lhs)) { - _py_set_opcode(instr, BINARY_OP_MULTIPLY_FLOAT); + instr->op.code = BINARY_OP_MULTIPLY_FLOAT; goto success; } break; @@ -1918,18 +1918,18 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, break; } if (PyLong_CheckExact(lhs)) { - _py_set_opcode(instr, BINARY_OP_SUBTRACT_INT); + instr->op.code = BINARY_OP_SUBTRACT_INT; goto success; } if (PyFloat_CheckExact(lhs)) { - _py_set_opcode(instr, BINARY_OP_SUBTRACT_FLOAT); + instr->op.code = BINARY_OP_SUBTRACT_FLOAT; goto success; } break; } SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs)); STAT_INC(BINARY_OP, failure); - _py_set_opcode(instr, BINARY_OP); + instr->op.code = BINARY_OP; cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1981,7 +1981,7 @@ _Py_Specialize_CompareAndBranch(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *inst assert(_PyOpcode_Caches[COMPARE_AND_BRANCH] == INLINE_CACHE_ENTRIES_COMPARE_OP); _PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1); #ifndef NDEBUG - int next_opcode = _Py_OPCODE(instr[INLINE_CACHE_ENTRIES_COMPARE_OP + 1]); + int next_opcode = instr[INLINE_CACHE_ENTRIES_COMPARE_OP + 1].op.code; assert(next_opcode == POP_JUMP_IF_FALSE || next_opcode == POP_JUMP_IF_TRUE); #endif if (Py_TYPE(lhs) != Py_TYPE(rhs)) { @@ -1989,12 +1989,12 @@ _Py_Specialize_CompareAndBranch(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *inst goto failure; } if (PyFloat_CheckExact(lhs)) { - _py_set_opcode(instr, COMPARE_AND_BRANCH_FLOAT); + instr->op.code = COMPARE_AND_BRANCH_FLOAT; goto success; } if (PyLong_CheckExact(lhs)) { if (Py_ABS(Py_SIZE(lhs)) <= 1 && Py_ABS(Py_SIZE(rhs)) <= 1) { - _py_set_opcode(instr, COMPARE_AND_BRANCH_INT); + instr->op.code = COMPARE_AND_BRANCH_INT; goto success; } else { @@ -2009,14 +2009,14 @@ _Py_Specialize_CompareAndBranch(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *inst goto failure; } else { - _py_set_opcode(instr, COMPARE_AND_BRANCH_STR); + instr->op.code = COMPARE_AND_BRANCH_STR; goto success; } } SPECIALIZATION_FAIL(COMPARE_AND_BRANCH, compare_op_fail_kind(lhs, rhs)); failure: STAT_INC(COMPARE_AND_BRANCH, failure); - _py_set_opcode(instr, COMPARE_AND_BRANCH); + instr->op.code = COMPARE_AND_BRANCH; cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -2051,10 +2051,10 @@ _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) goto failure; } if (PyTuple_GET_SIZE(seq) == 2) { - _py_set_opcode(instr, UNPACK_SEQUENCE_TWO_TUPLE); + instr->op.code = UNPACK_SEQUENCE_TWO_TUPLE; goto success; } - _py_set_opcode(instr, UNPACK_SEQUENCE_TUPLE); + instr->op.code = UNPACK_SEQUENCE_TUPLE; goto success; } if (PyList_CheckExact(seq)) { @@ -2062,13 +2062,13 @@ _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) SPECIALIZATION_FAIL(UNPACK_SEQUENCE, SPEC_FAIL_EXPECTED_ERROR); goto failure; } - _py_set_opcode(instr, UNPACK_SEQUENCE_LIST); + instr->op.code = UNPACK_SEQUENCE_LIST; goto success; } SPECIALIZATION_FAIL(UNPACK_SEQUENCE, unpack_sequence_fail_kind(seq)); failure: STAT_INC(UNPACK_SEQUENCE, failure); - _py_set_opcode(instr, UNPACK_SEQUENCE); + instr->op.code = UNPACK_SEQUENCE; cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -2156,28 +2156,28 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) _PyForIterCache *cache = (_PyForIterCache *)(instr + 1); PyTypeObject *tp = Py_TYPE(iter); _Py_CODEUNIT next = instr[1+INLINE_CACHE_ENTRIES_FOR_ITER]; - int next_op = _PyOpcode_Deopt[_Py_OPCODE(next)]; + int next_op = _PyOpcode_Deopt[next.op.code]; if (tp == &PyListIter_Type) { - _py_set_opcode(instr, FOR_ITER_LIST); + instr->op.code = FOR_ITER_LIST; goto success; } else if (tp == &PyTupleIter_Type) { - _py_set_opcode(instr, FOR_ITER_TUPLE); + instr->op.code = FOR_ITER_TUPLE; goto success; } else if (tp == &PyRangeIter_Type && next_op == STORE_FAST) { - _py_set_opcode(instr, FOR_ITER_RANGE); + instr->op.code = FOR_ITER_RANGE; goto success; } else if (tp == &PyGen_Type && oparg <= SHRT_MAX) { - assert(_Py_OPCODE(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1]) == END_FOR); - _py_set_opcode(instr, FOR_ITER_GEN); + assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == END_FOR); + instr->op.code = FOR_ITER_GEN; goto success; } SPECIALIZATION_FAIL(FOR_ITER, _PySpecialization_ClassifyIterator(iter)); STAT_INC(FOR_ITER, failure); - _py_set_opcode(instr, FOR_ITER); + instr->op.code = FOR_ITER; cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -2193,13 +2193,13 @@ _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr) _PySendCache *cache = (_PySendCache *)(instr + 1); PyTypeObject *tp = Py_TYPE(receiver); if (tp == &PyGen_Type || tp == &PyCoro_Type) { - _py_set_opcode(instr, SEND_GEN); + instr->op.code = SEND_GEN; goto success; } SPECIALIZATION_FAIL(SEND, _PySpecialization_ClassifyIterator(receiver)); STAT_INC(SEND, failure); - _py_set_opcode(instr, SEND); + instr->op.code = SEND; cache->counter = adaptive_counter_backoff(cache->counter); return; success: diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index aa8e14075c8738..c7f52b55a5eb99 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -8,6 +8,7 @@ import contextlib import dataclasses import os +import posixpath import re import sys import typing @@ -17,7 +18,7 @@ HERE = os.path.dirname(__file__) ROOT = os.path.join(HERE, "../..") -THIS = os.path.relpath(__file__, ROOT) +THIS = os.path.relpath(__file__, ROOT).replace(os.path.sep, posixpath.sep) DEFAULT_INPUT = os.path.relpath(os.path.join(ROOT, "Python/bytecodes.c")) DEFAULT_OUTPUT = os.path.relpath(os.path.join(ROOT, "Python/generated_cases.c.h")) @@ -930,7 +931,7 @@ def write_metadata(self) -> None: with open(self.metadata_filename, "w") as f: # Write provenance header f.write(f"// This file is generated by {THIS} --metadata\n") - f.write(f"// from {os.path.relpath(self.filename, ROOT)}\n") + f.write(f"// from {os.path.relpath(self.filename, ROOT).replace(os.path.sep, posixpath.sep)}\n") f.write(f"// Do not edit!\n") # Create formatter; the rest of the code uses this @@ -1009,7 +1010,7 @@ def write_instructions(self) -> None: with open(self.output_filename, "w") as f: # Write provenance header f.write(f"// This file is generated by {THIS}\n") - f.write(f"// from {os.path.relpath(self.filename, ROOT)}\n") + f.write(f"// from {os.path.relpath(self.filename, ROOT).replace(os.path.sep, posixpath.sep)}\n") f.write(f"// Do not edit!\n") # Create formatter; the rest of the code uses this From ed01addb59a554804995303ad3e7bf0c6067737b Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Tue, 21 Feb 2023 00:20:18 +0900 Subject: [PATCH 55/76] gh-101981: Apply HOMEBREW related environment variables (gh-102074) --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index acc8d936774af5..eec11e25a7c7f6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -154,6 +154,9 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: + HOMEBREW_NO_ANALYTICS: 1 + HOMEBREW_NO_AUTO_UPDATE: 1 + HOMEBREW_NO_INSTALL_CLEANUP: 1 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v3 From 59e86caca812fc993c5eb7dc8ccd1508ffccba86 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Mon, 20 Feb 2023 09:07:03 -0800 Subject: [PATCH 56/76] gh-88233: zipfile: handle extras after a zip64 extra (GH-96161) Previously, any data _after_ the zip64 extra would be removed. With many new tests. Fixes #88233 Automerge-Triggered-By: GH:jaraco --- Lib/test/test_zipfile/test_core.py | 62 +++++++++++++++++++ Lib/zipfile/__init__.py | 2 + ...2-09-05-12-17-34.gh-issue-88233.gff9qJ.rst | 2 + 3 files changed, 66 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2022-09-05-12-17-34.gh-issue-88233.gff9qJ.rst diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index cf41d0e8cb8d53..e23f5c2a8556f2 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -3010,5 +3010,67 @@ def test_cli_with_metadata_encoding_extract(self): self.assertIn(name, listing) +class StripExtraTests(unittest.TestCase): + # Note: all of the "z" characters are technically invalid, but up + # to 3 bytes at the end of the extra will be passed through as they + # are too short to encode a valid extra. + + ZIP64_EXTRA = 1 + + def test_no_data(self): + s = struct.Struct(" Date: Mon, 20 Feb 2023 19:21:10 +0000 Subject: [PATCH 57/76] GH-99818: improve the documentation for zipfile.Path and Traversable (GH-101589) Automerge-Triggered-By: GH:FFY00 --- Doc/library/importlib.resources.abc.rst | 5 ++++- Doc/library/zipfile.rst | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index 7747e89a833c02..2d0f137ffc7996 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -89,9 +89,12 @@ .. class:: Traversable - An object with a subset of pathlib.Path methods suitable for + An object with a subset of :class:`pathlib.Path` methods suitable for traversing directories and opening files. + For a representation of the object on the file-system, use + :meth:`importlib.resources.as_file`. + .. versionadded:: 3.9 .. deprecated-removed:: 3.12 3.14 diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 1c516723f04a33..0195abc3a992c1 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -55,8 +55,9 @@ The module defines the following items: .. class:: Path :noindex: - A pathlib-compatible wrapper for zip files. See section - :ref:`path-objects` for details. + Class that implements a subset of the interface provided by + :class:`pathlib.Path`, including the full + :class:`importlib.resources.abc.Traversable` interface. .. versionadded:: 3.8 From 36854bbb240e417c0df6f0014924fcc899388186 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 20 Feb 2023 16:01:58 -0500 Subject: [PATCH 58/76] gh-101566: Sync with zipp 3.14. (GH-102018) --- Lib/test/test_zipfile/_context.py | 30 ++++ Lib/test/test_zipfile/_func_timeout_compat.py | 8 + Lib/test/test_zipfile/_itertools.py | 29 ++++ Lib/test/test_zipfile/test_path.py | 137 +++++++++++------- Lib/zipfile/_path.py | 63 +++++++- ...-02-17-20-24-15.gh-issue-101566.FjgWBt.rst | 4 + 6 files changed, 215 insertions(+), 56 deletions(-) create mode 100644 Lib/test/test_zipfile/_context.py create mode 100644 Lib/test/test_zipfile/_func_timeout_compat.py create mode 100644 Misc/NEWS.d/next/Library/2023-02-17-20-24-15.gh-issue-101566.FjgWBt.rst diff --git a/Lib/test/test_zipfile/_context.py b/Lib/test/test_zipfile/_context.py new file mode 100644 index 00000000000000..348798aa08d452 --- /dev/null +++ b/Lib/test/test_zipfile/_context.py @@ -0,0 +1,30 @@ +import contextlib +import time + + +class DeadlineExceeded(Exception): + pass + + +class TimedContext(contextlib.ContextDecorator): + """ + A context that will raise DeadlineExceeded if the + max duration is reached during the execution. + + >>> TimedContext(1)(time.sleep)(.1) + >>> TimedContext(0)(time.sleep)(.1) + Traceback (most recent call last): + ... + tests._context.DeadlineExceeded: (..., 0) + """ + + def __init__(self, max_duration: int): + self.max_duration = max_duration + + def __enter__(self): + self.start = time.monotonic() + + def __exit__(self, *err): + duration = time.monotonic() - self.start + if duration > self.max_duration: + raise DeadlineExceeded(duration, self.max_duration) diff --git a/Lib/test/test_zipfile/_func_timeout_compat.py b/Lib/test/test_zipfile/_func_timeout_compat.py new file mode 100644 index 00000000000000..b1f2b2698a8538 --- /dev/null +++ b/Lib/test/test_zipfile/_func_timeout_compat.py @@ -0,0 +1,8 @@ +try: + from func_timeout import func_set_timeout as set_timeout +except ImportError: # pragma: no cover + # provide a fallback that doesn't actually time out + from ._context import TimedContext as set_timeout + + +__all__ = ['set_timeout'] diff --git a/Lib/test/test_zipfile/_itertools.py b/Lib/test/test_zipfile/_itertools.py index 559f3f111b88a3..74f01fe5ba3de2 100644 --- a/Lib/test/test_zipfile/_itertools.py +++ b/Lib/test/test_zipfile/_itertools.py @@ -1,3 +1,32 @@ +import itertools + + +# from jaraco.itertools 6.3.0 +class Counter: + """ + Wrap an iterable in an object that stores the count of items + that pass through it. + + >>> items = Counter(range(20)) + >>> items.count + 0 + >>> values = list(items) + >>> items.count + 20 + """ + + def __init__(self, i): + self.count = 0 + self.iter = zip(itertools.count(1), i) + + def __iter__(self): + return self + + def __next__(self): + self.count, result = next(self.iter) + return result + + # from more_itertools v8.13.0 def always_iterable(obj, base_type=(str, bytes)): if obj is None: diff --git a/Lib/test/test_zipfile/test_path.py b/Lib/test/test_zipfile/test_path.py index 3086fd2080a97d..92fda690b2fc50 100644 --- a/Lib/test/test_zipfile/test_path.py +++ b/Lib/test/test_zipfile/test_path.py @@ -4,36 +4,25 @@ import pathlib import pickle import string -from test.support.script_helper import assert_python_ok +import sys import unittest import zipfile -from ._test_params import parameterize, Invoked from ._functools import compose +from ._itertools import Counter +from ._test_params import parameterize, Invoked +from ._func_timeout_compat import set_timeout from test.support.os_helper import temp_dir -# Poor man's technique to consume a (smallish) iterable. -consume = tuple - - -# from jaraco.itertools 5.0 class jaraco: class itertools: - class Counter: - def __init__(self, i): - self.count = 0 - self._orig_iter = iter(i) + Counter = Counter - def __iter__(self): - return self - def __next__(self): - result = next(self._orig_iter) - self.count += 1 - return result +consume = tuple def add_dirs(zf): @@ -161,10 +150,10 @@ def test_open_encoding_utf16(self): u16 = path.joinpath("16.txt") with u16.open('r', "utf-16") as strm: data = strm.read() - self.assertEqual(data, "This was utf-16") + assert data == "This was utf-16" with u16.open(encoding="utf-16") as strm: data = strm.read() - self.assertEqual(data, "This was utf-16") + assert data == "This was utf-16" def test_open_encoding_errors(self): in_memory_file = io.BytesIO() @@ -177,9 +166,9 @@ def test_open_encoding_errors(self): # encoding= as a positional argument for gh-101144. data = u16.read_text("utf-8", errors="ignore") - self.assertEqual(data, "invalid utf-8: .") + assert data == "invalid utf-8: ." with u16.open("r", "utf-8", errors="surrogateescape") as f: - self.assertEqual(f.read(), "invalid utf-8: \udcff\udcff.") + assert f.read() == "invalid utf-8: \udcff\udcff." # encoding= both positional and keyword is an error; gh-101144. with self.assertRaisesRegex(TypeError, "encoding"): @@ -191,24 +180,21 @@ def test_open_encoding_errors(self): with self.assertRaises(UnicodeDecodeError): f.read() - def test_encoding_warnings(self): + @unittest.skipIf( + not getattr(sys.flags, 'warn_default_encoding', 0), + "Requires warn_default_encoding", + ) + @pass_alpharep + def test_encoding_warnings(self, alpharep): """EncodingWarning must blame the read_text and open calls.""" - code = '''\ -import io, zipfile -with zipfile.ZipFile(io.BytesIO(), "w") as zf: - zf.filename = '' - zf.writestr("path/file.txt", b"Spanish Inquisition") - root = zipfile.Path(zf) - (path,) = root.iterdir() - file_path = path.joinpath("file.txt") - unused = file_path.read_text() # should warn - file_path.open("r").close() # should warn -''' - proc = assert_python_ok('-X', 'warn_default_encoding', '-c', code) - warnings = proc.err.splitlines() - self.assertEqual(len(warnings), 2, proc.err) - self.assertRegex(warnings[0], rb"^:8: EncodingWarning:") - self.assertRegex(warnings[1], rb"^:9: EncodingWarning:") + assert sys.flags.warn_default_encoding + root = zipfile.Path(alpharep) + with self.assertWarns(EncodingWarning) as wc: + root.joinpath("a.txt").read_text() + assert __file__ == wc.filename + with self.assertWarns(EncodingWarning) as wc: + root.joinpath("a.txt").open("r").close() + assert __file__ == wc.filename def test_open_write(self): """ @@ -250,7 +236,8 @@ def test_read(self, alpharep): root = zipfile.Path(alpharep) a, b, g = root.iterdir() assert a.read_text(encoding="utf-8") == "content of a" - a.read_text("utf-8") # No positional arg TypeError per gh-101144. + # Also check positional encoding arg (gh-101144). + assert a.read_text("utf-8") == "content of a" assert a.read_bytes() == b"content of a" @pass_alpharep @@ -275,19 +262,6 @@ def test_traverse_truediv(self, alpharep): e = root / "b" / "d" / "e.txt" assert e.read_text(encoding="utf-8") == "content of e" - @pass_alpharep - def test_traverse_simplediv(self, alpharep): - """ - Disable the __future__.division when testing traversal. - """ - code = compile( - source="zipfile.Path(alpharep) / 'a'", - filename="(test)", - mode="eval", - dont_inherit=True, - ) - eval(code) - @pass_alpharep def test_pathlike_construction(self, alpharep): """ @@ -356,7 +330,7 @@ def test_joinpath_constant_time(self): # Check the file iterated all items assert entries.count == self.HUGE_ZIPFILE_NUM_ENTRIES - # @func_timeout.func_set_timeout(3) + @set_timeout(3) def test_implied_dirs_performance(self): data = ['/'.join(string.ascii_lowercase + str(n)) for n in range(10000)] zipfile.CompleteDirs._implied_dirs(data) @@ -472,6 +446,52 @@ def test_root_unnamed(self, alpharep): assert sub.name == "b" assert sub.parent + @pass_alpharep + def test_match_and_glob(self, alpharep): + root = zipfile.Path(alpharep) + assert not root.match("*.txt") + + assert list(root.glob("b/c.*")) == [zipfile.Path(alpharep, "b/c.txt")] + + files = root.glob("**/*.txt") + assert all(each.match("*.txt") for each in files) + + assert list(root.glob("**/*.txt")) == list(root.rglob("*.txt")) + + def test_glob_empty(self): + root = zipfile.Path(zipfile.ZipFile(io.BytesIO(), 'w')) + with self.assertRaises(ValueError): + root.glob('') + + @pass_alpharep + def test_eq_hash(self, alpharep): + root = zipfile.Path(alpharep) + assert root == zipfile.Path(alpharep) + + assert root != (root / "a.txt") + assert (root / "a.txt") == (root / "a.txt") + + root = zipfile.Path(alpharep) + assert root in {root} + + @pass_alpharep + def test_is_symlink(self, alpharep): + """ + See python/cpython#82102 for symlink support beyond this object. + """ + + root = zipfile.Path(alpharep) + assert not root.is_symlink() + + @pass_alpharep + def test_relative_to(self, alpharep): + root = zipfile.Path(alpharep) + relative = root.joinpath("b", "c.txt").relative_to(root / "b") + assert str(relative) == "c.txt" + + relative = root.joinpath("b", "d", "e.txt").relative_to(root / "b") + assert str(relative) == "d/e.txt" + @pass_alpharep def test_inheritance(self, alpharep): cls = type('PathChild', (zipfile.Path,), {}) @@ -493,3 +513,14 @@ def test_pickle(self, alpharep, path_type, subpath): restored_1 = pickle.loads(saved_1) first, *rest = restored_1.iterdir() assert first.read_text().startswith('content of ') + + @pass_alpharep + def test_extract_orig_with_implied_dirs(self, alpharep): + """ + A zip file wrapped in a Path should extract even with implied dirs. + """ + source_path = self.zipfile_ondisk(alpharep) + zf = zipfile.ZipFile(source_path) + # wrap the zipfile for its side effect + zipfile.Path(zf) + zf.extractall(source_path.parent) diff --git a/Lib/zipfile/_path.py b/Lib/zipfile/_path.py index 7c7a6a0e2c0d32..c2c804e96d6b6c 100644 --- a/Lib/zipfile/_path.py +++ b/Lib/zipfile/_path.py @@ -4,6 +4,8 @@ import itertools import contextlib import pathlib +import re +import fnmatch __all__ = ['Path'] @@ -93,7 +95,7 @@ def _implied_dirs(names): return _dedupe(_difference(as_dirs, names)) def namelist(self): - names = super(CompleteDirs, self).namelist() + names = super().namelist() return names + list(self._implied_dirs(names)) def _name_set(self): @@ -109,6 +111,17 @@ def resolve_dir(self, name): dir_match = name not in names and dirname in names return dirname if dir_match else name + def getinfo(self, name): + """ + Supplement getinfo for implied dirs. + """ + try: + return super().getinfo(name) + except KeyError: + if not name.endswith('/') or name not in self._name_set(): + raise + return zipfile.ZipInfo(filename=name) + @classmethod def make(cls, source): """ @@ -138,13 +151,13 @@ class FastLookup(CompleteDirs): def namelist(self): with contextlib.suppress(AttributeError): return self.__names - self.__names = super(FastLookup, self).namelist() + self.__names = super().namelist() return self.__names def _name_set(self): with contextlib.suppress(AttributeError): return self.__lookup - self.__lookup = super(FastLookup, self)._name_set() + self.__lookup = super()._name_set() return self.__lookup @@ -246,6 +259,18 @@ def __init__(self, root, at=""): self.root = FastLookup.make(root) self.at = at + def __eq__(self, other): + """ + >>> Path(zipfile.ZipFile(io.BytesIO(), 'w')) == 'foo' + False + """ + if self.__class__ is not other.__class__: + return NotImplemented + return (self.root, self.at) == (other.root, other.at) + + def __hash__(self): + return hash((self.root, self.at)) + def open(self, mode='r', *args, pwd=None, **kwargs): """ Open this entry as text or binary following the semantics @@ -316,6 +341,38 @@ def iterdir(self): subs = map(self._next, self.root.namelist()) return filter(self._is_child, subs) + def match(self, path_pattern): + return pathlib.Path(self.at).match(path_pattern) + + def is_symlink(self): + """ + Return whether this path is a symlink. Always false (python/cpython#82102). + """ + return False + + def _descendants(self): + for child in self.iterdir(): + yield child + if child.is_dir(): + yield from child._descendants() + + def glob(self, pattern): + if not pattern: + raise ValueError(f"Unacceptable pattern: {pattern!r}") + + matches = re.compile(fnmatch.translate(pattern)).fullmatch + return ( + child + for child in self._descendants() + if matches(str(child.relative_to(self))) + ) + + def rglob(self, pattern): + return self.glob(f'**/{pattern}') + + def relative_to(self, other, *extra): + return posixpath.relpath(str(self), str(other.joinpath(*extra))) + def __str__(self): return posixpath.join(self.root.filename, self.at) diff --git a/Misc/NEWS.d/next/Library/2023-02-17-20-24-15.gh-issue-101566.FjgWBt.rst b/Misc/NEWS.d/next/Library/2023-02-17-20-24-15.gh-issue-101566.FjgWBt.rst new file mode 100644 index 00000000000000..5fc1a0288a82dc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-17-20-24-15.gh-issue-101566.FjgWBt.rst @@ -0,0 +1,4 @@ +In zipfile, sync Path with `zipp 3.14 +`_, including +fix for extractall on the underlying zipfile after being wrapped in +``Path``. From 4d3bc89a3f54c4f09756a9b644b3912bf54191a7 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 20 Feb 2023 21:54:19 +0000 Subject: [PATCH 59/76] gh-102011: use sys.exception() instead of sys.exc_info() in docs where possible (#102012) --- Doc/library/exceptions.rst | 2 +- Doc/library/traceback.rst | 21 ++++++++++---------- Doc/library/types.rst | 2 +- Doc/library/wsgiref.rst | 2 +- Doc/reference/compound_stmts.rst | 33 +++++++++++++++----------------- Doc/reference/datamodel.rst | 1 + 6 files changed, 29 insertions(+), 32 deletions(-) diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 1217b817b4e843..4a57e9c8799336 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -123,7 +123,7 @@ The following exceptions are used mostly as base classes for other exceptions. try: ... except SomeException: - tb = sys.exc_info()[2] + tb = sys.exception().__traceback__ raise OtherException(...).with_traceback(tb) .. method:: add_note(note) diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 69818baf184d7c..561c85290463ef 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -16,9 +16,8 @@ interpreter. .. index:: object: traceback -The module uses traceback objects --- this is the object type that is stored in -the :data:`sys.last_traceback` variable and returned as the third item from -:func:`sys.exc_info`. +The module uses traceback objects --- these are objects of type :class:`types.TracebackType`, +which are assigned to the ``__traceback__`` field of :class:`BaseException` instances. .. seealso:: @@ -81,7 +80,7 @@ The module defines the following functions: .. function:: print_exc(limit=None, file=None, chain=True) - This is a shorthand for ``print_exception(*sys.exc_info(), limit, file, + This is a shorthand for ``print_exception(sys.exception(), limit, file, chain)``. @@ -444,11 +443,11 @@ exception and traceback: try: lumberjack() except IndexError: - exc_type, exc_value, exc_traceback = sys.exc_info() + exc = sys.exception() print("*** print_tb:") - traceback.print_tb(exc_traceback, limit=1, file=sys.stdout) + traceback.print_tb(exc.__traceback__, limit=1, file=sys.stdout) print("*** print_exception:") - traceback.print_exception(exc_value, limit=2, file=sys.stdout) + traceback.print_exception(exc, limit=2, file=sys.stdout) print("*** print_exc:") traceback.print_exc(limit=2, file=sys.stdout) print("*** format_exc, first and last line:") @@ -456,12 +455,12 @@ exception and traceback: print(formatted_lines[0]) print(formatted_lines[-1]) print("*** format_exception:") - print(repr(traceback.format_exception(exc_value))) + print(repr(traceback.format_exception(exc))) print("*** extract_tb:") - print(repr(traceback.extract_tb(exc_traceback))) + print(repr(traceback.extract_tb(exc.__traceback__))) print("*** format_tb:") - print(repr(traceback.format_tb(exc_traceback))) - print("*** tb_lineno:", exc_traceback.tb_lineno) + print(repr(traceback.format_tb(exc.__traceback__))) + print("*** tb_lineno:", exc.__traceback__.tb_lineno) The output for the example would look similar to this: diff --git a/Doc/library/types.rst b/Doc/library/types.rst index cce0ad960edf97..415413c4cd9747 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -320,7 +320,7 @@ Standard names are defined for the following types: .. class:: TracebackType(tb_next, tb_frame, tb_lasti, tb_lineno) - The type of traceback objects such as found in ``sys.exc_info()[2]``. + The type of traceback objects such as found in ``sys.exception().__traceback__``. See :ref:`the language reference ` for details of the available attributes and operations, and guidance on creating tracebacks diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index 75dea466335163..39a4c1ba142338 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -674,7 +674,7 @@ input, output, and error streams. This method is a WSGI application to generate an error page for the user. It is only invoked if an error occurs before headers are sent to the client. - This method can access the current error information using ``sys.exc_info()``, + This method can access the current error using ``sys.exception()``, and should pass that information to *start_response* when calling it (as described in the "Error Handling" section of :pep:`3333`). diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index d5cb2899c231b4..52bb0b22b042f6 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -301,31 +301,28 @@ keeping all locals in that frame alive until the next garbage collection occurs. object: traceback Before an :keyword:`!except` clause's suite is executed, -details about the exception are -stored in the :mod:`sys` module and can be accessed via :func:`sys.exc_info`. -:func:`sys.exc_info` returns a 3-tuple consisting of the exception class, the -exception instance and a traceback object (see section :ref:`types`) identifying -the point in the program where the exception occurred. The details about the -exception accessed via :func:`sys.exc_info` are restored to their previous values -when leaving an exception handler:: - - >>> print(sys.exc_info()) - (None, None, None) +the exception is stored in the :mod:`sys` module, where it can be accessed +from within the body of the :keyword:`!except` clause by calling +:func:`sys.exception`. When leaving an exception handler, the exception +stored in the :mod:`sys` module is reset to its previous value:: + + >>> print(sys.exception()) + None >>> try: ... raise TypeError ... except: - ... print(sys.exc_info()) + ... print(repr(sys.exception())) ... try: ... raise ValueError ... except: - ... print(sys.exc_info()) - ... print(sys.exc_info()) + ... print(repr(sys.exception())) + ... print(repr(sys.exception())) ... - (, TypeError(), ) - (, ValueError(), ) - (, TypeError(), ) - >>> print(sys.exc_info()) - (None, None, None) + TypeError() + ValueError() + TypeError() + >>> print(sys.exception()) + None .. index:: diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index e4e471e50079ae..f447bbb1216d52 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1122,6 +1122,7 @@ Internal types single: exc_info (in module sys) single: last_traceback (in module sys) single: sys.exc_info + single: sys.exception single: sys.last_traceback Traceback objects represent a stack trace of an exception. A traceback object From 022b44f2546c44183e4df7b67e3e64502fae9143 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 20 Feb 2023 22:16:09 +0000 Subject: [PATCH 60/76] gh-102056: Fix a few bugs in error handling of exception printing code (#102078) --- Lib/test/test_threading.py | 31 +++++++++++++++++++ ...-02-20-15-18-33.gh-issue-102056.uHKuwH.rst | 1 + Python/pythonrun.c | 23 +++++++++----- 3 files changed, 48 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-20-15-18-33.gh-issue-102056.uHKuwH.rst diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 7fea2d38673eff..a39a267b403d83 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1523,6 +1523,37 @@ def run(): self.assertEqual(out, b'') self.assertNotIn("Unhandled exception", err.decode()) + def test_print_exception_gh_102056(self): + # This used to crash. See gh-102056. + script = r"""if True: + import time + import threading + import _thread + + def f(): + try: + f() + except RecursionError: + f() + + def g(): + try: + raise ValueError() + except* ValueError: + f() + + def h(): + time.sleep(1) + _thread.interrupt_main() + + t = threading.Thread(target=h) + t.start() + g() + t.join() + """ + + assert_python_failure("-c", script) + def test_bare_raise_in_brand_new_thread(self): def bare_raise(): raise diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-20-15-18-33.gh-issue-102056.uHKuwH.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-20-15-18-33.gh-issue-102056.uHKuwH.rst new file mode 100644 index 00000000000000..78cd525b365fe5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-20-15-18-33.gh-issue-102056.uHKuwH.rst @@ -0,0 +1 @@ +Fix error handling bugs in interpreter's exception printing code, which could cause a crash on infinite recursion. diff --git a/Python/pythonrun.c b/Python/pythonrun.c index ce993ea8796cb7..34a44dd92847ba 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1246,8 +1246,7 @@ print_chained(struct exception_print_context* ctx, PyObject *value, const char * message, const char *tag) { PyObject *f = ctx->file; - - if (_Py_EnterRecursiveCall(" in print_chained") < 0) { + if (_Py_EnterRecursiveCall(" in print_chained")) { return -1; } bool need_close = ctx->need_close; @@ -1374,7 +1373,9 @@ print_exception_group(struct exception_print_context *ctx, PyObject *value) if (ctx->exception_group_depth == 0) { ctx->exception_group_depth += 1; } - print_exception(ctx, value); + if (print_exception(ctx, value) < 0) { + return -1; + } PyObject *excs = ((PyBaseExceptionGroupObject *)value)->excs; assert(excs && PyTuple_Check(excs)); @@ -1424,7 +1425,7 @@ print_exception_group(struct exception_print_context *ctx, PyObject *value) PyObject *exc = PyTuple_GET_ITEM(excs, i); if (!truncated) { - if (_Py_EnterRecursiveCall(" in print_exception_group") != 0) { + if (_Py_EnterRecursiveCall(" in print_exception_group")) { return -1; } int res = print_exception_recursive(ctx, exc); @@ -1477,22 +1478,30 @@ print_exception_group(struct exception_print_context *ctx, PyObject *value) static int print_exception_recursive(struct exception_print_context *ctx, PyObject *value) { + if (_Py_EnterRecursiveCall(" in print_exception_recursive")) { + return -1; + } if (ctx->seen != NULL) { /* Exception chaining */ if (print_exception_cause_and_context(ctx, value) < 0) { - return -1; + goto error; } } if (!_PyBaseExceptionGroup_Check(value)) { if (print_exception(ctx, value) < 0) { - return -1; + goto error; } } else if (print_exception_group(ctx, value) < 0) { - return -1; + goto error; } assert(!PyErr_Occurred()); + + _Py_LeaveRecursiveCall(); return 0; +error: + _Py_LeaveRecursiveCall(); + return -1; } #define PyErr_MAX_GROUP_WIDTH 15 From 6f25657b83d7a680a97849490f6e973b3a695e1a Mon Sep 17 00:00:00 2001 From: Gihwan Kim Date: Tue, 21 Feb 2023 12:10:29 +0900 Subject: [PATCH 61/76] gh-101961 fileinput.hookcompressed should not set the encoding value for the binary mode (gh-102068) --- Lib/fileinput.py | 2 +- Lib/test/test_fileinput.py | 39 +++++++++++++------ Misc/ACKS | 1 + ...-02-21-10-05-33.gh-issue-101961.7e56jh.rst | 2 + 4 files changed, 32 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-21-10-05-33.gh-issue-101961.7e56jh.rst diff --git a/Lib/fileinput.py b/Lib/fileinput.py index e234dc9ea65f15..1b25f28f3d3432 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -399,7 +399,7 @@ def isstdin(self): def hook_compressed(filename, mode, *, encoding=None, errors=None): - if encoding is None: # EncodingWarning is emitted in FileInput() already. + if encoding is None and "b" not in mode: # EncodingWarning is emitted in FileInput() already. encoding = "locale" ext = os.path.splitext(filename)[1] if ext == '.gz': diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index ac20c74baa09e2..786d9186634305 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -855,29 +855,29 @@ def setUp(self): self.fake_open = InvocationRecorder() def test_empty_string(self): - self.do_test_use_builtin_open("", 1) + self.do_test_use_builtin_open_text("", "r") def test_no_ext(self): - self.do_test_use_builtin_open("abcd", 2) + self.do_test_use_builtin_open_text("abcd", "r") @unittest.skipUnless(gzip, "Requires gzip and zlib") def test_gz_ext_fake(self): original_open = gzip.open gzip.open = self.fake_open try: - result = fileinput.hook_compressed("test.gz", "3") + result = fileinput.hook_compressed("test.gz", "r") finally: gzip.open = original_open self.assertEqual(self.fake_open.invocation_count, 1) - self.assertEqual(self.fake_open.last_invocation, (("test.gz", "3"), {})) + self.assertEqual(self.fake_open.last_invocation, (("test.gz", "r"), {})) @unittest.skipUnless(gzip, "Requires gzip and zlib") def test_gz_with_encoding_fake(self): original_open = gzip.open gzip.open = lambda filename, mode: io.BytesIO(b'Ex-binary string') try: - result = fileinput.hook_compressed("test.gz", "3", encoding="utf-8") + result = fileinput.hook_compressed("test.gz", "r", encoding="utf-8") finally: gzip.open = original_open self.assertEqual(list(result), ['Ex-binary string']) @@ -887,23 +887,40 @@ def test_bz2_ext_fake(self): original_open = bz2.BZ2File bz2.BZ2File = self.fake_open try: - result = fileinput.hook_compressed("test.bz2", "4") + result = fileinput.hook_compressed("test.bz2", "r") finally: bz2.BZ2File = original_open self.assertEqual(self.fake_open.invocation_count, 1) - self.assertEqual(self.fake_open.last_invocation, (("test.bz2", "4"), {})) + self.assertEqual(self.fake_open.last_invocation, (("test.bz2", "r"), {})) def test_blah_ext(self): - self.do_test_use_builtin_open("abcd.blah", "5") + self.do_test_use_builtin_open_binary("abcd.blah", "rb") def test_gz_ext_builtin(self): - self.do_test_use_builtin_open("abcd.Gz", "6") + self.do_test_use_builtin_open_binary("abcd.Gz", "rb") def test_bz2_ext_builtin(self): - self.do_test_use_builtin_open("abcd.Bz2", "7") + self.do_test_use_builtin_open_binary("abcd.Bz2", "rb") - def do_test_use_builtin_open(self, filename, mode): + def test_binary_mode_encoding(self): + self.do_test_use_builtin_open_binary("abcd", "rb") + + def test_text_mode_encoding(self): + self.do_test_use_builtin_open_text("abcd", "r") + + def do_test_use_builtin_open_binary(self, filename, mode): + original_open = self.replace_builtin_open(self.fake_open) + try: + result = fileinput.hook_compressed(filename, mode) + finally: + self.replace_builtin_open(original_open) + + self.assertEqual(self.fake_open.invocation_count, 1) + self.assertEqual(self.fake_open.last_invocation, + ((filename, mode), {'encoding': None, 'errors': None})) + + def do_test_use_builtin_open_text(self, filename, mode): original_open = self.replace_builtin_open(self.fake_open) try: result = fileinput.hook_compressed(filename, mode) diff --git a/Misc/ACKS b/Misc/ACKS index ca92608868f23f..a9d4e453c7b59c 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -927,6 +927,7 @@ Tyler Kieft Mads Kiilerich Jason Killen Derek D. Kim +Gihwan Kim Jan Kim Taek Joo Kim Sam Kimbrel diff --git a/Misc/NEWS.d/next/Library/2023-02-21-10-05-33.gh-issue-101961.7e56jh.rst b/Misc/NEWS.d/next/Library/2023-02-21-10-05-33.gh-issue-101961.7e56jh.rst new file mode 100644 index 00000000000000..a3d4119e7cbdce --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-21-10-05-33.gh-issue-101961.7e56jh.rst @@ -0,0 +1,2 @@ +For the binary mode, :func:`fileinput.hookcompressed` doesn't set the ``encoding`` value +even if the value is ``None``. Patch by Gihwan Kim. From 02d9f1504beebd98dea807f4b8761d3675a500d0 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 21 Feb 2023 09:15:49 +0100 Subject: [PATCH 62/76] gh-101578: Amend exception docs (#102057) Co-authored-by: C.A.M. Gerlach --- Doc/c-api/exceptions.rst | 45 ++++++++++++++++++++-------------------- Doc/data/refcounts.dat | 2 ++ 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index e735f8db402363..8836c973f1f579 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -438,7 +438,9 @@ Querying the error indicator .. c:function:: void PyErr_Fetch(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) - As of 3.12, this function is deprecated. Use :c:func:`PyErr_GetRaisedException` instead. + .. deprecated:: 3.12 + + Use :c:func:`PyErr_GetRaisedException` instead. Retrieve the error indicator into three variables whose addresses are passed. If the error indicator is not set, set all three variables to ``NULL``. If it is @@ -447,8 +449,10 @@ Querying the error indicator .. note:: - This function is normally only used by code that needs to catch exceptions or - by code that needs to save and restore the error indicator temporarily, e.g.:: + This function is normally only used by legacy code that needs to catch + exceptions or save and restore the error indicator temporarily. + + For example:: { PyObject *type, *value, *traceback; @@ -459,15 +463,17 @@ Querying the error indicator PyErr_Restore(type, value, traceback); } - .. deprecated:: 3.12 - .. c:function:: void PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback) - As of 3.12, this function is deprecated. Use :c:func:`PyErr_SetRaisedException` instead. + .. deprecated:: 3.12 + + Use :c:func:`PyErr_SetRaisedException` instead. - Set the error indicator from the three objects. If the error indicator is - already set, it is cleared first. If the objects are ``NULL``, the error + Set the error indicator from the three objects, + *type*, *value*, and *traceback*, + clearing the existing exception if one is set. + If the objects are ``NULL``, the error indicator is cleared. Do not pass a ``NULL`` type and non-``NULL`` value or traceback. The exception type should be a class. Do not pass an invalid exception type or value. (Violating these rules will cause subtle problems @@ -478,18 +484,17 @@ Querying the error indicator .. note:: - This function is normally only used by code that needs to save and restore the - error indicator temporarily. Use :c:func:`PyErr_Fetch` to save the current - error indicator. - - .. deprecated:: 3.12 + This function is normally only used by legacy code that needs to + save and restore the error indicator temporarily. + Use :c:func:`PyErr_Fetch` to save the current error indicator. .. c:function:: void PyErr_NormalizeException(PyObject **exc, PyObject **val, PyObject **tb) - As of 3.12, this function is deprecated. - Use :c:func:`PyErr_GetRaisedException` instead of :c:func:`PyErr_Fetch` to avoid - any possible de-normalization. + .. deprecated:: 3.12 + + Use :c:func:`PyErr_GetRaisedException` instead, + to avoid any possible de-normalization. Under certain circumstances, the values returned by :c:func:`PyErr_Fetch` below can be "unnormalized", meaning that ``*exc`` is a class object but ``*val`` is @@ -507,8 +512,6 @@ Querying the error indicator PyException_SetTraceback(val, tb); } - .. deprecated:: 3.12 - .. c:function:: PyObject* PyErr_GetHandledException(void) @@ -756,14 +759,12 @@ Exception Objects .. c:function:: PyObject* PyException_GetArgs(PyObject *ex) - Return args of the given exception as a new reference, - as accessible from Python through :attr:`args`. + Return :attr:`~BaseException.args` of exception *ex*. .. c:function:: void PyException_SetArgs(PyObject *ex, PyObject *args) - Set the args of the given exception, - as accessible from Python through :attr:`args`. + Set :attr:`~BaseException.args` of exception *ex* to *args*. .. _unicodeexceptions: diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index ed2958f8cd2205..ee64ffdc91662e 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -839,6 +839,8 @@ PyEval_EvalFrameEx:int:throwflag:: PyEval_MergeCompilerFlags:int::: PyEval_MergeCompilerFlags:PyCompilerFlags*:cf:: +PyException_GetArgs:PyObject*::+1: + PyException_GetCause:PyObject*::+1: PyException_GetCause:PyObject*:ex:0: From 0f7a9725300e750947159409abbf5722a4a79f8b Mon Sep 17 00:00:00 2001 From: Owain Davies <116417456+OTheDev@users.noreply.github.com> Date: Tue, 21 Feb 2023 17:03:50 +0700 Subject: [PATCH 63/76] gh-101965: Fix usage of Py_EnterRecursiveCall return value in _bisectmodule.c (GH-101966) Closes #101965 Automerge-Triggered-By: GH:erlend-aasland --- Modules/_bisectmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 9ceb3ae46fe56d..d3bec535ee512d 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -66,7 +66,7 @@ internal_bisect_right(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t if (sq_item == NULL) { return -1; } - if (Py_EnterRecursiveCall("in _bisect.bisect_right") < 0) { + if (Py_EnterRecursiveCall("in _bisect.bisect_right")) { return -1; } PyTypeObject *tp = Py_TYPE(item); @@ -246,7 +246,7 @@ internal_bisect_left(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t h if (sq_item == NULL) { return -1; } - if (Py_EnterRecursiveCall("in _bisect.bisect_left") < 0) { + if (Py_EnterRecursiveCall("in _bisect.bisect_left")) { return -1; } PyTypeObject *tp = Py_TYPE(item); From 350ba7c07f8983537883e093c5c623287a2a22e5 Mon Sep 17 00:00:00 2001 From: Owain Davies <116417456+OTheDev@users.noreply.github.com> Date: Tue, 21 Feb 2023 17:24:33 +0700 Subject: [PATCH 64/76] gh-101777: Make `PriorityQueue` docs slightly clearer (#102026) Adjust wording slightly, and use min(entries) instead of sorted(list(entries))[0] as an example. --- Doc/library/queue.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index c67f15e953bccc..46b8e9b18a3c1f 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -57,8 +57,8 @@ The :mod:`queue` module defines the following classes and exceptions: *maxsize* is less than or equal to zero, the queue size is infinite. The lowest valued entries are retrieved first (the lowest valued entry is the - one returned by ``sorted(list(entries))[0]``). A typical pattern for entries - is a tuple in the form: ``(priority_number, data)``. + one that would be returned by ``min(entries)``). A typical pattern for + entries is a tuple in the form: ``(priority_number, data)``. If the *data* elements are not comparable, the data can be wrapped in a class that ignores the data item and only compares the priority number:: From b40dd71241f092d90c192ebff5d58cbd7e84dc52 Mon Sep 17 00:00:00 2001 From: ram vikram singh Date: Tue, 21 Feb 2023 16:04:56 +0530 Subject: [PATCH 65/76] gh-100556: Improve clarity of `or` docs (#100589) Co-authored-by: Hugo van Kemenade --- Doc/library/stdtypes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 98bda6ded30f79..41947d66c15134 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -84,8 +84,8 @@ These are the Boolean operations, ordered by ascending priority: +-------------+---------------------------------+-------+ | Operation | Result | Notes | +=============+=================================+=======+ -| ``x or y`` | if *x* is false, then *y*, else | \(1) | -| | *x* | | +| ``x or y`` | if *x* is true, then *x*, else | \(1) | +| | *y* | | +-------------+---------------------------------+-------+ | ``x and y`` | if *x* is false, then *x*, else | \(2) | | | *y* | | From 7346a381be1bc5911924fcf298408e6909f3949f Mon Sep 17 00:00:00 2001 From: Owain Davies <116417456+OTheDev@users.noreply.github.com> Date: Tue, 21 Feb 2023 18:58:47 +0700 Subject: [PATCH 66/76] gh-101903: Remove obsolete undefs for previously removed macros Py_EnterRecursiveCall and Py_LeaveRecursiveCall (#101923) --- Python/ceval.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index b85231a1df8a95..001bdb15c0f755 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3068,15 +3068,11 @@ maybe_dtrace_line(_PyInterpreterFrame *frame, /* Implement Py_EnterRecursiveCall() and Py_LeaveRecursiveCall() as functions for the limited API. */ -#undef Py_EnterRecursiveCall - int Py_EnterRecursiveCall(const char *where) { return _Py_EnterRecursiveCall(where); } -#undef Py_LeaveRecursiveCall - void Py_LeaveRecursiveCall(void) { _Py_LeaveRecursiveCall(); From c2b85a95a50687a2e5d1873e17266370876e77e9 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 21 Feb 2023 15:11:31 +0000 Subject: [PATCH 67/76] gh-102008: simplify test_except_star by using sys.exception() instead of sys.exc_info() (#102009) --- Lib/test/test_except_star.py | 44 ++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/Lib/test/test_except_star.py b/Lib/test/test_except_star.py index 9de72dbd5a3264..c5167c5bba38af 100644 --- a/Lib/test/test_except_star.py +++ b/Lib/test/test_except_star.py @@ -208,44 +208,38 @@ def assertMetadataNotEqual(self, e1, e2): class TestExceptStarSplitSemantics(ExceptStarTest): def doSplitTestNamed(self, exc, T, match_template, rest_template): - initial_exc_info = sys.exc_info() - exc_info = match = rest = None + initial_sys_exception = sys.exception() + sys_exception = match = rest = None try: try: raise exc except* T as e: - exc_info = sys.exc_info() + sys_exception = sys.exception() match = e except BaseException as e: rest = e - if match_template: - self.assertEqual(exc_info[1], match) - else: - self.assertIsNone(exc_info) + self.assertEqual(sys_exception, match) self.assertExceptionIsLike(match, match_template) self.assertExceptionIsLike(rest, rest_template) - self.assertEqual(sys.exc_info(), initial_exc_info) + self.assertEqual(sys.exception(), initial_sys_exception) def doSplitTestUnnamed(self, exc, T, match_template, rest_template): - initial_exc_info = sys.exc_info() - exc_info = match = rest = None + initial_sys_exception = sys.exception() + sys_exception = match = rest = None try: try: raise exc except* T: - exc_info = sys.exc_info() - match = sys.exc_info()[1] + sys_exception = match = sys.exception() else: if rest_template: self.fail("Exception not raised") except BaseException as e: rest = e self.assertExceptionIsLike(match, match_template) - if match_template: - self.assertEqual(exc_info[0], type(match_template)) self.assertExceptionIsLike(rest, rest_template) - self.assertEqual(sys.exc_info(), initial_exc_info) + self.assertEqual(sys.exception(), initial_sys_exception) def doSplitTestInExceptHandler(self, exc, T, match_template, rest_template): try: @@ -409,11 +403,11 @@ def test_multiple_matches_unnamed(self): try: raise ExceptionGroup("mmu", [OSError("os"), BlockingIOError("io")]) except* BlockingIOError: - e = sys.exc_info()[1] + e = sys.exception() self.assertExceptionIsLike(e, ExceptionGroup("mmu", [BlockingIOError("io")])) except* OSError: - e = sys.exc_info()[1] + e = sys.exception() self.assertExceptionIsLike(e, ExceptionGroup("mmu", [OSError("os")])) else: @@ -434,7 +428,7 @@ def test_first_match_wins_unnamed(self): try: raise ExceptionGroup("fstu", [BlockingIOError("io")]) except* OSError: - e = sys.exc_info()[1] + e = sys.exception() self.assertExceptionIsLike(e, ExceptionGroup("fstu", [BlockingIOError("io")])) except* BlockingIOError: @@ -452,7 +446,7 @@ def test_nested_except_stars(self): pass else: self.fail("Exception not raised") - e = sys.exc_info()[1] + e = sys.exception() self.assertExceptionIsLike(e, ExceptionGroup("n", [BlockingIOError("io")])) else: @@ -766,7 +760,7 @@ def test_raise_unnamed(self): try: raise orig except* OSError: - e = sys.exc_info()[1] + e = sys.exception() raise TypeError(3) from e except ExceptionGroup as e: exc = e @@ -821,7 +815,7 @@ def test_raise_handle_all_raise_one_unnamed(self): try: raise orig except* (TypeError, ValueError) as e: - e = sys.exc_info()[1] + e = sys.exception() raise SyntaxError(3) from e except ExceptionGroup as e: exc = e @@ -882,10 +876,10 @@ def test_raise_handle_all_raise_two_unnamed(self): try: raise orig except* TypeError: - e = sys.exc_info()[1] + e = sys.exception() raise SyntaxError(3) from e except* ValueError: - e = sys.exc_info()[1] + e = sys.exception() raise SyntaxError(4) from e except ExceptionGroup as e: exc = e @@ -982,7 +976,7 @@ def derive(self, excs): class TestExceptStarCleanup(ExceptStarTest): - def test_exc_info_restored(self): + def test_sys_exception_restored(self): try: try: raise ValueError(42) @@ -997,7 +991,7 @@ def test_exc_info_restored(self): self.assertExceptionIsLike(exc, ZeroDivisionError('division by zero')) self.assertExceptionIsLike(exc.__context__, ValueError(42)) - self.assertEqual(sys.exc_info(), (None, None, None)) + self.assertEqual(sys.exception(), None) class TestExceptStar_WeirdLeafExceptions(ExceptStarTest): From 0d4c7fcd4f078708a5ac6499af378ce5ee8eb211 Mon Sep 17 00:00:00 2001 From: Vo Hoang Long Date: Tue, 21 Feb 2023 22:14:41 +0700 Subject: [PATCH 68/76] gh-101936: Update the default value of fp from io.StringIO to io.BytesIO (gh-102100) Co-authored-by: Long Vo --- Lib/test/test_urllib2.py | 1 + Lib/urllib/error.py | 2 +- Misc/ACKS | 1 + .../next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst | 2 ++ 4 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 498c0382d2137b..633d596ac3de3f 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1827,6 +1827,7 @@ def test_HTTPError_interface(self): def test_gh_98778(self): x = urllib.error.HTTPError("url", 405, "METHOD NOT ALLOWED", None, None) self.assertEqual(getattr(x, "__notes__", ()), ()) + self.assertIsInstance(x.fp.read(), bytes) def test_parse_proxy(self): parse_proxy_test_cases = [ diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index feec0e7f848e46..a9cd1ecadd63f2 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -43,7 +43,7 @@ def __init__(self, url, code, msg, hdrs, fp): self.fp = fp self.filename = url if fp is None: - fp = io.StringIO() + fp = io.BytesIO() self.__super_init(fp, hdrs, url, code) def __str__(self): diff --git a/Misc/ACKS b/Misc/ACKS index a9d4e453c7b59c..ab19d69b0133d2 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1898,6 +1898,7 @@ Kurt Vile Norman Vine Pauli Virtanen Frank Visser +Long Vo Johannes Vogel Michael Vogt Radu Voicilas diff --git a/Misc/NEWS.d/next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst b/Misc/NEWS.d/next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst new file mode 100644 index 00000000000000..55841da44b1146 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst @@ -0,0 +1,2 @@ +The default value of ``fp`` becomes :class:`io.BytesIO` if :exc:`~urllib.error.HTTPError` +is initialized without a designated ``fp`` parameter. Patch by Long Vo. From d5c7954d0c3ff874d2d27d33dcc207bb7356f328 Mon Sep 17 00:00:00 2001 From: Hyunkyun Moon Date: Wed, 22 Feb 2023 02:39:00 +0900 Subject: [PATCH 69/76] gh-95672 fix typo SkitTest to SkipTest (gh-102119) Co-authored-by: HyunKyun Moon --- Lib/test/test_fcntl.py | 2 +- Lib/test/test_subprocess.py | 2 +- Misc/ACKS | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 113c7802821dd4..26de67d924272a 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -202,7 +202,7 @@ def test_fcntl_f_pipesize(self): pipesize_default = fcntl.fcntl(test_pipe_w, fcntl.F_GETPIPE_SZ) pipesize = pipesize_default // 2 # A new value to detect change. if pipesize < 512: # the POSIX minimum - raise unittest.SkitTest( + raise unittest.SkipTest( 'default pipesize too small to perform test.') fcntl.fcntl(test_pipe_w, fcntl.F_SETPIPE_SZ, pipesize) self.assertEqual(fcntl.fcntl(test_pipe_w, fcntl.F_GETPIPE_SZ), diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index abd0dd8b25699b..727b0e6dc578c2 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -718,7 +718,7 @@ def test_pipesizes(self): os.close(test_pipe_w) pipesize = pipesize_default // 2 if pipesize < 512: # the POSIX minimum - raise unittest.SkitTest( + raise unittest.SkipTest( 'default pipesize too small to perform test.') p = subprocess.Popen( [sys.executable, "-c", diff --git a/Misc/ACKS b/Misc/ACKS index ab19d69b0133d2..33dbf4e989d96a 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1229,6 +1229,7 @@ The Dragon De Monsyne Bastien Montagne Skip Montanaro Peter Moody +HyunKyun Moon Alan D. Moore Nicolai Moore Paul Moore From 3ba7743b0696202e9caa283a0be253fd26a5cfbd Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 21 Feb 2023 20:21:24 -0500 Subject: [PATCH 70/76] gh-99942: python.pc on android/cygwin should link to libpython per configure.ac (GH-100356) In commit 254b309c801f82509597e3d7d4be56885ef94c11 a previous change to avoid linking to libpython was partially reverted for Android (and later Cygwin as well), to add back the link flags. This was applied to distutils and to python-config.sh, but not to python.pc. Add it back to python.pc as well. Automerge-Triggered-By: GH:gpshead --- .../next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst | 2 ++ Misc/python.pc.in | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst diff --git a/Misc/NEWS.d/next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst b/Misc/NEWS.d/next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst new file mode 100644 index 00000000000000..63a640a9cdf733 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst @@ -0,0 +1,2 @@ +On Android, python.pc now correctly reports the library to link to, the same +as python-config.sh. diff --git a/Misc/python.pc.in b/Misc/python.pc.in index 87e04decc2a2d5..027dba38585a89 100644 --- a/Misc/python.pc.in +++ b/Misc/python.pc.in @@ -9,5 +9,5 @@ Description: Build a C extension for Python Requires: Version: @VERSION@ Libs.private: @LIBS@ -Libs: +Libs: -L${libdir} @LIBPYTHON@ Cflags: -I${includedir}/python@VERSION@@ABIFLAGS@ From 8d46c7ed5e83e22d55fe4f4e6e873d87f340c1dc Mon Sep 17 00:00:00 2001 From: somebody <98094921+UndoneStudios@users.noreply.github.com> Date: Wed, 22 Feb 2023 14:11:30 +0400 Subject: [PATCH 71/76] gh-102135: Update turtle docs to rename wikipedia demo to rosette (#102137) --- Doc/library/turtle.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 5add61c759ea8e..b7eb569c18a6c4 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -2444,6 +2444,9 @@ The demo scripts are: | planet_and_moon| simulation of | compound shapes, | | | gravitational system | :class:`Vec2D` | +----------------+------------------------------+-----------------------+ +| rosette | a pattern from the wikipedia | :func:`clone`, | +| | article on turtle graphics | :func:`undo` | ++----------------+------------------------------+-----------------------+ | round_dance | dancing turtles rotating | compound shapes, clone| | | pairwise in opposite | shapesize, tilt, | | | direction | get_shapepoly, update | @@ -2457,9 +2460,6 @@ The demo scripts are: | two_canvases | simple design | turtles on two | | | | canvases | +----------------+------------------------------+-----------------------+ -| wikipedia | a pattern from the wikipedia | :func:`clone`, | -| | article on turtle graphics | :func:`undo` | -+----------------+------------------------------+-----------------------+ | yinyang | another elementary example | :func:`circle` | +----------------+------------------------------+-----------------------+ From 7c106a443f8cf1111947a425eed11ecf9e615ce3 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Feb 2023 11:11:57 +0000 Subject: [PATCH 72/76] GH-100982: Restrict `FOR_ITER_RANGE` to a single instruction to allow instrumentation. (GH-101985) --- ...-02-17-10-12-13.gh-issue-100982.mJGJQw.rst | 2 ++ Python/bytecodes.c | 24 +++++++------------ Python/generated_cases.c.h | 22 ++++++++--------- Python/specialize.c | 4 +--- 4 files changed, 22 insertions(+), 30 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-17-10-12-13.gh-issue-100982.mJGJQw.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-17-10-12-13.gh-issue-100982.mJGJQw.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-17-10-12-13.gh-issue-100982.mJGJQw.rst new file mode 100644 index 00000000000000..53bbc860c53f37 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-17-10-12-13.gh-issue-100982.mJGJQw.rst @@ -0,0 +1,2 @@ +Restrict the scope of the :opcode:`FOR_ITER_RANGE` instruction to the scope of the +original :opcode:`FOR_ITER` instruction, to allow instrumentation. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c5959f2f994fdc..ad68c794fe7acb 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2178,35 +2178,27 @@ dummy_func( // Common case: no jump, leave it to the code generator } - // This is slightly different, when the loop isn't terminated we - // jump over the immediately following STORE_FAST instruction. - inst(FOR_ITER_RANGE, (unused/1, iter -- iter, unused)) { + inst(FOR_ITER_RANGE, (unused/1, iter -- iter, next)) { assert(cframe.use_tracing == 0); _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); - _Py_CODEUNIT next = next_instr[INLINE_CACHE_ENTRIES_FOR_ITER]; - assert(_PyOpcode_Deopt[next.op.code] == STORE_FAST); if (r->len <= 0) { STACK_SHRINK(1); Py_DECREF(r); // Jump over END_FOR instruction. JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1); + DISPATCH(); } - else { - long value = r->start; - r->start = value + r->step; - r->len--; - if (_PyLong_AssignValue(&GETLOCAL(next.op.arg), value) < 0) { - goto error; - } - // The STORE_FAST is already done. - JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + 1); + long value = r->start; + r->start = value + r->step; + r->len--; + next = PyLong_FromLong(value); + if (next == NULL) { + goto error; } - DISPATCH(); } - // This is *not* a super-instruction, unique in the family. inst(FOR_ITER_GEN, (unused/1, iter -- iter, unused)) { assert(cframe.use_tracing == 0); PyGenObject *gen = (PyGenObject *)iter; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 487e63d855d14a..2987adc3bba566 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2756,28 +2756,28 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = PEEK(1); + PyObject *next; assert(cframe.use_tracing == 0); _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); - _Py_CODEUNIT next = next_instr[INLINE_CACHE_ENTRIES_FOR_ITER]; - assert(_PyOpcode_Deopt[next.op.code] == STORE_FAST); if (r->len <= 0) { STACK_SHRINK(1); Py_DECREF(r); // Jump over END_FOR instruction. JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1); + DISPATCH(); } - else { - long value = r->start; - r->start = value + r->step; - r->len--; - if (_PyLong_AssignValue(&GETLOCAL(next.op.arg), value) < 0) { - goto error; - } - // The STORE_FAST is already done. - JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + 1); + long value = r->start; + r->start = value + r->step; + r->len--; + next = PyLong_FromLong(value); + if (next == NULL) { + goto error; } + STACK_GROW(1); + POKE(1, next); + JUMPBY(1); DISPATCH(); } diff --git a/Python/specialize.c b/Python/specialize.c index c9555f8ad4dccb..3405d2b0ab0680 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2155,8 +2155,6 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) assert(_PyOpcode_Caches[FOR_ITER] == INLINE_CACHE_ENTRIES_FOR_ITER); _PyForIterCache *cache = (_PyForIterCache *)(instr + 1); PyTypeObject *tp = Py_TYPE(iter); - _Py_CODEUNIT next = instr[1+INLINE_CACHE_ENTRIES_FOR_ITER]; - int next_op = _PyOpcode_Deopt[next.op.code]; if (tp == &PyListIter_Type) { instr->op.code = FOR_ITER_LIST; goto success; @@ -2165,7 +2163,7 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) instr->op.code = FOR_ITER_TUPLE; goto success; } - else if (tp == &PyRangeIter_Type && next_op == STORE_FAST) { + else if (tp == &PyRangeIter_Type) { instr->op.code = FOR_ITER_RANGE; goto success; } From 592f65fdb551f64a2db7849500e5df2291637f25 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 22 Feb 2023 22:10:01 +0300 Subject: [PATCH 73/76] Few coverage nitpicks for the cmath module (#102067) - partial tests for cosh/sinh overflows (L535 and L771). I doubt both ||-ed conditions could be tested. - removed inaccessible case in sqrt (L832): ax=ay=0 is handled above (L823) because fabs() is exact. Also added test (checked with mpmath and gmpy2) for second condition on that line. - some trivial tests for isclose (cover all conditions on L1217-1218) - add comment for uncovered L1018 Co-authored-by: Mark Dickinson --- Lib/test/cmath_testcases.txt | 3 +++ Lib/test/test_cmath.py | 8 ++++++++ Modules/cmathmodule.c | 4 ++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Lib/test/cmath_testcases.txt b/Lib/test/cmath_testcases.txt index dd7e458ddcb7b1..0165e17634f41c 100644 --- a/Lib/test/cmath_testcases.txt +++ b/Lib/test/cmath_testcases.txt @@ -1536,6 +1536,7 @@ sqrt0141 sqrt -1.797e+308 -9.9999999999999999e+306 -> 3.7284476432057307e+152 -1 sqrt0150 sqrt 1.7976931348623157e+308 0.0 -> 1.3407807929942596355e+154 0.0 sqrt0151 sqrt 2.2250738585072014e-308 0.0 -> 1.4916681462400413487e-154 0.0 sqrt0152 sqrt 5e-324 0.0 -> 2.2227587494850774834e-162 0.0 +sqrt0153 sqrt 5e-324 1.0 -> 0.7071067811865476 0.7071067811865476 -- special values sqrt1000 sqrt 0.0 0.0 -> 0.0 0.0 @@ -1744,6 +1745,7 @@ cosh0023 cosh 2.218885944363501 2.0015727395883687 -> -1.94294321081968 4.129026 -- large real part cosh0030 cosh 710.5 2.3519999999999999 -> -1.2967465239355998e+308 1.3076707908857333e+308 cosh0031 cosh -710.5 0.69999999999999996 -> 1.4085466381392499e+308 -1.1864024666450239e+308 +cosh0032 cosh 720.0 0.0 -> inf 0.0 overflow -- Additional real values (mpmath) cosh0050 cosh 1e-150 0.0 -> 1.0 0.0 @@ -1853,6 +1855,7 @@ sinh0023 sinh 0.043713693678420068 0.22512549887532657 -> 0.042624198673416713 0 -- large real part sinh0030 sinh 710.5 -2.3999999999999999 -> -1.3579970564885919e+308 -1.24394470907798e+308 sinh0031 sinh -710.5 0.80000000000000004 -> -1.2830671601735164e+308 1.3210954193997678e+308 +sinh0032 sinh 720.0 0.0 -> inf 0.0 overflow -- Additional real values (mpmath) sinh0050 sinh 1e-100 0.0 -> 1.00000000000000002e-100 0.0 diff --git a/Lib/test/test_cmath.py b/Lib/test/test_cmath.py index 9fa08dc4ff3fa7..cd2c6939105d40 100644 --- a/Lib/test/test_cmath.py +++ b/Lib/test/test_cmath.py @@ -607,6 +607,14 @@ def test_complex_near_zero(self): self.assertIsClose(0.001-0.001j, 0.001+0.001j, abs_tol=2e-03) self.assertIsNotClose(0.001-0.001j, 0.001+0.001j, abs_tol=1e-03) + def test_complex_special(self): + self.assertIsNotClose(INF, INF*1j) + self.assertIsNotClose(INF*1j, INF) + self.assertIsNotClose(INF, -INF) + self.assertIsNotClose(-INF, INF) + self.assertIsNotClose(0, INF) + self.assertIsNotClose(0, INF*1j) + if __name__ == "__main__": unittest.main() diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 53e34061d53773..b4f7e5424b4ccf 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -829,7 +829,7 @@ cmath_sqrt_impl(PyObject *module, Py_complex z) ax = fabs(z.real); ay = fabs(z.imag); - if (ax < DBL_MIN && ay < DBL_MIN && (ax > 0. || ay > 0.)) { + if (ax < DBL_MIN && ay < DBL_MIN) { /* here we catch cases where hypot(ax, ay) is subnormal */ ax = ldexp(ax, CM_SCALE_UP); s = ldexp(sqrt(ax + hypot(ax, ldexp(ay, CM_SCALE_UP))), @@ -1013,7 +1013,7 @@ cmath_phase_impl(PyObject *module, Py_complex z) double phi; errno = 0; - phi = c_atan2(z); + phi = c_atan2(z); /* should not cause any exception */ if (errno != 0) return math_error(); else From 96bf24380e44dfa1516d65480250995e737c0cb9 Mon Sep 17 00:00:00 2001 From: Owain Davies <116417456+OTheDev@users.noreply.github.com> Date: Thu, 23 Feb 2023 03:21:38 +0700 Subject: [PATCH 74/76] GH-101777: `queue.rst`: use 2 spaces after a period to be consistent. (#102143) --- Doc/library/queue.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index 46b8e9b18a3c1f..b2b787c5a8260c 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -15,7 +15,7 @@ module implements all the required locking semantics. The module implements three types of queue, which differ only in the order in which the entries are retrieved. In a :abbr:`FIFO (first-in, first-out)` -queue, the first tasks added are the first retrieved. In a +queue, the first tasks added are the first retrieved. In a :abbr:`LIFO (last-in, first-out)` queue, the most recently added entry is the first retrieved (operating like a stack). With a priority queue, the entries are kept sorted (using the :mod:`heapq` module) and the @@ -57,7 +57,7 @@ The :mod:`queue` module defines the following classes and exceptions: *maxsize* is less than or equal to zero, the queue size is infinite. The lowest valued entries are retrieved first (the lowest valued entry is the - one that would be returned by ``min(entries)``). A typical pattern for + one that would be returned by ``min(entries)``). A typical pattern for entries is a tuple in the form: ``(priority_number, data)``. If the *data* elements are not comparable, the data can be wrapped in a class @@ -127,8 +127,8 @@ provide the public methods described below. .. method:: Queue.put(item, block=True, timeout=None) - Put *item* into the queue. If optional args *block* is true and *timeout* is - ``None`` (the default), block if necessary until a free slot is available. If + Put *item* into the queue. If optional args *block* is true and *timeout* is + ``None`` (the default), block if necessary until a free slot is available. If *timeout* is a positive number, it blocks at most *timeout* seconds and raises the :exc:`Full` exception if no free slot was available within that time. Otherwise (*block* is false), put an item on the queue if a free slot is @@ -143,7 +143,7 @@ provide the public methods described below. .. method:: Queue.get(block=True, timeout=None) - Remove and return an item from the queue. If optional args *block* is true and + Remove and return an item from the queue. If optional args *block* is true and *timeout* is ``None`` (the default), block if necessary until an item is available. If *timeout* is a positive number, it blocks at most *timeout* seconds and raises the :exc:`Empty` exception if no item was available within that time. @@ -152,7 +152,7 @@ provide the public methods described below. Prior to 3.0 on POSIX systems, and for all versions on Windows, if *block* is true and *timeout* is ``None``, this operation goes into - an uninterruptible wait on an underlying lock. This means that no exceptions + an uninterruptible wait on an underlying lock. This means that no exceptions can occur, and in particular a SIGINT will not trigger a :exc:`KeyboardInterrupt`. @@ -184,7 +184,7 @@ fully processed by daemon consumer threads. The count of unfinished tasks goes up whenever an item is added to the queue. The count goes down whenever a consumer thread calls :meth:`task_done` to - indicate that the item was retrieved and all work on it is complete. When the + indicate that the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, :meth:`join` unblocks. @@ -227,7 +227,7 @@ SimpleQueue Objects .. method:: SimpleQueue.empty() - Return ``True`` if the queue is empty, ``False`` otherwise. If empty() + Return ``True`` if the queue is empty, ``False`` otherwise. If empty() returns ``False`` it doesn't guarantee that a subsequent call to get() will not block. From fcadc7e405141847ab10daf5cff16be880083a24 Mon Sep 17 00:00:00 2001 From: Jonathan Protzenko Date: Wed, 22 Feb 2023 13:18:43 -0800 Subject: [PATCH 75/76] gh-99108: Import MD5 and SHA1 from HACL* (#102089) Replaces our fallback non-OpenSSL MD5 and SHA1 implementations with those from HACL* as we've already done with SHA2. --- Makefile.pre.in | 14 +- ...3-02-17-10-42-48.gh-issue-99108.MKA8-f.rst | 2 + Modules/Setup | 4 +- Modules/Setup.stdlib.in | 4 +- Modules/_hacl/Hacl_Hash_MD5.c | 1472 +++++++++++++++++ Modules/_hacl/Hacl_Hash_MD5.h | 65 + Modules/_hacl/Hacl_Hash_SHA1.c | 508 ++++++ Modules/_hacl/Hacl_Hash_SHA1.h | 65 + Modules/_hacl/Hacl_Streaming_SHA2.c | 172 +- Modules/_hacl/Hacl_Streaming_SHA2.h | 67 +- Modules/_hacl/Hacl_Streaming_Types.h | 59 + .../include/krml/FStar_UInt128_Verified.h | 3 +- .../include/krml/FStar_UInt_8_16_32_64.h | 4 +- Modules/_hacl/include/krml/internal/target.h | 4 + Modules/_hacl/internal/Hacl_Hash_MD5.h | 61 + Modules/_hacl/internal/Hacl_Hash_SHA1.h | 61 + Modules/_hacl/python_hacl_namespaces.h | 17 + Modules/_hacl/refresh.sh | 9 +- Modules/md5module.c | 304 +--- Modules/sha1module.c | 282 +--- PCbuild/pythoncore.vcxproj | 2 + PCbuild/pythoncore.vcxproj.filters | 6 + configure | 4 +- configure.ac | 8 +- 24 files changed, 2498 insertions(+), 699 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2023-02-17-10-42-48.gh-issue-99108.MKA8-f.rst create mode 100644 Modules/_hacl/Hacl_Hash_MD5.c create mode 100644 Modules/_hacl/Hacl_Hash_MD5.h create mode 100644 Modules/_hacl/Hacl_Hash_SHA1.c create mode 100644 Modules/_hacl/Hacl_Hash_SHA1.h create mode 100644 Modules/_hacl/Hacl_Streaming_Types.h create mode 100644 Modules/_hacl/internal/Hacl_Hash_MD5.h create mode 100644 Modules/_hacl/internal/Hacl_Hash_SHA1.h diff --git a/Makefile.pre.in b/Makefile.pre.in index b28c6067a535b8..b12a1bc060af90 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -578,17 +578,21 @@ LIBEXPAT_HEADERS= \ LIBHACL_SHA2_OBJS= \ Modules/_hacl/Hacl_Streaming_SHA2.o -LIBHACL_SHA2_HEADERS= \ - Modules/_hacl/Hacl_Streaming_SHA2.h \ +LIBHACL_HEADERS= \ Modules/_hacl/include/krml/FStar_UInt128_Verified.h \ Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h \ Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h \ Modules/_hacl/include/krml/internal/target.h \ Modules/_hacl/include/krml/lowstar_endianness.h \ Modules/_hacl/include/krml/types.h \ - Modules/_hacl/internal/Hacl_SHA2_Generic.h \ + Modules/_hacl/Hacl_Streaming_Types.h \ Modules/_hacl/python_hacl_namespaces.h +LIBHACL_SHA2_HEADERS= \ + Modules/_hacl/Hacl_Streaming_SHA2.h \ + Modules/_hacl/internal/Hacl_SHA2_Generic.h \ + $(LIBHACL_HEADERS) + ######################################################################### # Rules @@ -2635,8 +2639,8 @@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h -MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h -MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h +MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_MD5.h Modules/_hacl/Hacl_Hash_MD5.c +MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA1.h Modules/_hacl/Hacl_Hash_SHA1.c MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_A) MODULE__SHA3_DEPS=$(srcdir)/Modules/_sha3/sha3.c $(srcdir)/Modules/_sha3/sha3.h $(srcdir)/Modules/hashlib.h MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c diff --git a/Misc/NEWS.d/next/Security/2023-02-17-10-42-48.gh-issue-99108.MKA8-f.rst b/Misc/NEWS.d/next/Security/2023-02-17-10-42-48.gh-issue-99108.MKA8-f.rst new file mode 100644 index 00000000000000..723d8a43a09f9e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-02-17-10-42-48.gh-issue-99108.MKA8-f.rst @@ -0,0 +1,2 @@ +Replace builtin hashlib implementations of MD5 and SHA1 with verified ones +from the HACL* project. diff --git a/Modules/Setup b/Modules/Setup index 1d5183bc2df118..f9fa26cac9e233 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -163,8 +163,8 @@ PYTHONPATH=$(COREPYTHONPATH) # hashing builtins #_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c -#_md5 md5module.c -#_sha1 sha1module.c +#_md5 md5module.c -I$(srcdir)/Modules/_hacl/include _hacl/libHacl_Hash_MD5.c +#_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include _hacl/libHacl_Hash_SHA1.c #_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a #_sha3 _sha3/sha3module.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 8f5e14a4e80e22..d33cd82239995f 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -77,8 +77,8 @@ @MODULE_READLINE_TRUE@readline readline.c # hashing builtins, can be disabled with --without-builtin-hashlib-hashes -@MODULE__MD5_TRUE@_md5 md5module.c -@MODULE__SHA1_TRUE@_sha1 sha1module.c +@MODULE__MD5_TRUE@_md5 md5module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_MD5.c -D_BSD_SOURCE -D_DEFAULT_SOURCE +@MODULE__SHA1_TRUE@_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA1.c -D_BSD_SOURCE -D_DEFAULT_SOURCE @MODULE__SHA2_TRUE@_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a @MODULE__SHA3_TRUE@_sha3 _sha3/sha3module.c @MODULE__BLAKE2_TRUE@_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c diff --git a/Modules/_hacl/Hacl_Hash_MD5.c b/Modules/_hacl/Hacl_Hash_MD5.c new file mode 100644 index 00000000000000..2c613066d9f682 --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_MD5.c @@ -0,0 +1,1472 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#include "internal/Hacl_Hash_MD5.h" + +static uint32_t +_h0[4U] = + { (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U }; + +static uint32_t +_t[64U] = + { + (uint32_t)0xd76aa478U, (uint32_t)0xe8c7b756U, (uint32_t)0x242070dbU, (uint32_t)0xc1bdceeeU, + (uint32_t)0xf57c0fafU, (uint32_t)0x4787c62aU, (uint32_t)0xa8304613U, (uint32_t)0xfd469501U, + (uint32_t)0x698098d8U, (uint32_t)0x8b44f7afU, (uint32_t)0xffff5bb1U, (uint32_t)0x895cd7beU, + (uint32_t)0x6b901122U, (uint32_t)0xfd987193U, (uint32_t)0xa679438eU, (uint32_t)0x49b40821U, + (uint32_t)0xf61e2562U, (uint32_t)0xc040b340U, (uint32_t)0x265e5a51U, (uint32_t)0xe9b6c7aaU, + (uint32_t)0xd62f105dU, (uint32_t)0x02441453U, (uint32_t)0xd8a1e681U, (uint32_t)0xe7d3fbc8U, + (uint32_t)0x21e1cde6U, (uint32_t)0xc33707d6U, (uint32_t)0xf4d50d87U, (uint32_t)0x455a14edU, + (uint32_t)0xa9e3e905U, (uint32_t)0xfcefa3f8U, (uint32_t)0x676f02d9U, (uint32_t)0x8d2a4c8aU, + (uint32_t)0xfffa3942U, (uint32_t)0x8771f681U, (uint32_t)0x6d9d6122U, (uint32_t)0xfde5380cU, + (uint32_t)0xa4beea44U, (uint32_t)0x4bdecfa9U, (uint32_t)0xf6bb4b60U, (uint32_t)0xbebfbc70U, + (uint32_t)0x289b7ec6U, (uint32_t)0xeaa127faU, (uint32_t)0xd4ef3085U, (uint32_t)0x4881d05U, + (uint32_t)0xd9d4d039U, (uint32_t)0xe6db99e5U, (uint32_t)0x1fa27cf8U, (uint32_t)0xc4ac5665U, + (uint32_t)0xf4292244U, (uint32_t)0x432aff97U, (uint32_t)0xab9423a7U, (uint32_t)0xfc93a039U, + (uint32_t)0x655b59c3U, (uint32_t)0x8f0ccc92U, (uint32_t)0xffeff47dU, (uint32_t)0x85845dd1U, + (uint32_t)0x6fa87e4fU, (uint32_t)0xfe2ce6e0U, (uint32_t)0xa3014314U, (uint32_t)0x4e0811a1U, + (uint32_t)0xf7537e82U, (uint32_t)0xbd3af235U, (uint32_t)0x2ad7d2bbU, (uint32_t)0xeb86d391U + }; + +void Hacl_Hash_Core_MD5_legacy_init(uint32_t *s) +{ + KRML_MAYBE_FOR4(i, (uint32_t)0U, (uint32_t)4U, (uint32_t)1U, s[i] = _h0[i];); +} + +static void legacy_update(uint32_t *abcd, uint8_t *x) +{ + uint32_t aa = abcd[0U]; + uint32_t bb = abcd[1U]; + uint32_t cc = abcd[2U]; + uint32_t dd = abcd[3U]; + uint32_t va = abcd[0U]; + uint32_t vb0 = abcd[1U]; + uint32_t vc0 = abcd[2U]; + uint32_t vd0 = abcd[3U]; + uint8_t *b0 = x; + uint32_t u = load32_le(b0); + uint32_t xk = u; + uint32_t ti0 = _t[0U]; + uint32_t + v = + vb0 + + + ((va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) + << (uint32_t)7U + | (va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) >> (uint32_t)25U); + abcd[0U] = v; + uint32_t va0 = abcd[3U]; + uint32_t vb1 = abcd[0U]; + uint32_t vc1 = abcd[1U]; + uint32_t vd1 = abcd[2U]; + uint8_t *b1 = x + (uint32_t)4U; + uint32_t u0 = load32_le(b1); + uint32_t xk0 = u0; + uint32_t ti1 = _t[1U]; + uint32_t + v0 = + vb1 + + + ((va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) + << (uint32_t)12U + | (va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) >> (uint32_t)20U); + abcd[3U] = v0; + uint32_t va1 = abcd[2U]; + uint32_t vb2 = abcd[3U]; + uint32_t vc2 = abcd[0U]; + uint32_t vd2 = abcd[1U]; + uint8_t *b2 = x + (uint32_t)8U; + uint32_t u1 = load32_le(b2); + uint32_t xk1 = u1; + uint32_t ti2 = _t[2U]; + uint32_t + v1 = + vb2 + + + ((va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) + << (uint32_t)17U + | (va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) >> (uint32_t)15U); + abcd[2U] = v1; + uint32_t va2 = abcd[1U]; + uint32_t vb3 = abcd[2U]; + uint32_t vc3 = abcd[3U]; + uint32_t vd3 = abcd[0U]; + uint8_t *b3 = x + (uint32_t)12U; + uint32_t u2 = load32_le(b3); + uint32_t xk2 = u2; + uint32_t ti3 = _t[3U]; + uint32_t + v2 = + vb3 + + + ((va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) + << (uint32_t)22U + | (va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) >> (uint32_t)10U); + abcd[1U] = v2; + uint32_t va3 = abcd[0U]; + uint32_t vb4 = abcd[1U]; + uint32_t vc4 = abcd[2U]; + uint32_t vd4 = abcd[3U]; + uint8_t *b4 = x + (uint32_t)16U; + uint32_t u3 = load32_le(b4); + uint32_t xk3 = u3; + uint32_t ti4 = _t[4U]; + uint32_t + v3 = + vb4 + + + ((va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) + << (uint32_t)7U + | (va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) >> (uint32_t)25U); + abcd[0U] = v3; + uint32_t va4 = abcd[3U]; + uint32_t vb5 = abcd[0U]; + uint32_t vc5 = abcd[1U]; + uint32_t vd5 = abcd[2U]; + uint8_t *b5 = x + (uint32_t)20U; + uint32_t u4 = load32_le(b5); + uint32_t xk4 = u4; + uint32_t ti5 = _t[5U]; + uint32_t + v4 = + vb5 + + + ((va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) + << (uint32_t)12U + | (va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) >> (uint32_t)20U); + abcd[3U] = v4; + uint32_t va5 = abcd[2U]; + uint32_t vb6 = abcd[3U]; + uint32_t vc6 = abcd[0U]; + uint32_t vd6 = abcd[1U]; + uint8_t *b6 = x + (uint32_t)24U; + uint32_t u5 = load32_le(b6); + uint32_t xk5 = u5; + uint32_t ti6 = _t[6U]; + uint32_t + v5 = + vb6 + + + ((va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) + << (uint32_t)17U + | (va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) >> (uint32_t)15U); + abcd[2U] = v5; + uint32_t va6 = abcd[1U]; + uint32_t vb7 = abcd[2U]; + uint32_t vc7 = abcd[3U]; + uint32_t vd7 = abcd[0U]; + uint8_t *b7 = x + (uint32_t)28U; + uint32_t u6 = load32_le(b7); + uint32_t xk6 = u6; + uint32_t ti7 = _t[7U]; + uint32_t + v6 = + vb7 + + + ((va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) + << (uint32_t)22U + | (va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) >> (uint32_t)10U); + abcd[1U] = v6; + uint32_t va7 = abcd[0U]; + uint32_t vb8 = abcd[1U]; + uint32_t vc8 = abcd[2U]; + uint32_t vd8 = abcd[3U]; + uint8_t *b8 = x + (uint32_t)32U; + uint32_t u7 = load32_le(b8); + uint32_t xk7 = u7; + uint32_t ti8 = _t[8U]; + uint32_t + v7 = + vb8 + + + ((va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) + << (uint32_t)7U + | (va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) >> (uint32_t)25U); + abcd[0U] = v7; + uint32_t va8 = abcd[3U]; + uint32_t vb9 = abcd[0U]; + uint32_t vc9 = abcd[1U]; + uint32_t vd9 = abcd[2U]; + uint8_t *b9 = x + (uint32_t)36U; + uint32_t u8 = load32_le(b9); + uint32_t xk8 = u8; + uint32_t ti9 = _t[9U]; + uint32_t + v8 = + vb9 + + + ((va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) + << (uint32_t)12U + | (va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) >> (uint32_t)20U); + abcd[3U] = v8; + uint32_t va9 = abcd[2U]; + uint32_t vb10 = abcd[3U]; + uint32_t vc10 = abcd[0U]; + uint32_t vd10 = abcd[1U]; + uint8_t *b10 = x + (uint32_t)40U; + uint32_t u9 = load32_le(b10); + uint32_t xk9 = u9; + uint32_t ti10 = _t[10U]; + uint32_t + v9 = + vb10 + + + ((va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) + << (uint32_t)17U + | (va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) >> (uint32_t)15U); + abcd[2U] = v9; + uint32_t va10 = abcd[1U]; + uint32_t vb11 = abcd[2U]; + uint32_t vc11 = abcd[3U]; + uint32_t vd11 = abcd[0U]; + uint8_t *b11 = x + (uint32_t)44U; + uint32_t u10 = load32_le(b11); + uint32_t xk10 = u10; + uint32_t ti11 = _t[11U]; + uint32_t + v10 = + vb11 + + + ((va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) + << (uint32_t)22U + | (va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) >> (uint32_t)10U); + abcd[1U] = v10; + uint32_t va11 = abcd[0U]; + uint32_t vb12 = abcd[1U]; + uint32_t vc12 = abcd[2U]; + uint32_t vd12 = abcd[3U]; + uint8_t *b12 = x + (uint32_t)48U; + uint32_t u11 = load32_le(b12); + uint32_t xk11 = u11; + uint32_t ti12 = _t[12U]; + uint32_t + v11 = + vb12 + + + ((va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) + << (uint32_t)7U + | (va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) >> (uint32_t)25U); + abcd[0U] = v11; + uint32_t va12 = abcd[3U]; + uint32_t vb13 = abcd[0U]; + uint32_t vc13 = abcd[1U]; + uint32_t vd13 = abcd[2U]; + uint8_t *b13 = x + (uint32_t)52U; + uint32_t u12 = load32_le(b13); + uint32_t xk12 = u12; + uint32_t ti13 = _t[13U]; + uint32_t + v12 = + vb13 + + + ((va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) + << (uint32_t)12U + | (va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) >> (uint32_t)20U); + abcd[3U] = v12; + uint32_t va13 = abcd[2U]; + uint32_t vb14 = abcd[3U]; + uint32_t vc14 = abcd[0U]; + uint32_t vd14 = abcd[1U]; + uint8_t *b14 = x + (uint32_t)56U; + uint32_t u13 = load32_le(b14); + uint32_t xk13 = u13; + uint32_t ti14 = _t[14U]; + uint32_t + v13 = + vb14 + + + ((va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) + << (uint32_t)17U + | (va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) >> (uint32_t)15U); + abcd[2U] = v13; + uint32_t va14 = abcd[1U]; + uint32_t vb15 = abcd[2U]; + uint32_t vc15 = abcd[3U]; + uint32_t vd15 = abcd[0U]; + uint8_t *b15 = x + (uint32_t)60U; + uint32_t u14 = load32_le(b15); + uint32_t xk14 = u14; + uint32_t ti15 = _t[15U]; + uint32_t + v14 = + vb15 + + + ((va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) + << (uint32_t)22U + | (va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) >> (uint32_t)10U); + abcd[1U] = v14; + uint32_t va15 = abcd[0U]; + uint32_t vb16 = abcd[1U]; + uint32_t vc16 = abcd[2U]; + uint32_t vd16 = abcd[3U]; + uint8_t *b16 = x + (uint32_t)4U; + uint32_t u15 = load32_le(b16); + uint32_t xk15 = u15; + uint32_t ti16 = _t[16U]; + uint32_t + v15 = + vb16 + + + ((va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) + << (uint32_t)5U + | (va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) >> (uint32_t)27U); + abcd[0U] = v15; + uint32_t va16 = abcd[3U]; + uint32_t vb17 = abcd[0U]; + uint32_t vc17 = abcd[1U]; + uint32_t vd17 = abcd[2U]; + uint8_t *b17 = x + (uint32_t)24U; + uint32_t u16 = load32_le(b17); + uint32_t xk16 = u16; + uint32_t ti17 = _t[17U]; + uint32_t + v16 = + vb17 + + + ((va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) + << (uint32_t)9U + | (va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) >> (uint32_t)23U); + abcd[3U] = v16; + uint32_t va17 = abcd[2U]; + uint32_t vb18 = abcd[3U]; + uint32_t vc18 = abcd[0U]; + uint32_t vd18 = abcd[1U]; + uint8_t *b18 = x + (uint32_t)44U; + uint32_t u17 = load32_le(b18); + uint32_t xk17 = u17; + uint32_t ti18 = _t[18U]; + uint32_t + v17 = + vb18 + + + ((va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) + << (uint32_t)14U + | (va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) >> (uint32_t)18U); + abcd[2U] = v17; + uint32_t va18 = abcd[1U]; + uint32_t vb19 = abcd[2U]; + uint32_t vc19 = abcd[3U]; + uint32_t vd19 = abcd[0U]; + uint8_t *b19 = x; + uint32_t u18 = load32_le(b19); + uint32_t xk18 = u18; + uint32_t ti19 = _t[19U]; + uint32_t + v18 = + vb19 + + + ((va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) + << (uint32_t)20U + | (va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) >> (uint32_t)12U); + abcd[1U] = v18; + uint32_t va19 = abcd[0U]; + uint32_t vb20 = abcd[1U]; + uint32_t vc20 = abcd[2U]; + uint32_t vd20 = abcd[3U]; + uint8_t *b20 = x + (uint32_t)20U; + uint32_t u19 = load32_le(b20); + uint32_t xk19 = u19; + uint32_t ti20 = _t[20U]; + uint32_t + v19 = + vb20 + + + ((va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) + << (uint32_t)5U + | (va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) >> (uint32_t)27U); + abcd[0U] = v19; + uint32_t va20 = abcd[3U]; + uint32_t vb21 = abcd[0U]; + uint32_t vc21 = abcd[1U]; + uint32_t vd21 = abcd[2U]; + uint8_t *b21 = x + (uint32_t)40U; + uint32_t u20 = load32_le(b21); + uint32_t xk20 = u20; + uint32_t ti21 = _t[21U]; + uint32_t + v20 = + vb21 + + + ((va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) + << (uint32_t)9U + | (va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) >> (uint32_t)23U); + abcd[3U] = v20; + uint32_t va21 = abcd[2U]; + uint32_t vb22 = abcd[3U]; + uint32_t vc22 = abcd[0U]; + uint32_t vd22 = abcd[1U]; + uint8_t *b22 = x + (uint32_t)60U; + uint32_t u21 = load32_le(b22); + uint32_t xk21 = u21; + uint32_t ti22 = _t[22U]; + uint32_t + v21 = + vb22 + + + ((va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) + << (uint32_t)14U + | (va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) >> (uint32_t)18U); + abcd[2U] = v21; + uint32_t va22 = abcd[1U]; + uint32_t vb23 = abcd[2U]; + uint32_t vc23 = abcd[3U]; + uint32_t vd23 = abcd[0U]; + uint8_t *b23 = x + (uint32_t)16U; + uint32_t u22 = load32_le(b23); + uint32_t xk22 = u22; + uint32_t ti23 = _t[23U]; + uint32_t + v22 = + vb23 + + + ((va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) + << (uint32_t)20U + | (va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) >> (uint32_t)12U); + abcd[1U] = v22; + uint32_t va23 = abcd[0U]; + uint32_t vb24 = abcd[1U]; + uint32_t vc24 = abcd[2U]; + uint32_t vd24 = abcd[3U]; + uint8_t *b24 = x + (uint32_t)36U; + uint32_t u23 = load32_le(b24); + uint32_t xk23 = u23; + uint32_t ti24 = _t[24U]; + uint32_t + v23 = + vb24 + + + ((va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) + << (uint32_t)5U + | (va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) >> (uint32_t)27U); + abcd[0U] = v23; + uint32_t va24 = abcd[3U]; + uint32_t vb25 = abcd[0U]; + uint32_t vc25 = abcd[1U]; + uint32_t vd25 = abcd[2U]; + uint8_t *b25 = x + (uint32_t)56U; + uint32_t u24 = load32_le(b25); + uint32_t xk24 = u24; + uint32_t ti25 = _t[25U]; + uint32_t + v24 = + vb25 + + + ((va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) + << (uint32_t)9U + | (va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) >> (uint32_t)23U); + abcd[3U] = v24; + uint32_t va25 = abcd[2U]; + uint32_t vb26 = abcd[3U]; + uint32_t vc26 = abcd[0U]; + uint32_t vd26 = abcd[1U]; + uint8_t *b26 = x + (uint32_t)12U; + uint32_t u25 = load32_le(b26); + uint32_t xk25 = u25; + uint32_t ti26 = _t[26U]; + uint32_t + v25 = + vb26 + + + ((va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) + << (uint32_t)14U + | (va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) >> (uint32_t)18U); + abcd[2U] = v25; + uint32_t va26 = abcd[1U]; + uint32_t vb27 = abcd[2U]; + uint32_t vc27 = abcd[3U]; + uint32_t vd27 = abcd[0U]; + uint8_t *b27 = x + (uint32_t)32U; + uint32_t u26 = load32_le(b27); + uint32_t xk26 = u26; + uint32_t ti27 = _t[27U]; + uint32_t + v26 = + vb27 + + + ((va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) + << (uint32_t)20U + | (va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) >> (uint32_t)12U); + abcd[1U] = v26; + uint32_t va27 = abcd[0U]; + uint32_t vb28 = abcd[1U]; + uint32_t vc28 = abcd[2U]; + uint32_t vd28 = abcd[3U]; + uint8_t *b28 = x + (uint32_t)52U; + uint32_t u27 = load32_le(b28); + uint32_t xk27 = u27; + uint32_t ti28 = _t[28U]; + uint32_t + v27 = + vb28 + + + ((va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) + << (uint32_t)5U + | (va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) >> (uint32_t)27U); + abcd[0U] = v27; + uint32_t va28 = abcd[3U]; + uint32_t vb29 = abcd[0U]; + uint32_t vc29 = abcd[1U]; + uint32_t vd29 = abcd[2U]; + uint8_t *b29 = x + (uint32_t)8U; + uint32_t u28 = load32_le(b29); + uint32_t xk28 = u28; + uint32_t ti29 = _t[29U]; + uint32_t + v28 = + vb29 + + + ((va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) + << (uint32_t)9U + | (va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) >> (uint32_t)23U); + abcd[3U] = v28; + uint32_t va29 = abcd[2U]; + uint32_t vb30 = abcd[3U]; + uint32_t vc30 = abcd[0U]; + uint32_t vd30 = abcd[1U]; + uint8_t *b30 = x + (uint32_t)28U; + uint32_t u29 = load32_le(b30); + uint32_t xk29 = u29; + uint32_t ti30 = _t[30U]; + uint32_t + v29 = + vb30 + + + ((va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) + << (uint32_t)14U + | (va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) >> (uint32_t)18U); + abcd[2U] = v29; + uint32_t va30 = abcd[1U]; + uint32_t vb31 = abcd[2U]; + uint32_t vc31 = abcd[3U]; + uint32_t vd31 = abcd[0U]; + uint8_t *b31 = x + (uint32_t)48U; + uint32_t u30 = load32_le(b31); + uint32_t xk30 = u30; + uint32_t ti31 = _t[31U]; + uint32_t + v30 = + vb31 + + + ((va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) + << (uint32_t)20U + | (va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) >> (uint32_t)12U); + abcd[1U] = v30; + uint32_t va31 = abcd[0U]; + uint32_t vb32 = abcd[1U]; + uint32_t vc32 = abcd[2U]; + uint32_t vd32 = abcd[3U]; + uint8_t *b32 = x + (uint32_t)20U; + uint32_t u31 = load32_le(b32); + uint32_t xk31 = u31; + uint32_t ti32 = _t[32U]; + uint32_t + v31 = + vb32 + + + ((va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) + << (uint32_t)4U + | (va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) >> (uint32_t)28U); + abcd[0U] = v31; + uint32_t va32 = abcd[3U]; + uint32_t vb33 = abcd[0U]; + uint32_t vc33 = abcd[1U]; + uint32_t vd33 = abcd[2U]; + uint8_t *b33 = x + (uint32_t)32U; + uint32_t u32 = load32_le(b33); + uint32_t xk32 = u32; + uint32_t ti33 = _t[33U]; + uint32_t + v32 = + vb33 + + + ((va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) + << (uint32_t)11U + | (va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) >> (uint32_t)21U); + abcd[3U] = v32; + uint32_t va33 = abcd[2U]; + uint32_t vb34 = abcd[3U]; + uint32_t vc34 = abcd[0U]; + uint32_t vd34 = abcd[1U]; + uint8_t *b34 = x + (uint32_t)44U; + uint32_t u33 = load32_le(b34); + uint32_t xk33 = u33; + uint32_t ti34 = _t[34U]; + uint32_t + v33 = + vb34 + + + ((va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) + << (uint32_t)16U + | (va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) >> (uint32_t)16U); + abcd[2U] = v33; + uint32_t va34 = abcd[1U]; + uint32_t vb35 = abcd[2U]; + uint32_t vc35 = abcd[3U]; + uint32_t vd35 = abcd[0U]; + uint8_t *b35 = x + (uint32_t)56U; + uint32_t u34 = load32_le(b35); + uint32_t xk34 = u34; + uint32_t ti35 = _t[35U]; + uint32_t + v34 = + vb35 + + + ((va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) + << (uint32_t)23U + | (va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) >> (uint32_t)9U); + abcd[1U] = v34; + uint32_t va35 = abcd[0U]; + uint32_t vb36 = abcd[1U]; + uint32_t vc36 = abcd[2U]; + uint32_t vd36 = abcd[3U]; + uint8_t *b36 = x + (uint32_t)4U; + uint32_t u35 = load32_le(b36); + uint32_t xk35 = u35; + uint32_t ti36 = _t[36U]; + uint32_t + v35 = + vb36 + + + ((va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) + << (uint32_t)4U + | (va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) >> (uint32_t)28U); + abcd[0U] = v35; + uint32_t va36 = abcd[3U]; + uint32_t vb37 = abcd[0U]; + uint32_t vc37 = abcd[1U]; + uint32_t vd37 = abcd[2U]; + uint8_t *b37 = x + (uint32_t)16U; + uint32_t u36 = load32_le(b37); + uint32_t xk36 = u36; + uint32_t ti37 = _t[37U]; + uint32_t + v36 = + vb37 + + + ((va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) + << (uint32_t)11U + | (va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) >> (uint32_t)21U); + abcd[3U] = v36; + uint32_t va37 = abcd[2U]; + uint32_t vb38 = abcd[3U]; + uint32_t vc38 = abcd[0U]; + uint32_t vd38 = abcd[1U]; + uint8_t *b38 = x + (uint32_t)28U; + uint32_t u37 = load32_le(b38); + uint32_t xk37 = u37; + uint32_t ti38 = _t[38U]; + uint32_t + v37 = + vb38 + + + ((va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) + << (uint32_t)16U + | (va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) >> (uint32_t)16U); + abcd[2U] = v37; + uint32_t va38 = abcd[1U]; + uint32_t vb39 = abcd[2U]; + uint32_t vc39 = abcd[3U]; + uint32_t vd39 = abcd[0U]; + uint8_t *b39 = x + (uint32_t)40U; + uint32_t u38 = load32_le(b39); + uint32_t xk38 = u38; + uint32_t ti39 = _t[39U]; + uint32_t + v38 = + vb39 + + + ((va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) + << (uint32_t)23U + | (va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) >> (uint32_t)9U); + abcd[1U] = v38; + uint32_t va39 = abcd[0U]; + uint32_t vb40 = abcd[1U]; + uint32_t vc40 = abcd[2U]; + uint32_t vd40 = abcd[3U]; + uint8_t *b40 = x + (uint32_t)52U; + uint32_t u39 = load32_le(b40); + uint32_t xk39 = u39; + uint32_t ti40 = _t[40U]; + uint32_t + v39 = + vb40 + + + ((va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) + << (uint32_t)4U + | (va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) >> (uint32_t)28U); + abcd[0U] = v39; + uint32_t va40 = abcd[3U]; + uint32_t vb41 = abcd[0U]; + uint32_t vc41 = abcd[1U]; + uint32_t vd41 = abcd[2U]; + uint8_t *b41 = x; + uint32_t u40 = load32_le(b41); + uint32_t xk40 = u40; + uint32_t ti41 = _t[41U]; + uint32_t + v40 = + vb41 + + + ((va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) + << (uint32_t)11U + | (va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) >> (uint32_t)21U); + abcd[3U] = v40; + uint32_t va41 = abcd[2U]; + uint32_t vb42 = abcd[3U]; + uint32_t vc42 = abcd[0U]; + uint32_t vd42 = abcd[1U]; + uint8_t *b42 = x + (uint32_t)12U; + uint32_t u41 = load32_le(b42); + uint32_t xk41 = u41; + uint32_t ti42 = _t[42U]; + uint32_t + v41 = + vb42 + + + ((va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) + << (uint32_t)16U + | (va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) >> (uint32_t)16U); + abcd[2U] = v41; + uint32_t va42 = abcd[1U]; + uint32_t vb43 = abcd[2U]; + uint32_t vc43 = abcd[3U]; + uint32_t vd43 = abcd[0U]; + uint8_t *b43 = x + (uint32_t)24U; + uint32_t u42 = load32_le(b43); + uint32_t xk42 = u42; + uint32_t ti43 = _t[43U]; + uint32_t + v42 = + vb43 + + + ((va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) + << (uint32_t)23U + | (va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) >> (uint32_t)9U); + abcd[1U] = v42; + uint32_t va43 = abcd[0U]; + uint32_t vb44 = abcd[1U]; + uint32_t vc44 = abcd[2U]; + uint32_t vd44 = abcd[3U]; + uint8_t *b44 = x + (uint32_t)36U; + uint32_t u43 = load32_le(b44); + uint32_t xk43 = u43; + uint32_t ti44 = _t[44U]; + uint32_t + v43 = + vb44 + + + ((va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) + << (uint32_t)4U + | (va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) >> (uint32_t)28U); + abcd[0U] = v43; + uint32_t va44 = abcd[3U]; + uint32_t vb45 = abcd[0U]; + uint32_t vc45 = abcd[1U]; + uint32_t vd45 = abcd[2U]; + uint8_t *b45 = x + (uint32_t)48U; + uint32_t u44 = load32_le(b45); + uint32_t xk44 = u44; + uint32_t ti45 = _t[45U]; + uint32_t + v44 = + vb45 + + + ((va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) + << (uint32_t)11U + | (va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) >> (uint32_t)21U); + abcd[3U] = v44; + uint32_t va45 = abcd[2U]; + uint32_t vb46 = abcd[3U]; + uint32_t vc46 = abcd[0U]; + uint32_t vd46 = abcd[1U]; + uint8_t *b46 = x + (uint32_t)60U; + uint32_t u45 = load32_le(b46); + uint32_t xk45 = u45; + uint32_t ti46 = _t[46U]; + uint32_t + v45 = + vb46 + + + ((va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) + << (uint32_t)16U + | (va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) >> (uint32_t)16U); + abcd[2U] = v45; + uint32_t va46 = abcd[1U]; + uint32_t vb47 = abcd[2U]; + uint32_t vc47 = abcd[3U]; + uint32_t vd47 = abcd[0U]; + uint8_t *b47 = x + (uint32_t)8U; + uint32_t u46 = load32_le(b47); + uint32_t xk46 = u46; + uint32_t ti47 = _t[47U]; + uint32_t + v46 = + vb47 + + + ((va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) + << (uint32_t)23U + | (va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) >> (uint32_t)9U); + abcd[1U] = v46; + uint32_t va47 = abcd[0U]; + uint32_t vb48 = abcd[1U]; + uint32_t vc48 = abcd[2U]; + uint32_t vd48 = abcd[3U]; + uint8_t *b48 = x; + uint32_t u47 = load32_le(b48); + uint32_t xk47 = u47; + uint32_t ti48 = _t[48U]; + uint32_t + v47 = + vb48 + + + ((va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) + << (uint32_t)6U + | (va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) >> (uint32_t)26U); + abcd[0U] = v47; + uint32_t va48 = abcd[3U]; + uint32_t vb49 = abcd[0U]; + uint32_t vc49 = abcd[1U]; + uint32_t vd49 = abcd[2U]; + uint8_t *b49 = x + (uint32_t)28U; + uint32_t u48 = load32_le(b49); + uint32_t xk48 = u48; + uint32_t ti49 = _t[49U]; + uint32_t + v48 = + vb49 + + + ((va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) + << (uint32_t)10U + | (va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) >> (uint32_t)22U); + abcd[3U] = v48; + uint32_t va49 = abcd[2U]; + uint32_t vb50 = abcd[3U]; + uint32_t vc50 = abcd[0U]; + uint32_t vd50 = abcd[1U]; + uint8_t *b50 = x + (uint32_t)56U; + uint32_t u49 = load32_le(b50); + uint32_t xk49 = u49; + uint32_t ti50 = _t[50U]; + uint32_t + v49 = + vb50 + + + ((va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) + << (uint32_t)15U + | (va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) >> (uint32_t)17U); + abcd[2U] = v49; + uint32_t va50 = abcd[1U]; + uint32_t vb51 = abcd[2U]; + uint32_t vc51 = abcd[3U]; + uint32_t vd51 = abcd[0U]; + uint8_t *b51 = x + (uint32_t)20U; + uint32_t u50 = load32_le(b51); + uint32_t xk50 = u50; + uint32_t ti51 = _t[51U]; + uint32_t + v50 = + vb51 + + + ((va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) + << (uint32_t)21U + | (va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) >> (uint32_t)11U); + abcd[1U] = v50; + uint32_t va51 = abcd[0U]; + uint32_t vb52 = abcd[1U]; + uint32_t vc52 = abcd[2U]; + uint32_t vd52 = abcd[3U]; + uint8_t *b52 = x + (uint32_t)48U; + uint32_t u51 = load32_le(b52); + uint32_t xk51 = u51; + uint32_t ti52 = _t[52U]; + uint32_t + v51 = + vb52 + + + ((va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) + << (uint32_t)6U + | (va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) >> (uint32_t)26U); + abcd[0U] = v51; + uint32_t va52 = abcd[3U]; + uint32_t vb53 = abcd[0U]; + uint32_t vc53 = abcd[1U]; + uint32_t vd53 = abcd[2U]; + uint8_t *b53 = x + (uint32_t)12U; + uint32_t u52 = load32_le(b53); + uint32_t xk52 = u52; + uint32_t ti53 = _t[53U]; + uint32_t + v52 = + vb53 + + + ((va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) + << (uint32_t)10U + | (va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) >> (uint32_t)22U); + abcd[3U] = v52; + uint32_t va53 = abcd[2U]; + uint32_t vb54 = abcd[3U]; + uint32_t vc54 = abcd[0U]; + uint32_t vd54 = abcd[1U]; + uint8_t *b54 = x + (uint32_t)40U; + uint32_t u53 = load32_le(b54); + uint32_t xk53 = u53; + uint32_t ti54 = _t[54U]; + uint32_t + v53 = + vb54 + + + ((va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) + << (uint32_t)15U + | (va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) >> (uint32_t)17U); + abcd[2U] = v53; + uint32_t va54 = abcd[1U]; + uint32_t vb55 = abcd[2U]; + uint32_t vc55 = abcd[3U]; + uint32_t vd55 = abcd[0U]; + uint8_t *b55 = x + (uint32_t)4U; + uint32_t u54 = load32_le(b55); + uint32_t xk54 = u54; + uint32_t ti55 = _t[55U]; + uint32_t + v54 = + vb55 + + + ((va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) + << (uint32_t)21U + | (va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) >> (uint32_t)11U); + abcd[1U] = v54; + uint32_t va55 = abcd[0U]; + uint32_t vb56 = abcd[1U]; + uint32_t vc56 = abcd[2U]; + uint32_t vd56 = abcd[3U]; + uint8_t *b56 = x + (uint32_t)32U; + uint32_t u55 = load32_le(b56); + uint32_t xk55 = u55; + uint32_t ti56 = _t[56U]; + uint32_t + v55 = + vb56 + + + ((va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) + << (uint32_t)6U + | (va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) >> (uint32_t)26U); + abcd[0U] = v55; + uint32_t va56 = abcd[3U]; + uint32_t vb57 = abcd[0U]; + uint32_t vc57 = abcd[1U]; + uint32_t vd57 = abcd[2U]; + uint8_t *b57 = x + (uint32_t)60U; + uint32_t u56 = load32_le(b57); + uint32_t xk56 = u56; + uint32_t ti57 = _t[57U]; + uint32_t + v56 = + vb57 + + + ((va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) + << (uint32_t)10U + | (va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) >> (uint32_t)22U); + abcd[3U] = v56; + uint32_t va57 = abcd[2U]; + uint32_t vb58 = abcd[3U]; + uint32_t vc58 = abcd[0U]; + uint32_t vd58 = abcd[1U]; + uint8_t *b58 = x + (uint32_t)24U; + uint32_t u57 = load32_le(b58); + uint32_t xk57 = u57; + uint32_t ti58 = _t[58U]; + uint32_t + v57 = + vb58 + + + ((va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) + << (uint32_t)15U + | (va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) >> (uint32_t)17U); + abcd[2U] = v57; + uint32_t va58 = abcd[1U]; + uint32_t vb59 = abcd[2U]; + uint32_t vc59 = abcd[3U]; + uint32_t vd59 = abcd[0U]; + uint8_t *b59 = x + (uint32_t)52U; + uint32_t u58 = load32_le(b59); + uint32_t xk58 = u58; + uint32_t ti59 = _t[59U]; + uint32_t + v58 = + vb59 + + + ((va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) + << (uint32_t)21U + | (va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) >> (uint32_t)11U); + abcd[1U] = v58; + uint32_t va59 = abcd[0U]; + uint32_t vb60 = abcd[1U]; + uint32_t vc60 = abcd[2U]; + uint32_t vd60 = abcd[3U]; + uint8_t *b60 = x + (uint32_t)16U; + uint32_t u59 = load32_le(b60); + uint32_t xk59 = u59; + uint32_t ti60 = _t[60U]; + uint32_t + v59 = + vb60 + + + ((va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) + << (uint32_t)6U + | (va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) >> (uint32_t)26U); + abcd[0U] = v59; + uint32_t va60 = abcd[3U]; + uint32_t vb61 = abcd[0U]; + uint32_t vc61 = abcd[1U]; + uint32_t vd61 = abcd[2U]; + uint8_t *b61 = x + (uint32_t)44U; + uint32_t u60 = load32_le(b61); + uint32_t xk60 = u60; + uint32_t ti61 = _t[61U]; + uint32_t + v60 = + vb61 + + + ((va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) + << (uint32_t)10U + | (va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) >> (uint32_t)22U); + abcd[3U] = v60; + uint32_t va61 = abcd[2U]; + uint32_t vb62 = abcd[3U]; + uint32_t vc62 = abcd[0U]; + uint32_t vd62 = abcd[1U]; + uint8_t *b62 = x + (uint32_t)8U; + uint32_t u61 = load32_le(b62); + uint32_t xk61 = u61; + uint32_t ti62 = _t[62U]; + uint32_t + v61 = + vb62 + + + ((va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) + << (uint32_t)15U + | (va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) >> (uint32_t)17U); + abcd[2U] = v61; + uint32_t va62 = abcd[1U]; + uint32_t vb = abcd[2U]; + uint32_t vc = abcd[3U]; + uint32_t vd = abcd[0U]; + uint8_t *b63 = x + (uint32_t)36U; + uint32_t u62 = load32_le(b63); + uint32_t xk62 = u62; + uint32_t ti = _t[63U]; + uint32_t + v62 = + vb + + + ((va62 + (vc ^ (vb | ~vd)) + xk62 + ti) + << (uint32_t)21U + | (va62 + (vc ^ (vb | ~vd)) + xk62 + ti) >> (uint32_t)11U); + abcd[1U] = v62; + uint32_t a = abcd[0U]; + uint32_t b = abcd[1U]; + uint32_t c = abcd[2U]; + uint32_t d = abcd[3U]; + abcd[0U] = a + aa; + abcd[1U] = b + bb; + abcd[2U] = c + cc; + abcd[3U] = d + dd; +} + +static void legacy_pad(uint64_t len, uint8_t *dst) +{ + uint8_t *dst1 = dst; + dst1[0U] = (uint8_t)0x80U; + uint8_t *dst2 = dst + (uint32_t)1U; + for + (uint32_t + i = (uint32_t)0U; + i + < ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) % (uint32_t)64U; + i++) + { + dst2[i] = (uint8_t)0U; + } + uint8_t + *dst3 = + dst + + + (uint32_t)1U + + + ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) + % (uint32_t)64U; + store64_le(dst3, len << (uint32_t)3U); +} + +void Hacl_Hash_Core_MD5_legacy_finish(uint32_t *s, uint8_t *dst) +{ + KRML_MAYBE_FOR4(i, + (uint32_t)0U, + (uint32_t)4U, + (uint32_t)1U, + store32_le(dst + i * (uint32_t)4U, s[i]);); +} + +void Hacl_Hash_MD5_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks) +{ + for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + { + uint32_t sz = (uint32_t)64U; + uint8_t *block = blocks + sz * i; + legacy_update(s, block); + } +} + +void +Hacl_Hash_MD5_legacy_update_last( + uint32_t *s, + uint64_t prev_len, + uint8_t *input, + uint32_t input_len +) +{ + uint32_t blocks_n = input_len / (uint32_t)64U; + uint32_t blocks_len = blocks_n * (uint32_t)64U; + uint8_t *blocks = input; + uint32_t rest_len = input_len - blocks_len; + uint8_t *rest = input + blocks_len; + Hacl_Hash_MD5_legacy_update_multi(s, blocks, blocks_n); + uint64_t total_input_len = prev_len + (uint64_t)input_len; + uint32_t + pad_len = + (uint32_t)1U + + + ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(total_input_len % (uint64_t)(uint32_t)64U))) + % (uint32_t)64U + + (uint32_t)8U; + uint32_t tmp_len = rest_len + pad_len; + uint8_t tmp_twoblocks[128U] = { 0U }; + uint8_t *tmp = tmp_twoblocks; + uint8_t *tmp_rest = tmp; + uint8_t *tmp_pad = tmp + rest_len; + memcpy(tmp_rest, rest, rest_len * sizeof (uint8_t)); + legacy_pad(total_input_len, tmp_pad); + Hacl_Hash_MD5_legacy_update_multi(s, tmp, tmp_len / (uint32_t)64U); +} + +void Hacl_Hash_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +{ + uint32_t + s[4U] = + { (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U }; + uint32_t blocks_n0 = input_len / (uint32_t)64U; + uint32_t blocks_n1; + if (input_len % (uint32_t)64U == (uint32_t)0U && blocks_n0 > (uint32_t)0U) + { + blocks_n1 = blocks_n0 - (uint32_t)1U; + } + else + { + blocks_n1 = blocks_n0; + } + uint32_t blocks_len0 = blocks_n1 * (uint32_t)64U; + uint8_t *blocks0 = input; + uint32_t rest_len0 = input_len - blocks_len0; + uint8_t *rest0 = input + blocks_len0; + uint32_t blocks_n = blocks_n1; + uint32_t blocks_len = blocks_len0; + uint8_t *blocks = blocks0; + uint32_t rest_len = rest_len0; + uint8_t *rest = rest0; + Hacl_Hash_MD5_legacy_update_multi(s, blocks, blocks_n); + Hacl_Hash_MD5_legacy_update_last(s, (uint64_t)blocks_len, rest, rest_len); + Hacl_Hash_Core_MD5_legacy_finish(s, dst); +} + +Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_create_in(void) +{ + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)4U, sizeof (uint32_t)); + Hacl_Streaming_MD_state_32 + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + Hacl_Streaming_MD_state_32 + *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); + p[0U] = s; + Hacl_Hash_Core_MD5_legacy_init(block_state); + return p; +} + +void Hacl_Streaming_MD5_legacy_init(Hacl_Streaming_MD_state_32 *s) +{ + Hacl_Streaming_MD_state_32 scrut = *s; + uint8_t *buf = scrut.buf; + uint32_t *block_state = scrut.block_state; + Hacl_Hash_Core_MD5_legacy_init(block_state); + Hacl_Streaming_MD_state_32 + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s[0U] = tmp; +} + +/** +0 = success, 1 = max length exceeded +*/ +uint32_t +Hacl_Streaming_MD5_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) +{ + Hacl_Streaming_MD_state_32 s = *p; + uint64_t total_len = s.total_len; + if ((uint64_t)len > (uint64_t)2305843009213693951U - total_len) + { + return (uint32_t)1U; + } + uint32_t sz; + if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + { + sz = (uint32_t)64U; + } + else + { + sz = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + } + if (len <= (uint32_t)64U - sz) + { + Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t *block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = (uint32_t)64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + } + uint8_t *buf2 = buf + sz1; + memcpy(buf2, data, len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)len; + *p + = + ( + (Hacl_Streaming_MD_state_32){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len2 + } + ); + } + else if (sz == (uint32_t)0U) + { + Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t *block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = (uint32_t)64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + } + if (!(sz1 == (uint32_t)0U)) + { + Hacl_Hash_MD5_legacy_update_multi(block_state1, buf, (uint32_t)1U); + } + uint32_t ite; + if ((uint64_t)len % (uint64_t)(uint32_t)64U == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + { + ite = (uint32_t)64U; + } + else + { + ite = (uint32_t)((uint64_t)len % (uint64_t)(uint32_t)64U); + } + uint32_t n_blocks = (len - ite) / (uint32_t)64U; + uint32_t data1_len = n_blocks * (uint32_t)64U; + uint32_t data2_len = len - data1_len; + uint8_t *data1 = data; + uint8_t *data2 = data + data1_len; + Hacl_Hash_MD5_legacy_update_multi(block_state1, data1, data1_len / (uint32_t)64U); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_MD_state_32){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)len + } + ); + } + else + { + uint32_t diff = (uint32_t)64U - sz; + uint8_t *data1 = data; + uint8_t *data2 = data + diff; + Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t *block_state10 = s1.block_state; + uint8_t *buf0 = s1.buf; + uint64_t total_len10 = s1.total_len; + uint32_t sz10; + if (total_len10 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len10 > (uint64_t)0U) + { + sz10 = (uint32_t)64U; + } + else + { + sz10 = (uint32_t)(total_len10 % (uint64_t)(uint32_t)64U); + } + uint8_t *buf2 = buf0 + sz10; + memcpy(buf2, data1, diff * sizeof (uint8_t)); + uint64_t total_len2 = total_len10 + (uint64_t)diff; + *p + = + ( + (Hacl_Streaming_MD_state_32){ + .block_state = block_state10, + .buf = buf0, + .total_len = total_len2 + } + ); + Hacl_Streaming_MD_state_32 s10 = *p; + uint32_t *block_state1 = s10.block_state; + uint8_t *buf = s10.buf; + uint64_t total_len1 = s10.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = (uint32_t)64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + } + if (!(sz1 == (uint32_t)0U)) + { + Hacl_Hash_MD5_legacy_update_multi(block_state1, buf, (uint32_t)1U); + } + uint32_t ite; + if + ( + (uint64_t)(len - diff) + % (uint64_t)(uint32_t)64U + == (uint64_t)0U + && (uint64_t)(len - diff) > (uint64_t)0U + ) + { + ite = (uint32_t)64U; + } + else + { + ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)(uint32_t)64U); + } + uint32_t n_blocks = (len - diff - ite) / (uint32_t)64U; + uint32_t data1_len = n_blocks * (uint32_t)64U; + uint32_t data2_len = len - diff - data1_len; + uint8_t *data11 = data2; + uint8_t *data21 = data2 + data1_len; + Hacl_Hash_MD5_legacy_update_multi(block_state1, data11, data1_len / (uint32_t)64U); + uint8_t *dst = buf; + memcpy(dst, data21, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_MD_state_32){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)(len - diff) + } + ); + } + return (uint32_t)0U; +} + +void Hacl_Streaming_MD5_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) +{ + Hacl_Streaming_MD_state_32 scrut = *p; + uint32_t *block_state = scrut.block_state; + uint8_t *buf_ = scrut.buf; + uint64_t total_len = scrut.total_len; + uint32_t r; + if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + { + r = (uint32_t)64U; + } + else + { + r = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + } + uint8_t *buf_1 = buf_; + uint32_t tmp_block_state[4U] = { 0U }; + memcpy(tmp_block_state, block_state, (uint32_t)4U * sizeof (uint32_t)); + uint32_t ite; + if (r % (uint32_t)64U == (uint32_t)0U && r > (uint32_t)0U) + { + ite = (uint32_t)64U; + } + else + { + ite = r % (uint32_t)64U; + } + uint8_t *buf_last = buf_1 + r - ite; + uint8_t *buf_multi = buf_1; + Hacl_Hash_MD5_legacy_update_multi(tmp_block_state, buf_multi, (uint32_t)0U); + uint64_t prev_len_last = total_len - (uint64_t)r; + Hacl_Hash_MD5_legacy_update_last(tmp_block_state, prev_len_last, buf_last, r); + Hacl_Hash_Core_MD5_legacy_finish(tmp_block_state, dst); +} + +void Hacl_Streaming_MD5_legacy_free(Hacl_Streaming_MD_state_32 *s) +{ + Hacl_Streaming_MD_state_32 scrut = *s; + uint8_t *buf = scrut.buf; + uint32_t *block_state = scrut.block_state; + KRML_HOST_FREE(block_state); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(s); +} + +Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_copy(Hacl_Streaming_MD_state_32 *s0) +{ + Hacl_Streaming_MD_state_32 scrut = *s0; + uint32_t *block_state0 = scrut.block_state; + uint8_t *buf0 = scrut.buf; + uint64_t total_len0 = scrut.total_len; + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); + memcpy(buf, buf0, (uint32_t)64U * sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)4U, sizeof (uint32_t)); + memcpy(block_state, block_state0, (uint32_t)4U * sizeof (uint32_t)); + Hacl_Streaming_MD_state_32 + s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; + Hacl_Streaming_MD_state_32 + *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); + p[0U] = s; + return p; +} + +void Hacl_Streaming_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +{ + Hacl_Hash_MD5_legacy_hash(input, input_len, dst); +} + diff --git a/Modules/_hacl/Hacl_Hash_MD5.h b/Modules/_hacl/Hacl_Hash_MD5.h new file mode 100644 index 00000000000000..015e3668751b75 --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_MD5.h @@ -0,0 +1,65 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __Hacl_Hash_MD5_H +#define __Hacl_Hash_MD5_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "Hacl_Streaming_Types.h" + +typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_MD5_state; + +Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_create_in(void); + +void Hacl_Streaming_MD5_legacy_init(Hacl_Streaming_MD_state_32 *s); + +/** +0 = success, 1 = max length exceeded +*/ +uint32_t +Hacl_Streaming_MD5_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len); + +void Hacl_Streaming_MD5_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); + +void Hacl_Streaming_MD5_legacy_free(Hacl_Streaming_MD_state_32 *s); + +Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_copy(Hacl_Streaming_MD_state_32 *s0); + +void Hacl_Streaming_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); + +#if defined(__cplusplus) +} +#endif + +#define __Hacl_Hash_MD5_H_DEFINED +#endif diff --git a/Modules/_hacl/Hacl_Hash_SHA1.c b/Modules/_hacl/Hacl_Hash_SHA1.c new file mode 100644 index 00000000000000..e155e338271c56 --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_SHA1.c @@ -0,0 +1,508 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#include "internal/Hacl_Hash_SHA1.h" + +static uint32_t +_h0[5U] = + { + (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U, + (uint32_t)0xc3d2e1f0U + }; + +void Hacl_Hash_Core_SHA1_legacy_init(uint32_t *s) +{ + KRML_MAYBE_FOR5(i, (uint32_t)0U, (uint32_t)5U, (uint32_t)1U, s[i] = _h0[i];); +} + +static void legacy_update(uint32_t *h, uint8_t *l) +{ + uint32_t ha = h[0U]; + uint32_t hb = h[1U]; + uint32_t hc = h[2U]; + uint32_t hd = h[3U]; + uint32_t he = h[4U]; + uint32_t _w[80U] = { 0U }; + for (uint32_t i = (uint32_t)0U; i < (uint32_t)80U; i++) + { + uint32_t v; + if (i < (uint32_t)16U) + { + uint8_t *b = l + i * (uint32_t)4U; + uint32_t u = load32_be(b); + v = u; + } + else + { + uint32_t wmit3 = _w[i - (uint32_t)3U]; + uint32_t wmit8 = _w[i - (uint32_t)8U]; + uint32_t wmit14 = _w[i - (uint32_t)14U]; + uint32_t wmit16 = _w[i - (uint32_t)16U]; + v = + (wmit3 ^ (wmit8 ^ (wmit14 ^ wmit16))) + << (uint32_t)1U + | (wmit3 ^ (wmit8 ^ (wmit14 ^ wmit16))) >> (uint32_t)31U; + } + _w[i] = v; + } + for (uint32_t i = (uint32_t)0U; i < (uint32_t)80U; i++) + { + uint32_t _a = h[0U]; + uint32_t _b = h[1U]; + uint32_t _c = h[2U]; + uint32_t _d = h[3U]; + uint32_t _e = h[4U]; + uint32_t wmit = _w[i]; + uint32_t ite0; + if (i < (uint32_t)20U) + { + ite0 = (_b & _c) ^ (~_b & _d); + } + else if ((uint32_t)39U < i && i < (uint32_t)60U) + { + ite0 = (_b & _c) ^ ((_b & _d) ^ (_c & _d)); + } + else + { + ite0 = _b ^ (_c ^ _d); + } + uint32_t ite; + if (i < (uint32_t)20U) + { + ite = (uint32_t)0x5a827999U; + } + else if (i < (uint32_t)40U) + { + ite = (uint32_t)0x6ed9eba1U; + } + else if (i < (uint32_t)60U) + { + ite = (uint32_t)0x8f1bbcdcU; + } + else + { + ite = (uint32_t)0xca62c1d6U; + } + uint32_t _T = (_a << (uint32_t)5U | _a >> (uint32_t)27U) + ite0 + _e + ite + wmit; + h[0U] = _T; + h[1U] = _a; + h[2U] = _b << (uint32_t)30U | _b >> (uint32_t)2U; + h[3U] = _c; + h[4U] = _d; + } + for (uint32_t i = (uint32_t)0U; i < (uint32_t)80U; i++) + { + _w[i] = (uint32_t)0U; + } + uint32_t sta = h[0U]; + uint32_t stb = h[1U]; + uint32_t stc = h[2U]; + uint32_t std = h[3U]; + uint32_t ste = h[4U]; + h[0U] = sta + ha; + h[1U] = stb + hb; + h[2U] = stc + hc; + h[3U] = std + hd; + h[4U] = ste + he; +} + +static void legacy_pad(uint64_t len, uint8_t *dst) +{ + uint8_t *dst1 = dst; + dst1[0U] = (uint8_t)0x80U; + uint8_t *dst2 = dst + (uint32_t)1U; + for + (uint32_t + i = (uint32_t)0U; + i + < ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) % (uint32_t)64U; + i++) + { + dst2[i] = (uint8_t)0U; + } + uint8_t + *dst3 = + dst + + + (uint32_t)1U + + + ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) + % (uint32_t)64U; + store64_be(dst3, len << (uint32_t)3U); +} + +void Hacl_Hash_Core_SHA1_legacy_finish(uint32_t *s, uint8_t *dst) +{ + KRML_MAYBE_FOR5(i, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + store32_be(dst + i * (uint32_t)4U, s[i]);); +} + +void Hacl_Hash_SHA1_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks) +{ + for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + { + uint32_t sz = (uint32_t)64U; + uint8_t *block = blocks + sz * i; + legacy_update(s, block); + } +} + +void +Hacl_Hash_SHA1_legacy_update_last( + uint32_t *s, + uint64_t prev_len, + uint8_t *input, + uint32_t input_len +) +{ + uint32_t blocks_n = input_len / (uint32_t)64U; + uint32_t blocks_len = blocks_n * (uint32_t)64U; + uint8_t *blocks = input; + uint32_t rest_len = input_len - blocks_len; + uint8_t *rest = input + blocks_len; + Hacl_Hash_SHA1_legacy_update_multi(s, blocks, blocks_n); + uint64_t total_input_len = prev_len + (uint64_t)input_len; + uint32_t + pad_len = + (uint32_t)1U + + + ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(total_input_len % (uint64_t)(uint32_t)64U))) + % (uint32_t)64U + + (uint32_t)8U; + uint32_t tmp_len = rest_len + pad_len; + uint8_t tmp_twoblocks[128U] = { 0U }; + uint8_t *tmp = tmp_twoblocks; + uint8_t *tmp_rest = tmp; + uint8_t *tmp_pad = tmp + rest_len; + memcpy(tmp_rest, rest, rest_len * sizeof (uint8_t)); + legacy_pad(total_input_len, tmp_pad); + Hacl_Hash_SHA1_legacy_update_multi(s, tmp, tmp_len / (uint32_t)64U); +} + +void Hacl_Hash_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +{ + uint32_t + s[5U] = + { + (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U, + (uint32_t)0xc3d2e1f0U + }; + uint32_t blocks_n0 = input_len / (uint32_t)64U; + uint32_t blocks_n1; + if (input_len % (uint32_t)64U == (uint32_t)0U && blocks_n0 > (uint32_t)0U) + { + blocks_n1 = blocks_n0 - (uint32_t)1U; + } + else + { + blocks_n1 = blocks_n0; + } + uint32_t blocks_len0 = blocks_n1 * (uint32_t)64U; + uint8_t *blocks0 = input; + uint32_t rest_len0 = input_len - blocks_len0; + uint8_t *rest0 = input + blocks_len0; + uint32_t blocks_n = blocks_n1; + uint32_t blocks_len = blocks_len0; + uint8_t *blocks = blocks0; + uint32_t rest_len = rest_len0; + uint8_t *rest = rest0; + Hacl_Hash_SHA1_legacy_update_multi(s, blocks, blocks_n); + Hacl_Hash_SHA1_legacy_update_last(s, (uint64_t)blocks_len, rest, rest_len); + Hacl_Hash_Core_SHA1_legacy_finish(s, dst); +} + +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_create_in(void) +{ + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)5U, sizeof (uint32_t)); + Hacl_Streaming_MD_state_32 + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + Hacl_Streaming_MD_state_32 + *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); + p[0U] = s; + Hacl_Hash_Core_SHA1_legacy_init(block_state); + return p; +} + +void Hacl_Streaming_SHA1_legacy_init(Hacl_Streaming_MD_state_32 *s) +{ + Hacl_Streaming_MD_state_32 scrut = *s; + uint8_t *buf = scrut.buf; + uint32_t *block_state = scrut.block_state; + Hacl_Hash_Core_SHA1_legacy_init(block_state); + Hacl_Streaming_MD_state_32 + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s[0U] = tmp; +} + +/** +0 = success, 1 = max length exceeded +*/ +uint32_t +Hacl_Streaming_SHA1_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) +{ + Hacl_Streaming_MD_state_32 s = *p; + uint64_t total_len = s.total_len; + if ((uint64_t)len > (uint64_t)2305843009213693951U - total_len) + { + return (uint32_t)1U; + } + uint32_t sz; + if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + { + sz = (uint32_t)64U; + } + else + { + sz = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + } + if (len <= (uint32_t)64U - sz) + { + Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t *block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = (uint32_t)64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + } + uint8_t *buf2 = buf + sz1; + memcpy(buf2, data, len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)len; + *p + = + ( + (Hacl_Streaming_MD_state_32){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len2 + } + ); + } + else if (sz == (uint32_t)0U) + { + Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t *block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = (uint32_t)64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + } + if (!(sz1 == (uint32_t)0U)) + { + Hacl_Hash_SHA1_legacy_update_multi(block_state1, buf, (uint32_t)1U); + } + uint32_t ite; + if ((uint64_t)len % (uint64_t)(uint32_t)64U == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + { + ite = (uint32_t)64U; + } + else + { + ite = (uint32_t)((uint64_t)len % (uint64_t)(uint32_t)64U); + } + uint32_t n_blocks = (len - ite) / (uint32_t)64U; + uint32_t data1_len = n_blocks * (uint32_t)64U; + uint32_t data2_len = len - data1_len; + uint8_t *data1 = data; + uint8_t *data2 = data + data1_len; + Hacl_Hash_SHA1_legacy_update_multi(block_state1, data1, data1_len / (uint32_t)64U); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_MD_state_32){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)len + } + ); + } + else + { + uint32_t diff = (uint32_t)64U - sz; + uint8_t *data1 = data; + uint8_t *data2 = data + diff; + Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t *block_state10 = s1.block_state; + uint8_t *buf0 = s1.buf; + uint64_t total_len10 = s1.total_len; + uint32_t sz10; + if (total_len10 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len10 > (uint64_t)0U) + { + sz10 = (uint32_t)64U; + } + else + { + sz10 = (uint32_t)(total_len10 % (uint64_t)(uint32_t)64U); + } + uint8_t *buf2 = buf0 + sz10; + memcpy(buf2, data1, diff * sizeof (uint8_t)); + uint64_t total_len2 = total_len10 + (uint64_t)diff; + *p + = + ( + (Hacl_Streaming_MD_state_32){ + .block_state = block_state10, + .buf = buf0, + .total_len = total_len2 + } + ); + Hacl_Streaming_MD_state_32 s10 = *p; + uint32_t *block_state1 = s10.block_state; + uint8_t *buf = s10.buf; + uint64_t total_len1 = s10.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = (uint32_t)64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + } + if (!(sz1 == (uint32_t)0U)) + { + Hacl_Hash_SHA1_legacy_update_multi(block_state1, buf, (uint32_t)1U); + } + uint32_t ite; + if + ( + (uint64_t)(len - diff) + % (uint64_t)(uint32_t)64U + == (uint64_t)0U + && (uint64_t)(len - diff) > (uint64_t)0U + ) + { + ite = (uint32_t)64U; + } + else + { + ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)(uint32_t)64U); + } + uint32_t n_blocks = (len - diff - ite) / (uint32_t)64U; + uint32_t data1_len = n_blocks * (uint32_t)64U; + uint32_t data2_len = len - diff - data1_len; + uint8_t *data11 = data2; + uint8_t *data21 = data2 + data1_len; + Hacl_Hash_SHA1_legacy_update_multi(block_state1, data11, data1_len / (uint32_t)64U); + uint8_t *dst = buf; + memcpy(dst, data21, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_MD_state_32){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)(len - diff) + } + ); + } + return (uint32_t)0U; +} + +void Hacl_Streaming_SHA1_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) +{ + Hacl_Streaming_MD_state_32 scrut = *p; + uint32_t *block_state = scrut.block_state; + uint8_t *buf_ = scrut.buf; + uint64_t total_len = scrut.total_len; + uint32_t r; + if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + { + r = (uint32_t)64U; + } + else + { + r = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + } + uint8_t *buf_1 = buf_; + uint32_t tmp_block_state[5U] = { 0U }; + memcpy(tmp_block_state, block_state, (uint32_t)5U * sizeof (uint32_t)); + uint32_t ite; + if (r % (uint32_t)64U == (uint32_t)0U && r > (uint32_t)0U) + { + ite = (uint32_t)64U; + } + else + { + ite = r % (uint32_t)64U; + } + uint8_t *buf_last = buf_1 + r - ite; + uint8_t *buf_multi = buf_1; + Hacl_Hash_SHA1_legacy_update_multi(tmp_block_state, buf_multi, (uint32_t)0U); + uint64_t prev_len_last = total_len - (uint64_t)r; + Hacl_Hash_SHA1_legacy_update_last(tmp_block_state, prev_len_last, buf_last, r); + Hacl_Hash_Core_SHA1_legacy_finish(tmp_block_state, dst); +} + +void Hacl_Streaming_SHA1_legacy_free(Hacl_Streaming_MD_state_32 *s) +{ + Hacl_Streaming_MD_state_32 scrut = *s; + uint8_t *buf = scrut.buf; + uint32_t *block_state = scrut.block_state; + KRML_HOST_FREE(block_state); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(s); +} + +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_copy(Hacl_Streaming_MD_state_32 *s0) +{ + Hacl_Streaming_MD_state_32 scrut = *s0; + uint32_t *block_state0 = scrut.block_state; + uint8_t *buf0 = scrut.buf; + uint64_t total_len0 = scrut.total_len; + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); + memcpy(buf, buf0, (uint32_t)64U * sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)5U, sizeof (uint32_t)); + memcpy(block_state, block_state0, (uint32_t)5U * sizeof (uint32_t)); + Hacl_Streaming_MD_state_32 + s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; + Hacl_Streaming_MD_state_32 + *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); + p[0U] = s; + return p; +} + +void Hacl_Streaming_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +{ + Hacl_Hash_SHA1_legacy_hash(input, input_len, dst); +} + diff --git a/Modules/_hacl/Hacl_Hash_SHA1.h b/Modules/_hacl/Hacl_Hash_SHA1.h new file mode 100644 index 00000000000000..5e2ae8e713292d --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_SHA1.h @@ -0,0 +1,65 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __Hacl_Hash_SHA1_H +#define __Hacl_Hash_SHA1_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "Hacl_Streaming_Types.h" + +typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_SHA1_state; + +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_create_in(void); + +void Hacl_Streaming_SHA1_legacy_init(Hacl_Streaming_MD_state_32 *s); + +/** +0 = success, 1 = max length exceeded +*/ +uint32_t +Hacl_Streaming_SHA1_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len); + +void Hacl_Streaming_SHA1_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); + +void Hacl_Streaming_SHA1_legacy_free(Hacl_Streaming_MD_state_32 *s); + +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_copy(Hacl_Streaming_MD_state_32 *s0); + +void Hacl_Streaming_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); + +#if defined(__cplusplus) +} +#endif + +#define __Hacl_Hash_SHA1_H_DEFINED +#endif diff --git a/Modules/_hacl/Hacl_Streaming_SHA2.c b/Modules/_hacl/Hacl_Streaming_SHA2.c index 8169c7a356731e..69c3be8cdf7fd1 100644 --- a/Modules/_hacl/Hacl_Streaming_SHA2.c +++ b/Modules/_hacl/Hacl_Streaming_SHA2.c @@ -477,17 +477,14 @@ static inline void sha384_finish(uint64_t *st, uint8_t *h) Allocate initial state for the SHA2_256 hash. The state is to be freed by calling `free_256`. */ -Hacl_Streaming_SHA2_state_sha2_224 *Hacl_Streaming_SHA2_create_in_256(void) +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_256(void) { uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint32_t)); - Hacl_Streaming_SHA2_state_sha2_224 + Hacl_Streaming_MD_state_32 s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - Hacl_Streaming_SHA2_state_sha2_224 - *p = - (Hacl_Streaming_SHA2_state_sha2_224 *)KRML_HOST_MALLOC(sizeof ( - Hacl_Streaming_SHA2_state_sha2_224 - )); + Hacl_Streaming_MD_state_32 + *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); p[0U] = s; sha256_init(block_state); return p; @@ -499,10 +496,9 @@ The state is to be freed by calling `free_256`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_SHA2_state_sha2_224 -*Hacl_Streaming_SHA2_copy_256(Hacl_Streaming_SHA2_state_sha2_224 *s0) +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_copy_256(Hacl_Streaming_MD_state_32 *s0) { - Hacl_Streaming_SHA2_state_sha2_224 scrut = *s0; + Hacl_Streaming_MD_state_32 scrut = *s0; uint32_t *block_state0 = scrut.block_state; uint8_t *buf0 = scrut.buf; uint64_t total_len0 = scrut.total_len; @@ -510,13 +506,10 @@ Hacl_Streaming_SHA2_state_sha2_224 memcpy(buf, buf0, (uint32_t)64U * sizeof (uint8_t)); uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint32_t)); memcpy(block_state, block_state0, (uint32_t)8U * sizeof (uint32_t)); - Hacl_Streaming_SHA2_state_sha2_224 + Hacl_Streaming_MD_state_32 s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; - Hacl_Streaming_SHA2_state_sha2_224 - *p = - (Hacl_Streaming_SHA2_state_sha2_224 *)KRML_HOST_MALLOC(sizeof ( - Hacl_Streaming_SHA2_state_sha2_224 - )); + Hacl_Streaming_MD_state_32 + *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); p[0U] = s; return p; } @@ -524,21 +517,21 @@ Hacl_Streaming_SHA2_state_sha2_224 /** Reset an existing state to the initial hash state with empty data. */ -void Hacl_Streaming_SHA2_init_256(Hacl_Streaming_SHA2_state_sha2_224 *s) +void Hacl_Streaming_SHA2_init_256(Hacl_Streaming_MD_state_32 *s) { - Hacl_Streaming_SHA2_state_sha2_224 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *s; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; sha256_init(block_state); - Hacl_Streaming_SHA2_state_sha2_224 + Hacl_Streaming_MD_state_32 tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; s[0U] = tmp; } static inline uint32_t -update_224_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *data, uint32_t len) +update_224_256(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) { - Hacl_Streaming_SHA2_state_sha2_224 s = *p; + Hacl_Streaming_MD_state_32 s = *p; uint64_t total_len = s.total_len; if ((uint64_t)len > (uint64_t)2305843009213693951U - total_len) { @@ -555,7 +548,7 @@ update_224_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *data, uint32_t le } if (len <= (uint32_t)64U - sz) { - Hacl_Streaming_SHA2_state_sha2_224 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *p; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; @@ -574,7 +567,7 @@ update_224_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *data, uint32_t le *p = ( - (Hacl_Streaming_SHA2_state_sha2_224){ + (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, .total_len = total_len2 @@ -583,7 +576,7 @@ update_224_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *data, uint32_t le } else if (sz == (uint32_t)0U) { - Hacl_Streaming_SHA2_state_sha2_224 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *p; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; @@ -620,7 +613,7 @@ update_224_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *data, uint32_t le *p = ( - (Hacl_Streaming_SHA2_state_sha2_224){ + (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, .total_len = total_len1 + (uint64_t)len @@ -632,7 +625,7 @@ update_224_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *data, uint32_t le uint32_t diff = (uint32_t)64U - sz; uint8_t *data1 = data; uint8_t *data2 = data + diff; - Hacl_Streaming_SHA2_state_sha2_224 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *p; uint32_t *block_state10 = s1.block_state; uint8_t *buf0 = s1.buf; uint64_t total_len10 = s1.total_len; @@ -651,13 +644,13 @@ update_224_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *data, uint32_t le *p = ( - (Hacl_Streaming_SHA2_state_sha2_224){ + (Hacl_Streaming_MD_state_32){ .block_state = block_state10, .buf = buf0, .total_len = total_len2 } ); - Hacl_Streaming_SHA2_state_sha2_224 s10 = *p; + Hacl_Streaming_MD_state_32 s10 = *p; uint32_t *block_state1 = s10.block_state; uint8_t *buf = s10.buf; uint64_t total_len1 = s10.total_len; @@ -700,7 +693,7 @@ update_224_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *data, uint32_t le *p = ( - (Hacl_Streaming_SHA2_state_sha2_224){ + (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, .total_len = total_len1 + (uint64_t)(len - diff) @@ -719,7 +712,7 @@ This function is identical to the update function for SHA2_224. */ uint32_t Hacl_Streaming_SHA2_update_256( - Hacl_Streaming_SHA2_state_sha2_224 *p, + Hacl_Streaming_MD_state_32 *p, uint8_t *input, uint32_t input_len ) @@ -733,9 +726,9 @@ valid after a call to `finish_256`, meaning the user may feed more data into the hash via `update_256`. (The finish_256 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *dst) +void Hacl_Streaming_SHA2_finish_256(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) { - Hacl_Streaming_SHA2_state_sha2_224 scrut = *p; + Hacl_Streaming_MD_state_32 scrut = *p; uint32_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; @@ -773,9 +766,9 @@ Free a state allocated with `create_in_256`. This function is identical to the free function for SHA2_224. */ -void Hacl_Streaming_SHA2_free_256(Hacl_Streaming_SHA2_state_sha2_224 *s) +void Hacl_Streaming_SHA2_free_256(Hacl_Streaming_MD_state_32 *s) { - Hacl_Streaming_SHA2_state_sha2_224 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *s; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; KRML_HOST_FREE(block_state); @@ -802,36 +795,33 @@ void Hacl_Streaming_SHA2_sha256(uint8_t *input, uint32_t input_len, uint8_t *dst sha256_finish(st, rb); } -Hacl_Streaming_SHA2_state_sha2_224 *Hacl_Streaming_SHA2_create_in_224(void) +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_224(void) { uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint32_t)); - Hacl_Streaming_SHA2_state_sha2_224 + Hacl_Streaming_MD_state_32 s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - Hacl_Streaming_SHA2_state_sha2_224 - *p = - (Hacl_Streaming_SHA2_state_sha2_224 *)KRML_HOST_MALLOC(sizeof ( - Hacl_Streaming_SHA2_state_sha2_224 - )); + Hacl_Streaming_MD_state_32 + *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); p[0U] = s; sha224_init(block_state); return p; } -void Hacl_Streaming_SHA2_init_224(Hacl_Streaming_SHA2_state_sha2_224 *s) +void Hacl_Streaming_SHA2_init_224(Hacl_Streaming_MD_state_32 *s) { - Hacl_Streaming_SHA2_state_sha2_224 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *s; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; sha224_init(block_state); - Hacl_Streaming_SHA2_state_sha2_224 + Hacl_Streaming_MD_state_32 tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; s[0U] = tmp; } uint32_t Hacl_Streaming_SHA2_update_224( - Hacl_Streaming_SHA2_state_sha2_224 *p, + Hacl_Streaming_MD_state_32 *p, uint8_t *input, uint32_t input_len ) @@ -844,9 +834,9 @@ Write the resulting hash into `dst`, an array of 28 bytes. The state remains valid after a call to `finish_224`, meaning the user may feed more data into the hash via `update_224`. */ -void Hacl_Streaming_SHA2_finish_224(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *dst) +void Hacl_Streaming_SHA2_finish_224(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) { - Hacl_Streaming_SHA2_state_sha2_224 scrut = *p; + Hacl_Streaming_MD_state_32 scrut = *p; uint32_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; @@ -879,7 +869,7 @@ void Hacl_Streaming_SHA2_finish_224(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8 sha224_finish(tmp_block_state, dst); } -void Hacl_Streaming_SHA2_free_224(Hacl_Streaming_SHA2_state_sha2_224 *p) +void Hacl_Streaming_SHA2_free_224(Hacl_Streaming_MD_state_32 *p) { Hacl_Streaming_SHA2_free_256(p); } @@ -903,17 +893,14 @@ void Hacl_Streaming_SHA2_sha224(uint8_t *input, uint32_t input_len, uint8_t *dst sha224_finish(st, rb); } -Hacl_Streaming_SHA2_state_sha2_384 *Hacl_Streaming_SHA2_create_in_512(void) +Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_512(void) { uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)128U, sizeof (uint8_t)); uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); - Hacl_Streaming_SHA2_state_sha2_384 + Hacl_Streaming_MD_state_64 s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - Hacl_Streaming_SHA2_state_sha2_384 - *p = - (Hacl_Streaming_SHA2_state_sha2_384 *)KRML_HOST_MALLOC(sizeof ( - Hacl_Streaming_SHA2_state_sha2_384 - )); + Hacl_Streaming_MD_state_64 + *p = (Hacl_Streaming_MD_state_64 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_64)); p[0U] = s; Hacl_SHA2_Scalar32_sha512_init(block_state); return p; @@ -925,10 +912,9 @@ The state is to be freed by calling `free_512`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_SHA2_state_sha2_384 -*Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_SHA2_state_sha2_384 *s0) +Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_MD_state_64 *s0) { - Hacl_Streaming_SHA2_state_sha2_384 scrut = *s0; + Hacl_Streaming_MD_state_64 scrut = *s0; uint64_t *block_state0 = scrut.block_state; uint8_t *buf0 = scrut.buf; uint64_t total_len0 = scrut.total_len; @@ -936,32 +922,29 @@ Hacl_Streaming_SHA2_state_sha2_384 memcpy(buf, buf0, (uint32_t)128U * sizeof (uint8_t)); uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); memcpy(block_state, block_state0, (uint32_t)8U * sizeof (uint64_t)); - Hacl_Streaming_SHA2_state_sha2_384 + Hacl_Streaming_MD_state_64 s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; - Hacl_Streaming_SHA2_state_sha2_384 - *p = - (Hacl_Streaming_SHA2_state_sha2_384 *)KRML_HOST_MALLOC(sizeof ( - Hacl_Streaming_SHA2_state_sha2_384 - )); + Hacl_Streaming_MD_state_64 + *p = (Hacl_Streaming_MD_state_64 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_64)); p[0U] = s; return p; } -void Hacl_Streaming_SHA2_init_512(Hacl_Streaming_SHA2_state_sha2_384 *s) +void Hacl_Streaming_SHA2_init_512(Hacl_Streaming_MD_state_64 *s) { - Hacl_Streaming_SHA2_state_sha2_384 scrut = *s; + Hacl_Streaming_MD_state_64 scrut = *s; uint8_t *buf = scrut.buf; uint64_t *block_state = scrut.block_state; Hacl_SHA2_Scalar32_sha512_init(block_state); - Hacl_Streaming_SHA2_state_sha2_384 + Hacl_Streaming_MD_state_64 tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; s[0U] = tmp; } static inline uint32_t -update_384_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *data, uint32_t len) +update_384_512(Hacl_Streaming_MD_state_64 *p, uint8_t *data, uint32_t len) { - Hacl_Streaming_SHA2_state_sha2_384 s = *p; + Hacl_Streaming_MD_state_64 s = *p; uint64_t total_len = s.total_len; if ((uint64_t)len > (uint64_t)18446744073709551615U - total_len) { @@ -978,7 +961,7 @@ update_384_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *data, uint32_t le } if (len <= (uint32_t)128U - sz) { - Hacl_Streaming_SHA2_state_sha2_384 s1 = *p; + Hacl_Streaming_MD_state_64 s1 = *p; uint64_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; @@ -997,7 +980,7 @@ update_384_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *data, uint32_t le *p = ( - (Hacl_Streaming_SHA2_state_sha2_384){ + (Hacl_Streaming_MD_state_64){ .block_state = block_state1, .buf = buf, .total_len = total_len2 @@ -1006,7 +989,7 @@ update_384_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *data, uint32_t le } else if (sz == (uint32_t)0U) { - Hacl_Streaming_SHA2_state_sha2_384 s1 = *p; + Hacl_Streaming_MD_state_64 s1 = *p; uint64_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; @@ -1043,7 +1026,7 @@ update_384_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *data, uint32_t le *p = ( - (Hacl_Streaming_SHA2_state_sha2_384){ + (Hacl_Streaming_MD_state_64){ .block_state = block_state1, .buf = buf, .total_len = total_len1 + (uint64_t)len @@ -1055,7 +1038,7 @@ update_384_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *data, uint32_t le uint32_t diff = (uint32_t)128U - sz; uint8_t *data1 = data; uint8_t *data2 = data + diff; - Hacl_Streaming_SHA2_state_sha2_384 s1 = *p; + Hacl_Streaming_MD_state_64 s1 = *p; uint64_t *block_state10 = s1.block_state; uint8_t *buf0 = s1.buf; uint64_t total_len10 = s1.total_len; @@ -1074,13 +1057,13 @@ update_384_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *data, uint32_t le *p = ( - (Hacl_Streaming_SHA2_state_sha2_384){ + (Hacl_Streaming_MD_state_64){ .block_state = block_state10, .buf = buf0, .total_len = total_len2 } ); - Hacl_Streaming_SHA2_state_sha2_384 s10 = *p; + Hacl_Streaming_MD_state_64 s10 = *p; uint64_t *block_state1 = s10.block_state; uint8_t *buf = s10.buf; uint64_t total_len1 = s10.total_len; @@ -1123,7 +1106,7 @@ update_384_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *data, uint32_t le *p = ( - (Hacl_Streaming_SHA2_state_sha2_384){ + (Hacl_Streaming_MD_state_64){ .block_state = block_state1, .buf = buf, .total_len = total_len1 + (uint64_t)(len - diff) @@ -1142,7 +1125,7 @@ This function is identical to the update function for SHA2_384. */ uint32_t Hacl_Streaming_SHA2_update_512( - Hacl_Streaming_SHA2_state_sha2_384 *p, + Hacl_Streaming_MD_state_64 *p, uint8_t *input, uint32_t input_len ) @@ -1156,9 +1139,9 @@ valid after a call to `finish_512`, meaning the user may feed more data into the hash via `update_512`. (The finish_512 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *dst) +void Hacl_Streaming_SHA2_finish_512(Hacl_Streaming_MD_state_64 *p, uint8_t *dst) { - Hacl_Streaming_SHA2_state_sha2_384 scrut = *p; + Hacl_Streaming_MD_state_64 scrut = *p; uint64_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; @@ -1200,9 +1183,9 @@ Free a state allocated with `create_in_512`. This function is identical to the free function for SHA2_384. */ -void Hacl_Streaming_SHA2_free_512(Hacl_Streaming_SHA2_state_sha2_384 *s) +void Hacl_Streaming_SHA2_free_512(Hacl_Streaming_MD_state_64 *s) { - Hacl_Streaming_SHA2_state_sha2_384 scrut = *s; + Hacl_Streaming_MD_state_64 scrut = *s; uint8_t *buf = scrut.buf; uint64_t *block_state = scrut.block_state; KRML_HOST_FREE(block_state); @@ -1229,36 +1212,33 @@ void Hacl_Streaming_SHA2_sha512(uint8_t *input, uint32_t input_len, uint8_t *dst sha512_finish(st, rb); } -Hacl_Streaming_SHA2_state_sha2_384 *Hacl_Streaming_SHA2_create_in_384(void) +Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_384(void) { uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)128U, sizeof (uint8_t)); uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); - Hacl_Streaming_SHA2_state_sha2_384 + Hacl_Streaming_MD_state_64 s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - Hacl_Streaming_SHA2_state_sha2_384 - *p = - (Hacl_Streaming_SHA2_state_sha2_384 *)KRML_HOST_MALLOC(sizeof ( - Hacl_Streaming_SHA2_state_sha2_384 - )); + Hacl_Streaming_MD_state_64 + *p = (Hacl_Streaming_MD_state_64 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_64)); p[0U] = s; sha384_init(block_state); return p; } -void Hacl_Streaming_SHA2_init_384(Hacl_Streaming_SHA2_state_sha2_384 *s) +void Hacl_Streaming_SHA2_init_384(Hacl_Streaming_MD_state_64 *s) { - Hacl_Streaming_SHA2_state_sha2_384 scrut = *s; + Hacl_Streaming_MD_state_64 scrut = *s; uint8_t *buf = scrut.buf; uint64_t *block_state = scrut.block_state; sha384_init(block_state); - Hacl_Streaming_SHA2_state_sha2_384 + Hacl_Streaming_MD_state_64 tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; s[0U] = tmp; } uint32_t Hacl_Streaming_SHA2_update_384( - Hacl_Streaming_SHA2_state_sha2_384 *p, + Hacl_Streaming_MD_state_64 *p, uint8_t *input, uint32_t input_len ) @@ -1271,9 +1251,9 @@ Write the resulting hash into `dst`, an array of 48 bytes. The state remains valid after a call to `finish_384`, meaning the user may feed more data into the hash via `update_384`. */ -void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *dst) +void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_MD_state_64 *p, uint8_t *dst) { - Hacl_Streaming_SHA2_state_sha2_384 scrut = *p; + Hacl_Streaming_MD_state_64 scrut = *p; uint64_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; @@ -1310,7 +1290,7 @@ void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8 sha384_finish(tmp_block_state, dst); } -void Hacl_Streaming_SHA2_free_384(Hacl_Streaming_SHA2_state_sha2_384 *p) +void Hacl_Streaming_SHA2_free_384(Hacl_Streaming_MD_state_64 *p) { Hacl_Streaming_SHA2_free_512(p); } diff --git a/Modules/_hacl/Hacl_Streaming_SHA2.h b/Modules/_hacl/Hacl_Streaming_SHA2.h index 2c905854f336fd..b58df4c4d121c9 100644 --- a/Modules/_hacl/Hacl_Streaming_SHA2.h +++ b/Modules/_hacl/Hacl_Streaming_SHA2.h @@ -36,33 +36,22 @@ extern "C" { #include "krml/lowstar_endianness.h" #include "krml/internal/target.h" +#include "Hacl_Streaming_Types.h" -typedef struct Hacl_Streaming_SHA2_state_sha2_224_s -{ - uint32_t *block_state; - uint8_t *buf; - uint64_t total_len; -} -Hacl_Streaming_SHA2_state_sha2_224; +typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_SHA2_state_sha2_224; -typedef Hacl_Streaming_SHA2_state_sha2_224 Hacl_Streaming_SHA2_state_sha2_256; +typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_SHA2_state_sha2_256; -typedef struct Hacl_Streaming_SHA2_state_sha2_384_s -{ - uint64_t *block_state; - uint8_t *buf; - uint64_t total_len; -} -Hacl_Streaming_SHA2_state_sha2_384; +typedef Hacl_Streaming_MD_state_64 Hacl_Streaming_SHA2_state_sha2_384; -typedef Hacl_Streaming_SHA2_state_sha2_384 Hacl_Streaming_SHA2_state_sha2_512; +typedef Hacl_Streaming_MD_state_64 Hacl_Streaming_SHA2_state_sha2_512; /** Allocate initial state for the SHA2_256 hash. The state is to be freed by calling `free_256`. */ -Hacl_Streaming_SHA2_state_sha2_224 *Hacl_Streaming_SHA2_create_in_256(void); +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_256(void); /** Copies the state passed as argument into a newly allocated state (deep copy). @@ -70,13 +59,12 @@ The state is to be freed by calling `free_256`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_SHA2_state_sha2_224 -*Hacl_Streaming_SHA2_copy_256(Hacl_Streaming_SHA2_state_sha2_224 *s0); +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_copy_256(Hacl_Streaming_MD_state_32 *s0); /** Reset an existing state to the initial hash state with empty data. */ -void Hacl_Streaming_SHA2_init_256(Hacl_Streaming_SHA2_state_sha2_224 *s); +void Hacl_Streaming_SHA2_init_256(Hacl_Streaming_MD_state_32 *s); /** Feed an arbitrary amount of data into the hash. This function returns 0 for @@ -87,7 +75,7 @@ This function is identical to the update function for SHA2_224. */ uint32_t Hacl_Streaming_SHA2_update_256( - Hacl_Streaming_SHA2_state_sha2_224 *p, + Hacl_Streaming_MD_state_32 *p, uint8_t *input, uint32_t input_len ); @@ -98,27 +86,27 @@ valid after a call to `finish_256`, meaning the user may feed more data into the hash via `update_256`. (The finish_256 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *dst); +void Hacl_Streaming_SHA2_finish_256(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); /** Free a state allocated with `create_in_256`. This function is identical to the free function for SHA2_224. */ -void Hacl_Streaming_SHA2_free_256(Hacl_Streaming_SHA2_state_sha2_224 *s); +void Hacl_Streaming_SHA2_free_256(Hacl_Streaming_MD_state_32 *s); /** Hash `input`, of len `input_len`, into `dst`, an array of 32 bytes. */ void Hacl_Streaming_SHA2_sha256(uint8_t *input, uint32_t input_len, uint8_t *dst); -Hacl_Streaming_SHA2_state_sha2_224 *Hacl_Streaming_SHA2_create_in_224(void); +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_224(void); -void Hacl_Streaming_SHA2_init_224(Hacl_Streaming_SHA2_state_sha2_224 *s); +void Hacl_Streaming_SHA2_init_224(Hacl_Streaming_MD_state_32 *s); uint32_t Hacl_Streaming_SHA2_update_224( - Hacl_Streaming_SHA2_state_sha2_224 *p, + Hacl_Streaming_MD_state_32 *p, uint8_t *input, uint32_t input_len ); @@ -128,16 +116,16 @@ Write the resulting hash into `dst`, an array of 28 bytes. The state remains valid after a call to `finish_224`, meaning the user may feed more data into the hash via `update_224`. */ -void Hacl_Streaming_SHA2_finish_224(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *dst); +void Hacl_Streaming_SHA2_finish_224(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); -void Hacl_Streaming_SHA2_free_224(Hacl_Streaming_SHA2_state_sha2_224 *p); +void Hacl_Streaming_SHA2_free_224(Hacl_Streaming_MD_state_32 *p); /** Hash `input`, of len `input_len`, into `dst`, an array of 28 bytes. */ void Hacl_Streaming_SHA2_sha224(uint8_t *input, uint32_t input_len, uint8_t *dst); -Hacl_Streaming_SHA2_state_sha2_384 *Hacl_Streaming_SHA2_create_in_512(void); +Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_512(void); /** Copies the state passed as argument into a newly allocated state (deep copy). @@ -145,10 +133,9 @@ The state is to be freed by calling `free_512`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_SHA2_state_sha2_384 -*Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_SHA2_state_sha2_384 *s0); +Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_MD_state_64 *s0); -void Hacl_Streaming_SHA2_init_512(Hacl_Streaming_SHA2_state_sha2_384 *s); +void Hacl_Streaming_SHA2_init_512(Hacl_Streaming_MD_state_64 *s); /** Feed an arbitrary amount of data into the hash. This function returns 0 for @@ -159,7 +146,7 @@ This function is identical to the update function for SHA2_384. */ uint32_t Hacl_Streaming_SHA2_update_512( - Hacl_Streaming_SHA2_state_sha2_384 *p, + Hacl_Streaming_MD_state_64 *p, uint8_t *input, uint32_t input_len ); @@ -170,27 +157,27 @@ valid after a call to `finish_512`, meaning the user may feed more data into the hash via `update_512`. (The finish_512 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *dst); +void Hacl_Streaming_SHA2_finish_512(Hacl_Streaming_MD_state_64 *p, uint8_t *dst); /** Free a state allocated with `create_in_512`. This function is identical to the free function for SHA2_384. */ -void Hacl_Streaming_SHA2_free_512(Hacl_Streaming_SHA2_state_sha2_384 *s); +void Hacl_Streaming_SHA2_free_512(Hacl_Streaming_MD_state_64 *s); /** Hash `input`, of len `input_len`, into `dst`, an array of 64 bytes. */ void Hacl_Streaming_SHA2_sha512(uint8_t *input, uint32_t input_len, uint8_t *dst); -Hacl_Streaming_SHA2_state_sha2_384 *Hacl_Streaming_SHA2_create_in_384(void); +Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_384(void); -void Hacl_Streaming_SHA2_init_384(Hacl_Streaming_SHA2_state_sha2_384 *s); +void Hacl_Streaming_SHA2_init_384(Hacl_Streaming_MD_state_64 *s); uint32_t Hacl_Streaming_SHA2_update_384( - Hacl_Streaming_SHA2_state_sha2_384 *p, + Hacl_Streaming_MD_state_64 *p, uint8_t *input, uint32_t input_len ); @@ -200,9 +187,9 @@ Write the resulting hash into `dst`, an array of 48 bytes. The state remains valid after a call to `finish_384`, meaning the user may feed more data into the hash via `update_384`. */ -void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *dst); +void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_MD_state_64 *p, uint8_t *dst); -void Hacl_Streaming_SHA2_free_384(Hacl_Streaming_SHA2_state_sha2_384 *p); +void Hacl_Streaming_SHA2_free_384(Hacl_Streaming_MD_state_64 *p); /** Hash `input`, of len `input_len`, into `dst`, an array of 48 bytes. diff --git a/Modules/_hacl/Hacl_Streaming_Types.h b/Modules/_hacl/Hacl_Streaming_Types.h new file mode 100644 index 00000000000000..51057611ca978d --- /dev/null +++ b/Modules/_hacl/Hacl_Streaming_Types.h @@ -0,0 +1,59 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __Hacl_Streaming_Types_H +#define __Hacl_Streaming_Types_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +typedef struct Hacl_Streaming_MD_state_32_s +{ + uint32_t *block_state; + uint8_t *buf; + uint64_t total_len; +} +Hacl_Streaming_MD_state_32; + +typedef struct Hacl_Streaming_MD_state_64_s +{ + uint64_t *block_state; + uint8_t *buf; + uint64_t total_len; +} +Hacl_Streaming_MD_state_64; + +#if defined(__cplusplus) +} +#endif + +#define __Hacl_Streaming_Types_H_DEFINED +#endif diff --git a/Modules/_hacl/include/krml/FStar_UInt128_Verified.h b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h index ee160193539e28..3d36d440735530 100644 --- a/Modules/_hacl/include/krml/FStar_UInt128_Verified.h +++ b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h @@ -7,13 +7,12 @@ #ifndef __FStar_UInt128_Verified_H #define __FStar_UInt128_Verified_H - - #include "FStar_UInt_8_16_32_64.h" #include #include #include "krml/types.h" #include "krml/internal/target.h" + static inline uint64_t FStar_UInt128_constant_time_carry(uint64_t a, uint64_t b) { return (a ^ ((a ^ b) | ((a - b) ^ b))) >> (uint32_t)63U; diff --git a/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h b/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h index 965afc836fd12b..a56c7d613498b7 100644 --- a/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h +++ b/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h @@ -7,15 +7,13 @@ #ifndef __FStar_UInt_8_16_32_64_H #define __FStar_UInt_8_16_32_64_H - - - #include #include #include "krml/lowstar_endianness.h" #include "krml/types.h" #include "krml/internal/target.h" + static inline uint64_t FStar_UInt64_eq_mask(uint64_t a, uint64_t b) { uint64_t x = a ^ b; diff --git a/Modules/_hacl/include/krml/internal/target.h b/Modules/_hacl/include/krml/internal/target.h index 9ef59859a554b5..dcbe7007b17be8 100644 --- a/Modules/_hacl/include/krml/internal/target.h +++ b/Modules/_hacl/include/krml/internal/target.h @@ -31,6 +31,10 @@ # define KRML_HOST_FREE free #endif +#ifndef KRML_HOST_IGNORE +# define KRML_HOST_IGNORE(x) (void)(x) +#endif + /* Macros for prettier unrolling of loops */ #define KRML_LOOP1(i, n, x) { \ x \ diff --git a/Modules/_hacl/internal/Hacl_Hash_MD5.h b/Modules/_hacl/internal/Hacl_Hash_MD5.h new file mode 100644 index 00000000000000..87ad4cf228d91b --- /dev/null +++ b/Modules/_hacl/internal/Hacl_Hash_MD5.h @@ -0,0 +1,61 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __internal_Hacl_Hash_MD5_H +#define __internal_Hacl_Hash_MD5_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "../Hacl_Hash_MD5.h" + +void Hacl_Hash_Core_MD5_legacy_init(uint32_t *s); + +void Hacl_Hash_Core_MD5_legacy_finish(uint32_t *s, uint8_t *dst); + +void Hacl_Hash_MD5_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks); + +void +Hacl_Hash_MD5_legacy_update_last( + uint32_t *s, + uint64_t prev_len, + uint8_t *input, + uint32_t input_len +); + +void Hacl_Hash_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); + +#if defined(__cplusplus) +} +#endif + +#define __internal_Hacl_Hash_MD5_H_DEFINED +#endif diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA1.h b/Modules/_hacl/internal/Hacl_Hash_SHA1.h new file mode 100644 index 00000000000000..d2d9df44c6c14c --- /dev/null +++ b/Modules/_hacl/internal/Hacl_Hash_SHA1.h @@ -0,0 +1,61 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __internal_Hacl_Hash_SHA1_H +#define __internal_Hacl_Hash_SHA1_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "../Hacl_Hash_SHA1.h" + +void Hacl_Hash_Core_SHA1_legacy_init(uint32_t *s); + +void Hacl_Hash_Core_SHA1_legacy_finish(uint32_t *s, uint8_t *dst); + +void Hacl_Hash_SHA1_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks); + +void +Hacl_Hash_SHA1_legacy_update_last( + uint32_t *s, + uint64_t prev_len, + uint8_t *input, + uint32_t input_len +); + +void Hacl_Hash_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); + +#if defined(__cplusplus) +} +#endif + +#define __internal_Hacl_Hash_SHA1_H_DEFINED +#endif diff --git a/Modules/_hacl/python_hacl_namespaces.h b/Modules/_hacl/python_hacl_namespaces.h index ac12f386257b19..ee28f244266b85 100644 --- a/Modules/_hacl/python_hacl_namespaces.h +++ b/Modules/_hacl/python_hacl_namespaces.h @@ -43,4 +43,21 @@ #define Hacl_Streaming_SHA2_sha512 python_hashlib_Hacl_Streaming_SHA2_sha512 #define Hacl_Streaming_SHA2_sha384 python_hashlib_Hacl_Streaming_SHA2_sha384 +#define Hacl_Streaming_MD5_legacy_create_in python_hashlib_Hacl_Streaming_MD5_legacy_create_in +#define Hacl_Streaming_MD5_legacy_init python_hashlib_Hacl_Streaming_MD5_legacy_init +#define Hacl_Streaming_MD5_legacy_update python_hashlib_Hacl_Streaming_MD5_legacy_update +#define Hacl_Streaming_MD5_legacy_finish python_hashlib_Hacl_Streaming_MD5_legacy_finish +#define Hacl_Streaming_MD5_legacy_free python_hashlib_Hacl_Streaming_MD5_legacy_free +#define Hacl_Streaming_MD5_legacy_copy python_hashlib_Hacl_Streaming_MD5_legacy_copy +#define Hacl_Streaming_MD5_legacy_hash python_hashlib_Hacl_Streaming_MD5_legacy_hash + +#define Hacl_Streaming_SHA1_legacy_create_in python_hashlib_Hacl_Streaming_SHA1_legacy_create_in +#define Hacl_Streaming_SHA1_legacy_init python_hashlib_Hacl_Streaming_SHA1_legacy_init +#define Hacl_Streaming_SHA1_legacy_update python_hashlib_Hacl_Streaming_SHA1_legacy_update +#define Hacl_Streaming_SHA1_legacy_finish python_hashlib_Hacl_Streaming_SHA1_legacy_finish +#define Hacl_Streaming_SHA1_legacy_free python_hashlib_Hacl_Streaming_SHA1_legacy_free +#define Hacl_Streaming_SHA1_legacy_copy python_hashlib_Hacl_Streaming_SHA1_legacy_copy +#define Hacl_Streaming_SHA1_legacy_hash python_hashlib_Hacl_Streaming_SHA1_legacy_hash + + #endif // _PYTHON_HACL_NAMESPACES_H diff --git a/Modules/_hacl/refresh.sh b/Modules/_hacl/refresh.sh index dba8cb3972ea17..76b92ec4599102 100755 --- a/Modules/_hacl/refresh.sh +++ b/Modules/_hacl/refresh.sh @@ -22,7 +22,7 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. -expected_hacl_star_rev=4751fc2b11639f651718abf8522fcc36902ca67c +expected_hacl_star_rev=13e0c6721ac9206c4249ecc1dc04ed617ad1e262 hacl_dir="$(realpath "$1")" cd "$(dirname "$0")" @@ -41,8 +41,15 @@ fi declare -a dist_files dist_files=( Hacl_Streaming_SHA2.h + Hacl_Streaming_Types.h + Hacl_Hash_SHA1.h + internal/Hacl_Hash_SHA1.h + Hacl_Hash_MD5.h + internal/Hacl_Hash_MD5.h internal/Hacl_SHA2_Generic.h Hacl_Streaming_SHA2.c + Hacl_Hash_SHA1.c + Hacl_Hash_MD5.c ) declare -a include_files diff --git a/Modules/md5module.c b/Modules/md5module.c index 48b11e0779f875..df3d6a4a70d789 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -43,283 +43,17 @@ typedef long long MD5_INT64; /* 64-bit integer */ #define MD5_BLOCKSIZE 64 #define MD5_DIGESTSIZE 16 -/* The structure for storing MD5 info */ +#include "_hacl/Hacl_Hash_MD5.h" -struct md5_state { - MD5_INT64 length; - MD5_INT32 state[4], curlen; - unsigned char buf[MD5_BLOCKSIZE]; -}; typedef struct { PyObject_HEAD - struct md5_state hash_state; + Hacl_Streaming_MD5_state *hash_state; } MD5object; #include "clinic/md5module.c.h" -/* ------------------------------------------------------------------------ - * - * This code for the MD5 algorithm was noted as public domain. The - * original headers are pasted below. - * - * Several changes have been made to make it more compatible with the - * Python environment and desired interface. - * - */ - -/* LibTomCrypt, modular cryptographic library -- Tom St Denis - * - * LibTomCrypt is a library that provides various cryptographic - * algorithms in a highly modular and flexible manner. - * - * The library is free for all purposes without any express - * guarantee it works. - * - * Tom St Denis, tomstdenis@gmail.com, https://www.libtom.net - */ - -/* rotate the hard way (platform optimizations could be done) */ -#define ROLc(x, y) ( (((unsigned long)(x)<<(unsigned long)((y)&31)) | (((unsigned long)(x)&0xFFFFFFFFUL)>>(unsigned long)(32-((y)&31)))) & 0xFFFFFFFFUL) - -/* Endian Neutral macros that work on all platforms */ - -#define STORE32L(x, y) \ - { (y)[3] = (unsigned char)(((x)>>24)&255); (y)[2] = (unsigned char)(((x)>>16)&255); \ - (y)[1] = (unsigned char)(((x)>>8)&255); (y)[0] = (unsigned char)((x)&255); } - -#define LOAD32L(x, y) \ - { x = ((unsigned long)((y)[3] & 255)<<24) | \ - ((unsigned long)((y)[2] & 255)<<16) | \ - ((unsigned long)((y)[1] & 255)<<8) | \ - ((unsigned long)((y)[0] & 255)); } - -#define STORE64L(x, y) \ - { (y)[7] = (unsigned char)(((x)>>56)&255); (y)[6] = (unsigned char)(((x)>>48)&255); \ - (y)[5] = (unsigned char)(((x)>>40)&255); (y)[4] = (unsigned char)(((x)>>32)&255); \ - (y)[3] = (unsigned char)(((x)>>24)&255); (y)[2] = (unsigned char)(((x)>>16)&255); \ - (y)[1] = (unsigned char)(((x)>>8)&255); (y)[0] = (unsigned char)((x)&255); } - - -/* MD5 macros */ - -#define F(x,y,z) (z ^ (x & (y ^ z))) -#define G(x,y,z) (y ^ (z & (y ^ x))) -#define H(x,y,z) (x^y^z) -#define I(x,y,z) (y^(x|(~z))) - -#define FF(a,b,c,d,M,s,t) \ - a = (a + F(b,c,d) + M + t); a = ROLc(a, s) + b; - -#define GG(a,b,c,d,M,s,t) \ - a = (a + G(b,c,d) + M + t); a = ROLc(a, s) + b; - -#define HH(a,b,c,d,M,s,t) \ - a = (a + H(b,c,d) + M + t); a = ROLc(a, s) + b; - -#define II(a,b,c,d,M,s,t) \ - a = (a + I(b,c,d) + M + t); a = ROLc(a, s) + b; - - -static void md5_compress(struct md5_state *md5, const unsigned char *buf) -{ - MD5_INT32 i, W[16], a, b, c, d; - - assert(md5 != NULL); - assert(buf != NULL); - - /* copy the state into 512-bits into W[0..15] */ - for (i = 0; i < 16; i++) { - LOAD32L(W[i], buf + (4*i)); - } - - /* copy state */ - a = md5->state[0]; - b = md5->state[1]; - c = md5->state[2]; - d = md5->state[3]; - - FF(a,b,c,d,W[0],7,0xd76aa478UL) - FF(d,a,b,c,W[1],12,0xe8c7b756UL) - FF(c,d,a,b,W[2],17,0x242070dbUL) - FF(b,c,d,a,W[3],22,0xc1bdceeeUL) - FF(a,b,c,d,W[4],7,0xf57c0fafUL) - FF(d,a,b,c,W[5],12,0x4787c62aUL) - FF(c,d,a,b,W[6],17,0xa8304613UL) - FF(b,c,d,a,W[7],22,0xfd469501UL) - FF(a,b,c,d,W[8],7,0x698098d8UL) - FF(d,a,b,c,W[9],12,0x8b44f7afUL) - FF(c,d,a,b,W[10],17,0xffff5bb1UL) - FF(b,c,d,a,W[11],22,0x895cd7beUL) - FF(a,b,c,d,W[12],7,0x6b901122UL) - FF(d,a,b,c,W[13],12,0xfd987193UL) - FF(c,d,a,b,W[14],17,0xa679438eUL) - FF(b,c,d,a,W[15],22,0x49b40821UL) - GG(a,b,c,d,W[1],5,0xf61e2562UL) - GG(d,a,b,c,W[6],9,0xc040b340UL) - GG(c,d,a,b,W[11],14,0x265e5a51UL) - GG(b,c,d,a,W[0],20,0xe9b6c7aaUL) - GG(a,b,c,d,W[5],5,0xd62f105dUL) - GG(d,a,b,c,W[10],9,0x02441453UL) - GG(c,d,a,b,W[15],14,0xd8a1e681UL) - GG(b,c,d,a,W[4],20,0xe7d3fbc8UL) - GG(a,b,c,d,W[9],5,0x21e1cde6UL) - GG(d,a,b,c,W[14],9,0xc33707d6UL) - GG(c,d,a,b,W[3],14,0xf4d50d87UL) - GG(b,c,d,a,W[8],20,0x455a14edUL) - GG(a,b,c,d,W[13],5,0xa9e3e905UL) - GG(d,a,b,c,W[2],9,0xfcefa3f8UL) - GG(c,d,a,b,W[7],14,0x676f02d9UL) - GG(b,c,d,a,W[12],20,0x8d2a4c8aUL) - HH(a,b,c,d,W[5],4,0xfffa3942UL) - HH(d,a,b,c,W[8],11,0x8771f681UL) - HH(c,d,a,b,W[11],16,0x6d9d6122UL) - HH(b,c,d,a,W[14],23,0xfde5380cUL) - HH(a,b,c,d,W[1],4,0xa4beea44UL) - HH(d,a,b,c,W[4],11,0x4bdecfa9UL) - HH(c,d,a,b,W[7],16,0xf6bb4b60UL) - HH(b,c,d,a,W[10],23,0xbebfbc70UL) - HH(a,b,c,d,W[13],4,0x289b7ec6UL) - HH(d,a,b,c,W[0],11,0xeaa127faUL) - HH(c,d,a,b,W[3],16,0xd4ef3085UL) - HH(b,c,d,a,W[6],23,0x04881d05UL) - HH(a,b,c,d,W[9],4,0xd9d4d039UL) - HH(d,a,b,c,W[12],11,0xe6db99e5UL) - HH(c,d,a,b,W[15],16,0x1fa27cf8UL) - HH(b,c,d,a,W[2],23,0xc4ac5665UL) - II(a,b,c,d,W[0],6,0xf4292244UL) - II(d,a,b,c,W[7],10,0x432aff97UL) - II(c,d,a,b,W[14],15,0xab9423a7UL) - II(b,c,d,a,W[5],21,0xfc93a039UL) - II(a,b,c,d,W[12],6,0x655b59c3UL) - II(d,a,b,c,W[3],10,0x8f0ccc92UL) - II(c,d,a,b,W[10],15,0xffeff47dUL) - II(b,c,d,a,W[1],21,0x85845dd1UL) - II(a,b,c,d,W[8],6,0x6fa87e4fUL) - II(d,a,b,c,W[15],10,0xfe2ce6e0UL) - II(c,d,a,b,W[6],15,0xa3014314UL) - II(b,c,d,a,W[13],21,0x4e0811a1UL) - II(a,b,c,d,W[4],6,0xf7537e82UL) - II(d,a,b,c,W[11],10,0xbd3af235UL) - II(c,d,a,b,W[2],15,0x2ad7d2bbUL) - II(b,c,d,a,W[9],21,0xeb86d391UL) - - md5->state[0] = md5->state[0] + a; - md5->state[1] = md5->state[1] + b; - md5->state[2] = md5->state[2] + c; - md5->state[3] = md5->state[3] + d; -} - - -/** - Initialize the hash state - @param md5 The hash state you wish to initialize -*/ -static void -md5_init(struct md5_state *md5) -{ - assert(md5 != NULL); - md5->state[0] = 0x67452301UL; - md5->state[1] = 0xefcdab89UL; - md5->state[2] = 0x98badcfeUL; - md5->state[3] = 0x10325476UL; - md5->curlen = 0; - md5->length = 0; -} - -/** - Process a block of memory though the hash - @param md5 The hash state - @param in The data to hash - @param inlen The length of the data (octets) -*/ -static void -md5_process(struct md5_state *md5, const unsigned char *in, Py_ssize_t inlen) -{ - Py_ssize_t n; - - assert(md5 != NULL); - assert(in != NULL); - assert(md5->curlen <= sizeof(md5->buf)); - - while (inlen > 0) { - if (md5->curlen == 0 && inlen >= MD5_BLOCKSIZE) { - md5_compress(md5, in); - md5->length += MD5_BLOCKSIZE * 8; - in += MD5_BLOCKSIZE; - inlen -= MD5_BLOCKSIZE; - } else { - n = Py_MIN(inlen, (Py_ssize_t)(MD5_BLOCKSIZE - md5->curlen)); - memcpy(md5->buf + md5->curlen, in, (size_t)n); - md5->curlen += (MD5_INT32)n; - in += n; - inlen -= n; - if (md5->curlen == MD5_BLOCKSIZE) { - md5_compress(md5, md5->buf); - md5->length += 8*MD5_BLOCKSIZE; - md5->curlen = 0; - } - } - } -} - -/** - Terminate the hash to get the digest - @param md5 The hash state - @param out [out] The destination of the hash (16 bytes) -*/ -static void -md5_done(struct md5_state *md5, unsigned char *out) -{ - int i; - - assert(md5 != NULL); - assert(out != NULL); - assert(md5->curlen < sizeof(md5->buf)); - - /* increase the length of the message */ - md5->length += md5->curlen * 8; - - /* append the '1' bit */ - md5->buf[md5->curlen++] = (unsigned char)0x80; - - /* if the length is currently above 56 bytes we append zeros - * then compress. Then we can fall back to padding zeros and length - * encoding like normal. - */ - if (md5->curlen > 56) { - while (md5->curlen < 64) { - md5->buf[md5->curlen++] = (unsigned char)0; - } - md5_compress(md5, md5->buf); - md5->curlen = 0; - } - - /* pad up to 56 bytes of zeroes */ - while (md5->curlen < 56) { - md5->buf[md5->curlen++] = (unsigned char)0; - } - - /* store length */ - STORE64L(md5->length, md5->buf+56); - md5_compress(md5, md5->buf); - - /* copy output */ - for (i = 0; i < 4; i++) { - STORE32L(md5->state[i], out+(4*i)); - } -} - -/* .Source: /cvs/libtom/libtomcrypt/src/hashes/md5.c,v $ */ -/* .Revision: 1.10 $ */ -/* .Date: 2007/05/12 14:25:28 $ */ - -/* - * End of copied MD5 code. - * - * ------------------------------------------------------------------------ - */ typedef struct { PyTypeObject* md5_type; @@ -350,8 +84,9 @@ MD5_traverse(PyObject *ptr, visitproc visit, void *arg) } static void -MD5_dealloc(PyObject *ptr) +MD5_dealloc(MD5object *ptr) { + Hacl_Streaming_MD5_legacy_free(ptr->hash_state); PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -379,7 +114,7 @@ MD5Type_copy_impl(MD5object *self, PyTypeObject *cls) if ((newobj = newMD5object(st))==NULL) return NULL; - newobj->hash_state = self->hash_state; + newobj->hash_state = Hacl_Streaming_MD5_legacy_copy(self->hash_state); return (PyObject *)newobj; } @@ -394,10 +129,7 @@ MD5Type_digest_impl(MD5object *self) /*[clinic end generated code: output=eb691dc4190a07ec input=bc0c4397c2994be6]*/ { unsigned char digest[MD5_DIGESTSIZE]; - struct md5_state temp; - - temp = self->hash_state; - md5_done(&temp, digest); + Hacl_Streaming_MD5_legacy_finish(self->hash_state, digest); return PyBytes_FromStringAndSize((const char *)digest, MD5_DIGESTSIZE); } @@ -412,15 +144,21 @@ MD5Type_hexdigest_impl(MD5object *self) /*[clinic end generated code: output=17badced1f3ac932 input=b60b19de644798dd]*/ { unsigned char digest[MD5_DIGESTSIZE]; - struct md5_state temp; - - /* Get the raw (binary) digest value */ - temp = self->hash_state; - md5_done(&temp, digest); - + Hacl_Streaming_MD5_legacy_finish(self->hash_state, digest); return _Py_strhex((const char*)digest, MD5_DIGESTSIZE); } +static void update(Hacl_Streaming_MD5_state *state, uint8_t *buf, Py_ssize_t len) { +#if PY_SSIZE_T_MAX > UINT32_MAX + while (len > UINT32_MAX) { + Hacl_Streaming_MD5_legacy_update(state, buf, UINT32_MAX); + len -= UINT32_MAX; + buf += UINT32_MAX; + } +#endif + Hacl_Streaming_MD5_legacy_update(state, buf, (uint32_t) len); +} + /*[clinic input] MD5Type.update @@ -438,7 +176,7 @@ MD5Type_update(MD5object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - md5_process(&self->hash_state, buf.buf, buf.len); + update(self->hash_state, buf.buf, buf.len); PyBuffer_Release(&buf); Py_RETURN_NONE; @@ -531,7 +269,7 @@ _md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - md5_init(&new->hash_state); + new->hash_state = Hacl_Streaming_MD5_legacy_create_in(); if (PyErr_Occurred()) { Py_DECREF(new); @@ -540,7 +278,7 @@ _md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - md5_process(&new->hash_state, buf.buf, buf.len); + update(new->hash_state, buf.buf, buf.len); PyBuffer_Release(&buf); } diff --git a/Modules/sha1module.c b/Modules/sha1module.c index 9153557fbde740..0f50d532acf925 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -43,260 +43,16 @@ typedef long long SHA1_INT64; /* 64-bit integer */ #define SHA1_BLOCKSIZE 64 #define SHA1_DIGESTSIZE 20 -/* The structure for storing SHA1 info */ - -struct sha1_state { - SHA1_INT64 length; - SHA1_INT32 state[5], curlen; - unsigned char buf[SHA1_BLOCKSIZE]; -}; +#include "_hacl/Hacl_Hash_SHA1.h" typedef struct { PyObject_HEAD - struct sha1_state hash_state; + Hacl_Streaming_SHA1_state *hash_state; } SHA1object; #include "clinic/sha1module.c.h" -/* ------------------------------------------------------------------------ - * - * This code for the SHA1 algorithm was noted as public domain. The - * original headers are pasted below. - * - * Several changes have been made to make it more compatible with the - * Python environment and desired interface. - * - */ - -/* LibTomCrypt, modular cryptographic library -- Tom St Denis - * - * LibTomCrypt is a library that provides various cryptographic - * algorithms in a highly modular and flexible manner. - * - * The library is free for all purposes without any express - * guarantee it works. - * - * Tom St Denis, tomstdenis@gmail.com, https://www.libtom.net - */ - -/* rotate the hard way (platform optimizations could be done) */ -#define ROL(x, y) ( (((unsigned long)(x)<<(unsigned long)((y)&31)) | (((unsigned long)(x)&0xFFFFFFFFUL)>>(unsigned long)(32-((y)&31)))) & 0xFFFFFFFFUL) -#define ROLc(x, y) ( (((unsigned long)(x)<<(unsigned long)((y)&31)) | (((unsigned long)(x)&0xFFFFFFFFUL)>>(unsigned long)(32-((y)&31)))) & 0xFFFFFFFFUL) - -/* Endian Neutral macros that work on all platforms */ - -#define STORE32H(x, y) \ - { (y)[0] = (unsigned char)(((x)>>24)&255); (y)[1] = (unsigned char)(((x)>>16)&255); \ - (y)[2] = (unsigned char)(((x)>>8)&255); (y)[3] = (unsigned char)((x)&255); } - -#define LOAD32H(x, y) \ - { x = ((unsigned long)((y)[0] & 255)<<24) | \ - ((unsigned long)((y)[1] & 255)<<16) | \ - ((unsigned long)((y)[2] & 255)<<8) | \ - ((unsigned long)((y)[3] & 255)); } - -#define STORE64H(x, y) \ - { (y)[0] = (unsigned char)(((x)>>56)&255); (y)[1] = (unsigned char)(((x)>>48)&255); \ - (y)[2] = (unsigned char)(((x)>>40)&255); (y)[3] = (unsigned char)(((x)>>32)&255); \ - (y)[4] = (unsigned char)(((x)>>24)&255); (y)[5] = (unsigned char)(((x)>>16)&255); \ - (y)[6] = (unsigned char)(((x)>>8)&255); (y)[7] = (unsigned char)((x)&255); } - - -/* SHA1 macros */ - -#define F0(x,y,z) (z ^ (x & (y ^ z))) -#define F1(x,y,z) (x ^ y ^ z) -#define F2(x,y,z) ((x & y) | (z & (x | y))) -#define F3(x,y,z) (x ^ y ^ z) - -static void sha1_compress(struct sha1_state *sha1, unsigned char *buf) -{ - SHA1_INT32 a,b,c,d,e,W[80],i; - - /* copy the state into 512-bits into W[0..15] */ - for (i = 0; i < 16; i++) { - LOAD32H(W[i], buf + (4*i)); - } - - /* copy state */ - a = sha1->state[0]; - b = sha1->state[1]; - c = sha1->state[2]; - d = sha1->state[3]; - e = sha1->state[4]; - - /* expand it */ - for (i = 16; i < 80; i++) { - W[i] = ROL(W[i-3] ^ W[i-8] ^ W[i-14] ^ W[i-16], 1); - } - - /* compress */ - /* round one */ - #define FF_0(a,b,c,d,e,i) e = (ROLc(a, 5) + F0(b,c,d) + e + W[i] + 0x5a827999UL); b = ROLc(b, 30); - #define FF_1(a,b,c,d,e,i) e = (ROLc(a, 5) + F1(b,c,d) + e + W[i] + 0x6ed9eba1UL); b = ROLc(b, 30); - #define FF_2(a,b,c,d,e,i) e = (ROLc(a, 5) + F2(b,c,d) + e + W[i] + 0x8f1bbcdcUL); b = ROLc(b, 30); - #define FF_3(a,b,c,d,e,i) e = (ROLc(a, 5) + F3(b,c,d) + e + W[i] + 0xca62c1d6UL); b = ROLc(b, 30); - - for (i = 0; i < 20; ) { - FF_0(a,b,c,d,e,i++); - FF_0(e,a,b,c,d,i++); - FF_0(d,e,a,b,c,i++); - FF_0(c,d,e,a,b,i++); - FF_0(b,c,d,e,a,i++); - } - - /* round two */ - for (; i < 40; ) { - FF_1(a,b,c,d,e,i++); - FF_1(e,a,b,c,d,i++); - FF_1(d,e,a,b,c,i++); - FF_1(c,d,e,a,b,i++); - FF_1(b,c,d,e,a,i++); - } - - /* round three */ - for (; i < 60; ) { - FF_2(a,b,c,d,e,i++); - FF_2(e,a,b,c,d,i++); - FF_2(d,e,a,b,c,i++); - FF_2(c,d,e,a,b,i++); - FF_2(b,c,d,e,a,i++); - } - - /* round four */ - for (; i < 80; ) { - FF_3(a,b,c,d,e,i++); - FF_3(e,a,b,c,d,i++); - FF_3(d,e,a,b,c,i++); - FF_3(c,d,e,a,b,i++); - FF_3(b,c,d,e,a,i++); - } - - #undef FF_0 - #undef FF_1 - #undef FF_2 - #undef FF_3 - - /* store */ - sha1->state[0] = sha1->state[0] + a; - sha1->state[1] = sha1->state[1] + b; - sha1->state[2] = sha1->state[2] + c; - sha1->state[3] = sha1->state[3] + d; - sha1->state[4] = sha1->state[4] + e; -} - -/** - Initialize the hash state - @param sha1 The hash state you wish to initialize -*/ -static void -sha1_init(struct sha1_state *sha1) -{ - assert(sha1 != NULL); - sha1->state[0] = 0x67452301UL; - sha1->state[1] = 0xefcdab89UL; - sha1->state[2] = 0x98badcfeUL; - sha1->state[3] = 0x10325476UL; - sha1->state[4] = 0xc3d2e1f0UL; - sha1->curlen = 0; - sha1->length = 0; -} - -/** - Process a block of memory though the hash - @param sha1 The hash state - @param in The data to hash - @param inlen The length of the data (octets) -*/ -static void -sha1_process(struct sha1_state *sha1, - const unsigned char *in, Py_ssize_t inlen) -{ - Py_ssize_t n; - - assert(sha1 != NULL); - assert(in != NULL); - assert(sha1->curlen <= sizeof(sha1->buf)); - - while (inlen > 0) { - if (sha1->curlen == 0 && inlen >= SHA1_BLOCKSIZE) { - sha1_compress(sha1, (unsigned char *)in); - sha1->length += SHA1_BLOCKSIZE * 8; - in += SHA1_BLOCKSIZE; - inlen -= SHA1_BLOCKSIZE; - } else { - n = Py_MIN(inlen, (Py_ssize_t)(SHA1_BLOCKSIZE - sha1->curlen)); - memcpy(sha1->buf + sha1->curlen, in, (size_t)n); - sha1->curlen += (SHA1_INT32)n; - in += n; - inlen -= n; - if (sha1->curlen == SHA1_BLOCKSIZE) { - sha1_compress(sha1, sha1->buf); - sha1->length += 8*SHA1_BLOCKSIZE; - sha1->curlen = 0; - } - } - } -} - -/** - Terminate the hash to get the digest - @param sha1 The hash state - @param out [out] The destination of the hash (20 bytes) -*/ -static void -sha1_done(struct sha1_state *sha1, unsigned char *out) -{ - int i; - - assert(sha1 != NULL); - assert(out != NULL); - assert(sha1->curlen < sizeof(sha1->buf)); - - /* increase the length of the message */ - sha1->length += sha1->curlen * 8; - - /* append the '1' bit */ - sha1->buf[sha1->curlen++] = (unsigned char)0x80; - - /* if the length is currently above 56 bytes we append zeros - * then compress. Then we can fall back to padding zeros and length - * encoding like normal. - */ - if (sha1->curlen > 56) { - while (sha1->curlen < 64) { - sha1->buf[sha1->curlen++] = (unsigned char)0; - } - sha1_compress(sha1, sha1->buf); - sha1->curlen = 0; - } - - /* pad up to 56 bytes of zeroes */ - while (sha1->curlen < 56) { - sha1->buf[sha1->curlen++] = (unsigned char)0; - } - - /* store length */ - STORE64H(sha1->length, sha1->buf+56); - sha1_compress(sha1, sha1->buf); - - /* copy output */ - for (i = 0; i < 5; i++) { - STORE32H(sha1->state[i], out+(4*i)); - } -} - - -/* .Source: /cvs/libtom/libtomcrypt/src/hashes/sha1.c,v $ */ -/* .Revision: 1.10 $ */ -/* .Date: 2007/05/12 14:25:28 $ */ - -/* - * End of copied SHA1 code. - * - * ------------------------------------------------------------------------ - */ typedef struct { PyTypeObject* sha1_type; @@ -328,8 +84,9 @@ SHA1_traverse(PyObject *ptr, visitproc visit, void *arg) } static void -SHA1_dealloc(PyObject *ptr) +SHA1_dealloc(SHA1object *ptr) { + Hacl_Streaming_SHA1_legacy_free(ptr->hash_state); PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -357,7 +114,7 @@ SHA1Type_copy_impl(SHA1object *self, PyTypeObject *cls) if ((newobj = newSHA1object(st)) == NULL) return NULL; - newobj->hash_state = self->hash_state; + newobj->hash_state = Hacl_Streaming_SHA1_legacy_copy(self->hash_state); return (PyObject *)newobj; } @@ -372,10 +129,7 @@ SHA1Type_digest_impl(SHA1object *self) /*[clinic end generated code: output=2f05302a7aa2b5cb input=13824b35407444bd]*/ { unsigned char digest[SHA1_DIGESTSIZE]; - struct sha1_state temp; - - temp = self->hash_state; - sha1_done(&temp, digest); + Hacl_Streaming_SHA1_legacy_finish(self->hash_state, digest); return PyBytes_FromStringAndSize((const char *)digest, SHA1_DIGESTSIZE); } @@ -390,15 +144,21 @@ SHA1Type_hexdigest_impl(SHA1object *self) /*[clinic end generated code: output=4161fd71e68c6659 input=97691055c0c74ab0]*/ { unsigned char digest[SHA1_DIGESTSIZE]; - struct sha1_state temp; - - /* Get the raw (binary) digest value */ - temp = self->hash_state; - sha1_done(&temp, digest); - + Hacl_Streaming_SHA1_legacy_finish(self->hash_state, digest); return _Py_strhex((const char *)digest, SHA1_DIGESTSIZE); } +static void update(Hacl_Streaming_SHA1_state *state, uint8_t *buf, Py_ssize_t len) { +#if PY_SSIZE_T_MAX > UINT32_MAX + while (len > UINT32_MAX) { + Hacl_Streaming_SHA1_legacy_update(state, buf, UINT32_MAX); + len -= UINT32_MAX; + buf += UINT32_MAX; + } +#endif + Hacl_Streaming_SHA1_legacy_update(state, buf, (uint32_t) len); +} + /*[clinic input] SHA1Type.update @@ -416,7 +176,7 @@ SHA1Type_update(SHA1object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - sha1_process(&self->hash_state, buf.buf, buf.len); + update(self->hash_state, buf.buf, buf.len); PyBuffer_Release(&buf); Py_RETURN_NONE; @@ -509,7 +269,7 @@ _sha1_sha1_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - sha1_init(&new->hash_state); + new->hash_state = Hacl_Streaming_SHA1_legacy_create_in(); if (PyErr_Occurred()) { Py_DECREF(new); @@ -518,7 +278,7 @@ _sha1_sha1_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - sha1_process(&new->hash_state, buf.buf, buf.len); + update(new->hash_state, buf.buf, buf.len); PyBuffer_Release(&buf); } diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 222963bc42d17c..85dc8caa458ed9 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -400,12 +400,14 @@ + + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index efb96222043ac2..98e7d59ba1020c 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -848,6 +848,9 @@ Modules + + Modules + Modules @@ -863,6 +866,9 @@ Modules + + Modules + Modules diff --git a/configure b/configure index 17dc62fb63de3b..557519ad86e06d 100755 --- a/configure +++ b/configure @@ -26844,7 +26844,7 @@ fi as_fn_append MODULE_BLOCK "MODULE__MD5_STATE=$py_cv_module__md5$as_nl" if test "x$py_cv_module__md5" = xyes; then : - + as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE$as_nl" fi @@ -26878,7 +26878,7 @@ fi as_fn_append MODULE_BLOCK "MODULE__SHA1_STATE=$py_cv_module__sha1$as_nl" if test "x$py_cv_module__sha1" = xyes; then : - + as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE$as_nl" fi diff --git a/configure.ac b/configure.ac index bc288b86cfa590..982f669acbcfe5 100644 --- a/configure.ac +++ b/configure.ac @@ -7194,8 +7194,12 @@ PY_STDLIB_MOD_SIMPLE([unicodedata]) dnl By default we always compile these even when OpenSSL is available dnl (issue #14693). The modules are small. -PY_STDLIB_MOD([_md5], [test "$with_builtin_md5" = yes]) -PY_STDLIB_MOD([_sha1], [test "$with_builtin_sha1" = yes]) +PY_STDLIB_MOD([_md5], + [test "$with_builtin_md5" = yes], [], + [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE]) +PY_STDLIB_MOD([_sha1], + [test "$with_builtin_sha1" = yes], [], + [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE]) PY_STDLIB_MOD([_sha2], [test "$with_builtin_sha2" = yes], [], [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE]) From 8f647477f0ab5362741d261701b5bcd76bd69ec1 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Wed, 22 Feb 2023 18:55:03 -0500 Subject: [PATCH 76/76] Fix syntax error in struct doc example (#102160) Missing closing ) reported on Discuss by Chukwudi Nwachukwu. --- Doc/library/struct.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 69d95f27cb61d9..9c0e32ba16bf68 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -371,7 +371,7 @@ ordering:: >>> from struct import * >>> pack(">bhl", 1, 2, 3) b'\x01\x00\x02\x00\x00\x00\x03' - >>> unpack('>bhl', b'\x01\x00\x02\x00\x00\x00\x03' + >>> unpack('>bhl', b'\x01\x00\x02\x00\x00\x00\x03') (1, 2, 3) >>> calcsize('>bhl') 7