diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e929ac8c474b2f..9162f9c7bb1576 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -88,7 +88,6 @@ Objects/exceptions.c @iritkatriel **/sha* @gpshead @tiran Modules/md5* @gpshead @tiran **/*blake* @gpshead @tiran -Modules/_blake2/** @gpshead @tiran Modules/_hacl/** @gpshead # logging @@ -255,8 +254,8 @@ Modules/_interp*module.c @ericsnowcurrently Lib/test/test_interpreters/ @ericsnowcurrently # Android -**/*Android* @mhsmith -**/*android* @mhsmith +**/*Android* @mhsmith @freakboy3742 +**/*android* @mhsmith @freakboy3742 # iOS (but not termios) **/iOS* @freakboy3742 @@ -267,7 +266,7 @@ Lib/test/test_interpreters/ @ericsnowcurrently **/*-ios* @freakboy3742 # WebAssembly -/Tools/wasm/ @brettcannon +/Tools/wasm/ @brettcannon @freakboy3742 # SBOM /Misc/externals.spdx.json @sethmlarson diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 754f179f105591..48f05818a38f96 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -28,7 +28,7 @@ concurrency: jobs: interpreter: name: Interpreter (Debug) - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 90 steps: - uses: actions/checkout@v4 @@ -61,7 +61,7 @@ jobs: - true - false llvm: - - 18 + - 19 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 @@ -85,19 +85,19 @@ jobs: compiler: clang - target: x86_64-unknown-linux-gnu/gcc architecture: x86_64 - runner: ubuntu-latest + runner: ubuntu-22.04 compiler: gcc - target: x86_64-unknown-linux-gnu/clang architecture: x86_64 - runner: ubuntu-latest + runner: ubuntu-22.04 compiler: clang - target: aarch64-unknown-linux-gnu/gcc architecture: aarch64 - runner: ubuntu-latest + runner: ubuntu-22.04 compiler: gcc - target: aarch64-unknown-linux-gnu/clang architecture: aarch64 - runner: ubuntu-latest + runner: ubuntu-22.04 compiler: clang env: CC: ${{ matrix.compiler }} @@ -121,10 +121,15 @@ jobs: choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} + # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. + # This is a bug in the macOS runner image where the pre-installed Python is installed in the same + # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes + # the symlink to the pre-installed Python so that the Homebrew Python is used instead. - name: Native macOS if: runner.os == 'macOS' run: | brew update + find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} SDKROOT="$(xcrun --show-sdk-path)" \ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} @@ -164,7 +169,11 @@ jobs: jit-with-disabled-gil: name: Free-Threaded (Debug) needs: interpreter - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 + strategy: + matrix: + llvm: + - 19 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -172,8 +181,8 @@ jobs: python-version: '3.11' - name: Build with JIT enabled and GIL disabled run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh 18 - export PATH="$(llvm-config-18 --bindir):$PATH" + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" ./configure --enable-experimental-jit --with-pydebug --disable-gil make all --jobs 4 - name: Run tests diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index 4aaf3905e81c8a..c92ef4c653a675 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -105,7 +105,7 @@ The following functions provide locale-independent string to number conversions. If ``s`` represents a value that is too large to store in a float (for example, ``"1e500"`` is such a string on many platforms) then - if ``overflow_exception`` is ``NULL`` return ``Py_HUGE_VAL`` (with + if ``overflow_exception`` is ``NULL`` return ``Py_INFINITY`` (with an appropriate sign) and don't set any exception. Otherwise, ``overflow_exception`` must point to a Python exception object; raise that exception and return ``-1.0``. In both cases, set diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 0c59b3da0795cb..8a185486fe44f1 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -355,7 +355,7 @@ slot typedefs +-----------------------------+-----------------------------+----------------------+ | :c:type:`newfunc` | .. line-block:: | :c:type:`PyObject` * | | | | | -| | :c:type:`PyObject` * | | +| | :c:type:`PyTypeObject` * | | | | :c:type:`PyObject` * | | | | :c:type:`PyObject` * | | +-----------------------------+-----------------------------+----------------------+ @@ -2647,7 +2647,7 @@ Slot Type typedefs See :c:member:`~PyTypeObject.tp_free`. -.. c:type:: PyObject *(*newfunc)(PyObject *, PyObject *, PyObject *) +.. c:type:: PyObject *(*newfunc)(PyTypeObject *, PyObject *, PyObject *) See :c:member:`~PyTypeObject.tp_new`. diff --git a/Doc/deprecations/pending-removal-in-3.14.rst b/Doc/deprecations/pending-removal-in-3.14.rst index b8791b8d6c387e..0863853339b8b5 100644 --- a/Doc/deprecations/pending-removal-in-3.14.rst +++ b/Doc/deprecations/pending-removal-in-3.14.rst @@ -85,7 +85,7 @@ Pending removal in Python 3.14 :meth:`~pathlib.PurePath.relative_to`: passing additional arguments is deprecated. -* :mod:`pkgutil`: :func:`~pkgutil.find_loader` and :func:`~pkgutil.get_loader` +* :mod:`pkgutil`: :func:`!pkgutil.find_loader` and :func:!pkgutil.get_loader` now raise :exc:`DeprecationWarning`; use :func:`importlib.util.find_spec` instead. (Contributed by Nikita Sobolev in :gh:`97850`.) @@ -103,9 +103,6 @@ Pending removal in Python 3.14 if :ref:`named placeholders ` are used and *parameters* is a sequence instead of a :class:`dict`. - * date and datetime adapter, date and timestamp converter: - see the :mod:`sqlite3` documentation for suggested replacement recipes. - * :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was deprecated in :pep:`626` since 3.10 and was planned to be removed in 3.12, diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index b77a36393b2769..42e40de152148c 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -143,7 +143,8 @@ ABC Inherits from Abstract Methods Mi :class:`Set` :class:`Collection` ``__contains__``, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``, ``__iter__``, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``, - ``__len__`` ``__sub__``, ``__xor__``, and ``isdisjoint`` + ``__len__`` ``__sub__``, ``__rsub__``, ``__xor__``, ``__rxor__`` + and ``isdisjoint`` :class:`MutableSet` :class:`Set` ``__contains__``, Inherited :class:`Set` methods and ``__iter__``, ``clear``, ``pop``, ``remove``, ``__ior__``, @@ -162,7 +163,7 @@ ABC Inherits from Abstract Methods Mi ``__len__`` -:class:`MappingView` :class:`Sized` ``__len__`` +:class:`MappingView` :class:`Sized` ``__init__``, ``__len__`` and ``__repr__`` :class:`ItemsView` :class:`MappingView`, ``__contains__``, :class:`Set` ``__iter__`` :class:`KeysView` :class:`MappingView`, ``__contains__``, diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 6c12d1b5e0dcea..cf203a714ba126 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1647,7 +1647,7 @@ iterations of the loop. .. versionadded:: 3.13 -.. opcode:: FORMAT_SPEC +.. opcode:: FORMAT_WITH_SPEC Formats the given value with the given format spec:: diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 242b2436439903..16a9b0326e9f3d 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -44,7 +44,7 @@ using function-call syntax:: ... BLUE = 3 >>> # functional syntax - >>> Color = Enum('Color', ['RED', 'GREEN', 'BLUE']) + >>> Color = Enum('Color', [('RED', 1), ('GREEN', 2), ('BLUE', 3)]) Even though we can use :keyword:`class` syntax to create Enums, Enums are not normal Python classes. See diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 5c0efd21318f2c..03fc41fa793977 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1293,9 +1293,10 @@ are always available. They are listed here in alphabetical order. .. class:: object() - Return a new featureless object. :class:`object` is a base for all classes. - It has methods that are common to all instances of Python classes. This - function does not accept any arguments. + This is the ultimate base class of all other classes. It has methods + that are common to all instances of Python classes. When the constructor + is called, it returns a new featureless object. The constructor does not + accept any arguments. .. note:: diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index ddfc5c06d825c0..37cd237357aa4b 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -229,10 +229,10 @@ Distribution metadata .. class:: PackageMetadata A concrete implementation of the - `PackageMetadata protocol `_. + `PackageMetadata protocol `_. - In addition to providing the defined protocol methods and attributes, subscripting - the instance is equivalent to calling the :meth:`!get` method. + In addition to providing the defined protocol methods and attributes, subscripting + the instance is equivalent to calling the :meth:`!get` method. Every `Distribution Package `_ includes some metadata, which you can extract using the :func:`!metadata` function:: diff --git a/Doc/library/math.rst b/Doc/library/math.rst index dd2ba419b5bd12..6be61c99274eb7 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -26,6 +26,92 @@ The following functions are provided by this module. Except when explicitly noted otherwise, all return values are floats. +==================================================== ============================================ +**Number-theoretic and representation functions** +-------------------------------------------------------------------------------------------------- +:func:`ceil(x) ` Ceiling of *x*, the smallest integer greater than or equal to *x* +:func:`comb(n, k) ` Number of ways to choose *k* items from *n* items without repetition and without order +:func:`copysign(x, y) ` Magnitude (absolute value) of *x* with the sign of *y* +:func:`fabs(x) ` Absolute value of *x* +:func:`factorial(n) ` *n* factorial +:func:`floor (x) ` Floor of *x*, the largest integer less than or equal to *x* +:func:`fma(x, y, z) ` Fused multiply-add operation: ``(x * y) + z`` +:func:`fmod(x, y) ` Remainder of division ``x / y`` +:func:`frexp(x) ` Mantissa and exponent of *x* +:func:`fsum(iterable) ` Sum of values in the input *iterable* +:func:`gcd(*integers) ` Greatest common divisor of the integer arguments +:func:`isclose(a, b, rel_tol, abs_tol) ` Check if the values *a* and *b* are close to each other +:func:`isfinite(x) ` Check if *x* is neither an infinity nor a NaN +:func:`isinf(x) ` Check if *x* is a positive or negative infinity +:func:`isnan(x) ` Check if *x* is a NaN (not a number) +:func:`isqrt(n) ` Integer square root of a nonnegative integer *n* +:func:`lcm(*integers) ` Least common multiple of the integer arguments +:func:`ldexp(x, i) ` ``x * (2**i)``, inverse of function :func:`frexp` +:func:`modf(x) ` Fractional and integer parts of *x* +:func:`nextafter(x, y, steps) ` Floating-point value *steps* steps after *x* towards *y* +:func:`perm(n, k) ` Number of ways to choose *k* items from *n* items without repetition and with order +:func:`prod(iterable, start) ` Product of elements in the input *iterable* with a *start* value +:func:`remainder(x, y) ` Remainder of *x* with respect to *y* +:func:`sumprod(p, q) ` Sum of products from two iterables *p* and *q* +:func:`trunc(x) ` Integer part of *x* +:func:`ulp(x) ` Value of the least significant bit of *x* + +**Power and logarithmic functions** +-------------------------------------------------------------------------------------------------- +:func:`cbrt(x) ` Cube root of *x* +:func:`exp(x) ` *e* raised to the power *x* +:func:`exp2(x) ` *2* raised to the power *x* +:func:`expm1(x) ` *e* raised to the power *x*, minus 1 +:func:`log(x, base) ` Logarithm of *x* to the given base (*e* by default) +:func:`log1p(x) ` Natural logarithm of *1+x* (base *e*) +:func:`log2(x) ` Base-2 logarithm of *x* +:func:`log10(x) ` Base-10 logarithm of *x* +:func:`pow(x, y) ` *x* raised to the power *y* +:func:`sqrt(x) ` Square root of *x* + +**Trigonometric functions** +-------------------------------------------------------------------------------------------------- +:func:`acos(x) ` Arc cosine of *x* +:func:`asin(x) ` Arc sine of *x* +:func:`atan(x) ` Arc tangent of *x* +:func:`atan2(y, x) ` ``atan(y / x)`` +:func:`cos(x) ` Cosine of *x* +:func:`dist(p, q) ` Euclidean distance between two points *p* and *q* given as an iterable of coordinates +:func:`hypot(*coordinates) ` Euclidean norm of an iterable of coordinates +:func:`sin(x) ` Sine of *x* +:func:`tan(x) ` Tangent of *x* + +**Angular conversion** +-------------------------------------------------------------------------------------------------- +:func:`degrees(x) ` Convert angle *x* from radians to degrees +:func:`radians(x) ` Convert angle *x* from degrees to radians + +**Hyperbolic functions** +-------------------------------------------------------------------------------------------------- +:func:`acosh(x) ` Inverse hyperbolic cosine of *x* +:func:`asinh(x) ` Inverse hyperbolic sine of *x* +:func:`atanh(x) ` Inverse hyperbolic tangent of *x* +:func:`cosh(x) ` Hyperbolic cosine of *x* +:func:`sinh(x) ` Hyperbolic sine of *x* +:func:`tanh(x) ` Hyperbolic tangent of *x* + +**Special functions** +-------------------------------------------------------------------------------------------------- +:func:`erf(x) ` `Error function `_ at *x* +:func:`erfc(x) ` `Complementary error function `_ at *x* +:func:`gamma(x) ` `Gamma function `_ at *x* +:func:`lgamma(x) ` Natural logarithm of the absolute value of the `Gamma function `_ at *x* + +**Constants** +-------------------------------------------------------------------------------------------------- +:data:`pi` *π* = 3.141592... +:data:`e` *e* = 2.718281... +:data:`tau` *τ* = 2\ *π* = 6.283185... +:data:`inf` Positive infinity +:data:`nan` "Not a number" (NaN) +==================================================== ============================================ + + Number-theoretic and representation functions --------------------------------------------- @@ -447,11 +533,11 @@ Power and logarithmic functions .. function:: pow(x, y) - Return ``x`` raised to the power ``y``. Exceptional cases follow + Return *x* raised to the power *y*. Exceptional cases follow the IEEE 754 standard as far as possible. In particular, ``pow(1.0, x)`` and ``pow(x, 0.0)`` always return ``1.0``, even - when ``x`` is a zero or a NaN. If both ``x`` and ``y`` are finite, - ``x`` is negative, and ``y`` is not an integer then ``pow(x, y)`` + when *x* is a zero or a NaN. If both *x* and *y* are finite, + *x* is negative, and *y* is not an integer then ``pow(x, y)`` is undefined, and raises :exc:`ValueError`. Unlike the built-in ``**`` operator, :func:`math.pow` converts both diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 081d7a6f97b079..f9cded40c2c755 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3675,6 +3675,7 @@ features: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name)) + os.rmdir(top) .. audit-event:: os.walk top,topdown,onerror,followlinks os.walk diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 4380122eb1be7d..b6fb36554f7cec 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1289,6 +1289,35 @@ Reading directories raised. +.. method:: Path.scandir() + + When the path points to a directory, return an iterator of + :class:`os.DirEntry` objects corresponding to entries in the directory. The + returned iterator supports the :term:`context manager` protocol. It is + implemented using :func:`os.scandir` and gives the same guarantees. + + Using :meth:`~Path.scandir` instead of :meth:`~Path.iterdir` can + significantly increase the performance of code that also needs file type or + file attribute information, because :class:`os.DirEntry` objects expose + this information if the operating system provides it when scanning a + directory. + + The following example displays the names of subdirectories. The + ``entry.is_dir()`` check will generally not make an additional system call:: + + >>> p = Path('docs') + >>> with p.scandir() as entries: + ... for entry in entries: + ... if entry.is_dir(): + ... entry.name + ... + '_templates' + '_build' + '_static' + + .. versionadded:: 3.14 + + .. method:: Path.glob(pattern, *, case_sensitive=None, recurse_symlinks=False) Glob the given relative *pattern* in the directory represented by this path, diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 4a39d53a5f1440..20b8f6bcf19117 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -49,25 +49,6 @@ support. this function to raise an exception (in line with :func:`os.path.isdir` behavior). -.. function:: find_loader(fullname) - - Retrieve a module :term:`loader` for the given *fullname*. - - This is a backwards compatibility wrapper around - :func:`importlib.util.find_spec` that converts most failures to - :exc:`ImportError` and only returns the loader rather than the full - :class:`importlib.machinery.ModuleSpec`. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal :pep:`302` import emulation. - - .. versionchanged:: 3.4 - Updated to be based on :pep:`451` - - .. deprecated-removed:: 3.12 3.14 - Use :func:`importlib.util.find_spec` instead. - .. function:: get_importer(path_item) @@ -84,27 +65,6 @@ support. on the package internal :pep:`302` import emulation. -.. function:: get_loader(module_or_name) - - Get a :term:`loader` object for *module_or_name*. - - If the module or package is accessible via the normal import mechanism, a - wrapper around the relevant part of that machinery is returned. Returns - ``None`` if the module cannot be found or imported. If the named module is - not already imported, its containing package (if any) is imported, in order - to establish the package ``__path__``. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal :pep:`302` import emulation. - - .. versionchanged:: 3.4 - Updated to be based on :pep:`451` - - .. deprecated-removed:: 3.12 3.14 - Use :func:`importlib.util.find_spec` instead. - - .. function:: iter_importers(fullname='') Yield :term:`finder` objects for the given module name. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index fc0383823a172b..096892b605b99c 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -2442,6 +2442,7 @@ Some useful URI tricks include: >>> con.execute("CREATE TABLE readonly(data)") Traceback (most recent call last): OperationalError: attempt to write a readonly database + >>> con.close() * Do not implicitly create a new database file if it does not already exist; will raise :exc:`~sqlite3.OperationalError` if unable to create a new file: diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 9e8e44a8abe770..a000bb49f14800 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -589,6 +589,11 @@ The available presentation types for :class:`float` and | | as altered by the other format modifiers. | +---------+----------------------------------------------------------+ +The result should be correctly rounded to a given precision ``p`` of digits +after the decimal point. The rounding mode for :class:`float` matches that +of the :func:`round` builtin. For :class:`~decimal.Decimal`, the rounding +mode of the current :ref:`context ` will be used. + The available presentation types for :class:`complex` are the same as those for :class:`float` (``'%'`` is not allowed). Both the real and imaginary components of a complex number are formatted as floating-point numbers, according to the diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 8e29e57d00f9b2..9cd5db768e9853 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -327,7 +327,7 @@ Functions .. impl-detail:: - On CPython, use the same clock than :func:`time.monotonic` and is a + On CPython, use the same clock as :func:`time.monotonic` and is a monotonic clock, i.e. a clock that cannot go backwards. Use :func:`perf_counter_ns` to avoid the precision loss caused by the @@ -339,7 +339,7 @@ Functions On Windows, the function is now system-wide. .. versionchanged:: 3.13 - Use the same clock than :func:`time.monotonic`. + Use the same clock as :func:`time.monotonic`. .. function:: perf_counter_ns() -> int diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 5ea65cbd8ca94c..0c7e8543f331db 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -180,6 +180,19 @@ If a warning is reported and doesn't match any registered filter then the "default" action is applied (hence its name). + +.. _repeated-warning-suppression-criteria: + +Repeated Warning Suppression Criteria +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The filters that suppress repeated warnings apply the following criteria to determine if a warning is considered a repeat: + +- ``"default"``: A warning is considered a repeat only if the (*message*, *category*, *module*, *lineno*) are all the same. +- ``"module"``: A warning is considered a repeat if the (*message*, *category*, *module*) are the same, ignoring the line number. +- ``"once"``: A warning is considered a repeat if the (*message*, *category*) are the same, ignoring the module and line number. + + .. _describing-warning-filters: Describing Warning Filters @@ -398,7 +411,7 @@ Available Functions ------------------- -.. function:: warn(message, category=None, stacklevel=1, source=None, *, skip_file_prefixes=None) +.. function:: warn(message, category=None, stacklevel=1, source=None, *, skip_file_prefixes=()) Issue a warning, or maybe ignore it or raise an exception. The *category* argument, if given, must be a :ref:`warning category class `; it diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index aceb95f2758e38..dfd1addf656a85 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2033,7 +2033,8 @@ Basic customization "informal" string representation of instances of that class is required. This is typically used for debugging, so it is important that the representation - is information-rich and unambiguous. + is information-rich and unambiguous. A default implementation is provided by the + :class:`object` class itself. .. index:: single: string; __str__() (object method) @@ -2043,10 +2044,10 @@ Basic customization .. method:: object.__str__(self) - Called by :func:`str(object) ` and the built-in functions - :func:`format` and :func:`print` to compute the "informal" or nicely + Called by :func:`str(object) `, the default :meth:`__format__` implementation, + and the built-in function :func:`print`, to compute the "informal" or nicely printable string representation of an object. The return value must be a - :ref:`string ` object. + :ref:`str ` object. This method differs from :meth:`object.__repr__` in that there is no expectation that :meth:`__str__` return a valid Python expression: a more @@ -2063,7 +2064,8 @@ Basic customization .. index:: pair: built-in function; bytes Called by :ref:`bytes ` to compute a byte-string representation - of an object. This should return a :class:`bytes` object. + of an object. This should return a :class:`bytes` object. The :class:`object` + class itself does not provide this method. .. index:: single: string; __format__() (object method) @@ -2087,6 +2089,9 @@ Basic customization The return value must be a string object. + The default implementation by the :class:`object` class should be given + an empty *format_spec* string. It delegates to :meth:`__str__`. + .. versionchanged:: 3.4 The __format__ method of ``object`` itself raises a :exc:`TypeError` if passed any non-empty string. @@ -2129,6 +2134,12 @@ Basic customization ``(x` (such as :class:`lists ` or +The following methods can be defined to implement container objects. None of them +are provided by the :class:`object` class itself. Containers usually are +:term:`sequences ` (such as :class:`lists ` or :class:`tuples `) or :term:`mappings ` (like -:class:`dictionaries `), +:term:`dictionaries `), but can represent other containers as well. The first set of methods is used either to emulate a sequence or to emulate a mapping; the difference is that for a sequence, the allowable keys should be the integers *k* for which ``0 <= k < @@ -3460,6 +3474,7 @@ Typical uses of context managers include saving and restoring various kinds of global state, locking and unlocking resources, closing opened files, etc. For more information on context managers, see :ref:`typecontextmanager`. +The :class:`object` class itself does not provide the context manager methods. .. method:: object.__enter__(self) @@ -3709,6 +3724,8 @@ are awaitable. Must return an :term:`iterator`. Should be used to implement :term:`awaitable` objects. For instance, :class:`asyncio.Future` implements this method to be compatible with the :keyword:`await` expression. + The :class:`object` class itself is not awaitable and does not provide + this method. .. note:: @@ -3794,6 +3811,9 @@ its ``__anext__`` method. Asynchronous iterators can be used in an :keyword:`async for` statement. +The :class:`object` class itself does not provide these methods. + + .. method:: object.__aiter__(self) Must return an *asynchronous iterator* object. @@ -3840,6 +3860,8 @@ suspend execution in its ``__aenter__`` and ``__aexit__`` methods. Asynchronous context managers can be used in an :keyword:`async with` statement. +The :class:`object` class itself does not provide these methods. + .. method:: object.__aenter__(self) Semantically similar to :meth:`~object.__enter__`, the only diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 2e6fd419b21106..35b8c7cd8eb049 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -100,6 +100,13 @@ yet another way to substitute values into strings, using placeholders like ``$x`` and replacing them with values from a dictionary, but offers much less control of the formatting. +.. index:: + single: formatted string literal + single: interpolated string literal + single: string; formatted literal + single: string; interpolated literal + single: f-string + single: fstring .. _tut-f-strings: diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 1354355894e375..3640095acbaa2b 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1229,7 +1229,7 @@ Deprecated your code *requires* ``'fork'``. See :ref:`contexts and start methods `. -* :mod:`pkgutil`: :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` +* :mod:`pkgutil`: :func:`!pkgutil.find_loader` and :func:`!pkgutil.get_loader` are deprecated and will be removed in Python 3.14; use :func:`importlib.util.find_spec` instead. (Contributed by Nikita Sobolev in :gh:`97850`.) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 7f9e3107a6e1a0..21bc289c2be5d8 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -380,6 +380,12 @@ pathlib (Contributed by Barney Gale in :gh:`73991`.) +* Add :meth:`pathlib.Path.scandir` to scan a directory and return an iterator + of :class:`os.DirEntry` objects. This is exactly equivalent to calling + :func:`os.scandir` on a path object. + + (Contributed by Barney Gale in :gh:`125413`.) + pdb --- @@ -615,6 +621,13 @@ pathlib :meth:`~pathlib.PurePath.is_relative_to`. In previous versions, any such arguments are joined onto *other*. +pkgutil +------- + +* Remove deprecated :func:`!pkgutil.get_loader` and :func:`!pkgutil.find_loader`. + These had previously raised a :exc:`DeprecationWarning` since Python 3.12. + (Contributed by Bénédikt Tran in :gh:`97850`.) + pty --- @@ -799,6 +812,10 @@ Porting to Python 3.14 Deprecated ---------- +* The :c:macro:`!Py_HUGE_VAL` macro is :term:`soft deprecated`, + use :c:macro:`!Py_INFINITY` instead. + (Contributed by Sergey B Kirpichev in :gh:`120026`.) + * Macros :c:macro:`!Py_IS_NAN`, :c:macro:`!Py_IS_INFINITY` and :c:macro:`!Py_IS_FINITE` are :term:`soft deprecated`, use instead :c:macro:`!isnan`, :c:macro:`!isinf` and diff --git a/Include/floatobject.h b/Include/floatobject.h index 8963c16832a4bc..4d24a76edd5de1 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -21,10 +21,10 @@ PyAPI_DATA(PyTypeObject) PyFloat_Type; #define Py_RETURN_INF(sign) \ do { \ if (copysign(1., sign) == 1.) { \ - return PyFloat_FromDouble(Py_HUGE_VAL); \ + return PyFloat_FromDouble(Py_INFINITY); \ } \ else { \ - return PyFloat_FromDouble(-Py_HUGE_VAL); \ + return PyFloat_FromDouble(-Py_INFINITY); \ } \ } while(0) diff --git a/Include/internal/pycore_pymath.h b/Include/internal/pycore_pymath.h index 7a4e1c1eb714f7..eea8996ba68ca0 100644 --- a/Include/internal/pycore_pymath.h +++ b/Include/internal/pycore_pymath.h @@ -33,7 +33,7 @@ extern "C" { static inline void _Py_ADJUST_ERANGE1(double x) { if (errno == 0) { - if (x == Py_HUGE_VAL || x == -Py_HUGE_VAL) { + if (x == Py_INFINITY || x == -Py_INFINITY) { errno = ERANGE; } } @@ -44,8 +44,8 @@ static inline void _Py_ADJUST_ERANGE1(double x) static inline void _Py_ADJUST_ERANGE2(double x, double y) { - if (x == Py_HUGE_VAL || x == -Py_HUGE_VAL || - y == Py_HUGE_VAL || y == -Py_HUGE_VAL) + if (x == Py_INFINITY || x == -Py_INFINITY || + y == Py_INFINITY || y == -Py_INFINITY) { if (errno == 0) { errno = ERANGE; diff --git a/Include/pymath.h b/Include/pymath.h index d8f763f808d662..0ead1f95670fde 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -49,7 +49,7 @@ /* Py_HUGE_VAL should always be the same as Py_INFINITY. But historically * this was not reliable and Python did not require IEEE floats and C99 - * conformity. Prefer Py_INFINITY for new code. + * conformity. The macro was soft deprecated in Python 3.14, use Py_INFINITY instead. */ #ifndef Py_HUGE_VAL # define Py_HUGE_VAL HUGE_VAL diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 7b6d10c008d3cb..42b0aea4e2eb2e 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -1480,6 +1480,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None): """ if self._fd >= 0: # Have to close the existing file first. + self._stat_atopen = None try: if self._closefd: os.close(self._fd) @@ -1583,6 +1584,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None): if e.errno != errno.ESPIPE: raise except: + self._stat_atopen = None if owned_fd is not None: os.close(owned_fd) raise @@ -1756,6 +1758,7 @@ def close(self): called more than once without error. """ if not self.closed: + self._stat_atopen = None try: if self._closefd: os.close(self._fd) diff --git a/Lib/getopt.py b/Lib/getopt.py index e5fd04fe12a7ee..1df5b96472a45c 100644 --- a/Lib/getopt.py +++ b/Lib/getopt.py @@ -34,11 +34,8 @@ __all__ = ["GetoptError","error","getopt","gnu_getopt"] import os -try: - from gettext import gettext as _ -except ImportError: - # Bootstrapping Python: gettext's dependencies not built yet - def _(s): return s +from gettext import gettext as _ + class GetoptError(Exception): opt = '' diff --git a/Lib/glob.py b/Lib/glob.py index 574e5ad51b601d..ce9b3698888dd9 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -364,12 +364,6 @@ def concat_path(path, text): """ raise NotImplementedError - @staticmethod - def parse_entry(entry): - """Returns the path of an entry yielded from scandir(). - """ - raise NotImplementedError - # High-level methods def compile(self, pat): @@ -438,6 +432,7 @@ def select_wildcard(path, exists=False): except OSError: pass else: + prefix = self.add_slash(path) for entry in entries: if match is None or match(entry.name): if dir_only: @@ -446,7 +441,7 @@ def select_wildcard(path, exists=False): continue except OSError: continue - entry_path = self.parse_entry(entry) + entry_path = self.concat_path(prefix, entry.name) if dir_only: yield from select_next(entry_path, exists=True) else: @@ -495,6 +490,7 @@ def select_recursive_step(stack, match_pos): except OSError: pass else: + prefix = self.add_slash(path) for entry in entries: is_dir = False try: @@ -504,7 +500,7 @@ def select_recursive_step(stack, match_pos): pass if is_dir or not dir_only: - entry_path = self.parse_entry(entry) + entry_path = self.concat_path(prefix, entry.name) if match is None or match(str(entry_path), match_pos): if dir_only: yield from select_next(entry_path, exists=True) @@ -533,7 +529,6 @@ class _StringGlobber(_GlobberBase): """ lexists = staticmethod(os.path.lexists) scandir = staticmethod(os.scandir) - parse_entry = operator.attrgetter('path') concat_path = operator.add if os.name == 'nt': diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 6b9ed24ad8ec78..d7e8d08b2d92c1 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -266,6 +266,8 @@ class Morsel(dict): "samesite" : "SameSite", } + _reserved_defaults = dict.fromkeys(_reserved, "") + _flags = {'secure', 'httponly'} def __init__(self): @@ -273,8 +275,7 @@ def __init__(self): self._key = self._value = self._coded_value = None # Set default attributes - for key in self._reserved: - dict.__setitem__(self, key, "") + dict.update(self, self._reserved_defaults) @property def key(self): diff --git a/Lib/nturl2path.py b/Lib/nturl2path.py index 61852aff58912d..6453f202c26d14 100644 --- a/Lib/nturl2path.py +++ b/Lib/nturl2path.py @@ -55,16 +55,11 @@ def pathname2url(p): if p[:4] == '\\\\?\\': p = p[4:] if p[:4].upper() == 'UNC\\': - p = '\\' + p[4:] + p = '\\\\' + p[4:] elif p[1:2] != ':': raise OSError('Bad path: ' + p) if not ':' in p: # No drive specifier, just convert slashes and quote the name - if p[:2] == '\\\\': - # path is something like \\host\path\on\remote\host - # convert this to ////host/path/on/remote/host - # (notice doubling of slashes at the start of the path) - p = '\\\\' + p components = p.split('\\') return urllib.parse.quote('/'.join(components)) comp = p.split(':', maxsplit=2) diff --git a/Lib/optparse.py b/Lib/optparse.py index 1c450c6fcbe3b6..04112eca37c801 100644 --- a/Lib/optparse.py +++ b/Lib/optparse.py @@ -75,6 +75,8 @@ import sys, os import textwrap +from gettext import gettext as _, ngettext + def _repr(self): return "<%s at 0x%x: %s>" % (self.__class__.__name__, id(self), self) @@ -86,19 +88,6 @@ def _repr(self): # Id: help.py 527 2006-07-23 15:21:30Z greg # Id: errors.py 509 2006-04-20 00:58:24Z gward -try: - from gettext import gettext, ngettext -except ImportError: - def gettext(message): - return message - - def ngettext(singular, plural, n): - if n == 1: - return singular - return plural - -_ = gettext - class OptParseError (Exception): def __init__(self, msg): diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 11c8018b28f26b..f5eed6f025c250 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -94,25 +94,13 @@ class PathGlobber(_GlobberBase): lexists = operator.methodcaller('exists', follow_symlinks=False) add_slash = operator.methodcaller('joinpath', '') - - @staticmethod - def scandir(path): - """Emulates os.scandir(), which returns an object that can be used as - a context manager. This method is called by walk() and glob(). - """ - import contextlib - return contextlib.nullcontext(path.iterdir()) + scandir = operator.methodcaller('scandir') @staticmethod def concat_path(path, text): """Appends text to the given path.""" return path.with_segments(path._raw_path + text) - @staticmethod - def parse_entry(entry): - """Returns the path of an entry yielded from scandir().""" - return entry - class PurePathBase: """Base class for pure path objects. @@ -639,13 +627,23 @@ def write_text(self, data, encoding=None, errors=None, newline=None): with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: return f.write(data) + def scandir(self): + """Yield os.DirEntry objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + raise UnsupportedOperation(self._unsupported_msg('scandir()')) + def iterdir(self): """Yield path objects of the directory contents. The children are yielded in arbitrary order, and the special entries '.' and '..' are not included. """ - raise UnsupportedOperation(self._unsupported_msg('iterdir()')) + with self.scandir() as entries: + names = [entry.name for entry in entries] + return map(self.joinpath, names) def _glob_selector(self, parts, case_sensitive, recurse_symlinks): if case_sensitive is None: @@ -695,16 +693,18 @@ def walk(self, top_down=True, on_error=None, follow_symlinks=False): if not top_down: paths.append((path, dirnames, filenames)) try: - for child in path.iterdir(): - try: - if child.is_dir(follow_symlinks=follow_symlinks): - if not top_down: - paths.append(child) - dirnames.append(child.name) - else: - filenames.append(child.name) - except OSError: - filenames.append(child.name) + with path.scandir() as entries: + for entry in entries: + name = entry.name + try: + if entry.is_dir(follow_symlinks=follow_symlinks): + if not top_down: + paths.append(path.joinpath(name)) + dirnames.append(name) + else: + filenames.append(name) + except OSError: + filenames.append(name) except OSError as error: if on_error is not None: on_error(error) diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index a78997179820b1..ef072b83d96904 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -615,6 +615,14 @@ def _filter_trailing_slash(self, paths): path_str = path_str[:-1] yield path_str + def scandir(self): + """Yield os.DirEntry objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + return os.scandir(self) + def iterdir(self): """Yield path objects of the directory contents. diff --git a/Lib/pickle.py b/Lib/pickle.py index ed8138beb908ee..965e1952fb8c5e 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -26,7 +26,7 @@ from types import FunctionType from copyreg import dispatch_table from copyreg import _extension_registry, _inverted_registry, _extension_cache -from itertools import islice +from itertools import batched from functools import partial import sys from sys import maxsize @@ -1033,31 +1033,26 @@ def _batch_appends(self, items, obj): write(APPEND) return - it = iter(items) start = 0 - while True: - tmp = list(islice(it, self._BATCHSIZE)) - n = len(tmp) - if n > 1: + for batch in batched(items, self._BATCHSIZE): + batch_len = len(batch) + if batch_len != 1: write(MARK) - for i, x in enumerate(tmp, start): + for i, x in enumerate(batch, start): try: save(x) except BaseException as exc: exc.add_note(f'when serializing {_T(obj)} item {i}') raise write(APPENDS) - elif n: + else: try: - save(tmp[0]) + save(batch[0]) except BaseException as exc: exc.add_note(f'when serializing {_T(obj)} item {start}') raise write(APPEND) - # else tmp is empty, and we're done - if n < self._BATCHSIZE: - return - start += n + start += batch_len def save_dict(self, obj): if self.bin: @@ -1086,13 +1081,10 @@ def _batch_setitems(self, items, obj): write(SETITEM) return - it = iter(items) - while True: - tmp = list(islice(it, self._BATCHSIZE)) - n = len(tmp) - if n > 1: + for batch in batched(items, self._BATCHSIZE): + if len(batch) != 1: write(MARK) - for k, v in tmp: + for k, v in batch: save(k) try: save(v) @@ -1100,8 +1092,8 @@ def _batch_setitems(self, items, obj): exc.add_note(f'when serializing {_T(obj)} item {k!r}') raise write(SETITEMS) - elif n: - k, v = tmp[0] + else: + k, v = batch[0] save(k) try: save(v) @@ -1109,9 +1101,6 @@ def _batch_setitems(self, items, obj): exc.add_note(f'when serializing {_T(obj)} item {k!r}') raise write(SETITEM) - # else tmp is empty, and we're done - if n < self._BATCHSIZE: - return def save_set(self, obj): save = self.save @@ -1124,21 +1113,15 @@ def save_set(self, obj): write(EMPTY_SET) self.memoize(obj) - it = iter(obj) - while True: - batch = list(islice(it, self._BATCHSIZE)) - n = len(batch) - if n > 0: - write(MARK) - try: - for item in batch: - save(item) - except BaseException as exc: - exc.add_note(f'when serializing {_T(obj)} element') - raise - write(ADDITEMS) - if n < self._BATCHSIZE: - return + for batch in batched(obj, self._BATCHSIZE): + write(MARK) + try: + for item in batch: + save(item) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} element') + raise + write(ADDITEMS) dispatch[set] = save_set def save_frozenset(self, obj): diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index dccbec52aa731e..b84d72f2395d45 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -12,7 +12,7 @@ import warnings __all__ = [ - 'get_importer', 'iter_importers', 'get_loader', 'find_loader', + 'get_importer', 'iter_importers', 'walk_packages', 'iter_modules', 'get_data', 'read_code', 'extend_path', 'ModuleInfo', @@ -263,59 +263,6 @@ def iter_importers(fullname=""): yield get_importer(item) -def get_loader(module_or_name): - """Get a "loader" object for module_or_name - - Returns None if the module cannot be found or imported. - If the named module is not already imported, its containing package - (if any) is imported, in order to establish the package __path__. - """ - warnings._deprecated("pkgutil.get_loader", - f"{warnings._DEPRECATED_MSG}; " - "use importlib.util.find_spec() instead", - remove=(3, 14)) - if module_or_name in sys.modules: - module_or_name = sys.modules[module_or_name] - if module_or_name is None: - return None - if isinstance(module_or_name, ModuleType): - module = module_or_name - loader = getattr(module, '__loader__', None) - if loader is not None: - return loader - if getattr(module, '__spec__', None) is None: - return None - fullname = module.__name__ - else: - fullname = module_or_name - return find_loader(fullname) - - -def find_loader(fullname): - """Find a "loader" object for fullname - - This is a backwards compatibility wrapper around - importlib.util.find_spec that converts most failures to ImportError - and only returns the loader rather than the full spec - """ - warnings._deprecated("pkgutil.find_loader", - f"{warnings._DEPRECATED_MSG}; " - "use importlib.util.find_spec() instead", - remove=(3, 14)) - if fullname.startswith('.'): - msg = "Relative module name {!r} not supported".format(fullname) - raise ImportError(msg) - try: - spec = importlib.util.find_spec(fullname) - except (ImportError, AttributeError, TypeError, ValueError) as ex: - # This hack fixes an impedance mismatch between pkgutil and - # importlib, where the latter raises other errors for cases where - # pkgutil previously raised ImportError - msg = "Error while finding loader for {!r} ({}: {})" - raise ImportError(msg.format(fullname, type(ex), ex)) from ex - return spec.loader if spec is not None else None - - def extend_path(path, name): """Extend a package's path. diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 1475b3da2d3293..a0fab46b24e249 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1760,6 +1760,8 @@ def __init__(self, name=None, mode="r", fileobj=None, format=None, # current position in the archive file self.inodes = {} # dictionary caching the inodes of # archive members already added + self._unames = {} # Cached mappings of uid -> uname + self._gnames = {} # Cached mappings of gid -> gname try: if self.mode == "r": @@ -2138,16 +2140,23 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): tarinfo.mtime = statres.st_mtime tarinfo.type = type tarinfo.linkname = linkname + + # Calls to pwd.getpwuid() and grp.getgrgid() tend to be expensive. To + # speed things up, cache the resolved usernames and group names. if pwd: - try: - tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0] - except KeyError: - pass + if tarinfo.uid not in self._unames: + try: + self._unames[tarinfo.uid] = pwd.getpwuid(tarinfo.uid)[0] + except KeyError: + self._unames[tarinfo.uid] = '' + tarinfo.uname = self._unames[tarinfo.uid] if grp: - try: - tarinfo.gname = grp.getgrgid(tarinfo.gid)[0] - except KeyError: - pass + if tarinfo.gid not in self._gnames: + try: + self._gnames[tarinfo.gid] = grp.getgrgid(tarinfo.gid)[0] + except KeyError: + self._gnames[tarinfo.gid] = '' + tarinfo.gname = self._gnames[tarinfo.gid] if type in (CHRTYPE, BLKTYPE): if hasattr(os, "major") and hasattr(os, "minor"): diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 2a071f8485a2b8..e22324efc490be 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -4148,36 +4148,32 @@ PyDoc_STRVAR(test_vararg_and_posonly__doc__, {"test_vararg_and_posonly", _PyCFunction_CAST(test_vararg_and_posonly), METH_FASTCALL, test_vararg_and_posonly__doc__}, static PyObject * -test_vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args); +test_vararg_and_posonly_impl(PyObject *module, PyObject *a, Py_ssize_t nargs, + PyObject *const *args); static PyObject * test_vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; + Py_ssize_t nvararg = nargs - 1; PyObject *a; - PyObject *__clinic_args = NULL; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("test_vararg_and_posonly", nargs, 1, PY_SSIZE_T_MAX)) { goto exit; } a = args[0]; - __clinic_args = PyTuple_New(nargs - 1); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 1; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[1 + i])); - } - return_value = test_vararg_and_posonly_impl(module, a, __clinic_args); + __clinic_args = args + 1; + return_value = test_vararg_and_posonly_impl(module, a, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } static PyObject * -test_vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=79b75dc07decc8d6 input=9cfa748bbff09877]*/ +test_vararg_and_posonly_impl(PyObject *module, PyObject *a, Py_ssize_t nargs, + PyObject *const *args) +/*[clinic end generated code: output=dc2dd9483cc0459e input=9cfa748bbff09877]*/ /*[clinic input] test_vararg @@ -4931,14 +4927,14 @@ PyDoc_STRVAR(Test___init____doc__, "Varargs init method. For example, nargs is translated to PyTuple_GET_SIZE."); static int -Test___init___impl(TestObj *self, PyObject *args); +Test___init___impl(TestObj *self, Py_ssize_t nargs, PyObject *const *args); static int Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) { int return_value = -1; PyTypeObject *base_tp = TestType; - PyObject *__clinic_args = NULL; + PyObject *const *__clinic_args = NULL; if ((Py_IS_TYPE(self, base_tp) || Py_TYPE(self)->tp_new == base_tp->tp_new) && @@ -4948,17 +4944,16 @@ Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) if (!_PyArg_CheckPositional("Test", PyTuple_GET_SIZE(args), 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_GetSlice(0, -1); - return_value = Test___init___impl((TestObj *)self, __clinic_args); + __clinic_args = _PyTuple_CAST(args)->ob_item; + return_value = Test___init___impl((TestObj *)self, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } static int -Test___init___impl(TestObj *self, PyObject *args) -/*[clinic end generated code: output=0ed1009fe0dcf98d input=2a8bd0033c9ac772]*/ +Test___init___impl(TestObj *self, Py_ssize_t nargs, PyObject *const *args) +/*[clinic end generated code: output=6a64b417c9080a73 input=2a8bd0033c9ac772]*/ /*[clinic input] @@ -4976,14 +4971,14 @@ PyDoc_STRVAR(Test__doc__, "Varargs new method. For example, nargs is translated to PyTuple_GET_SIZE."); static PyObject * -Test_impl(PyTypeObject *type, PyObject *args); +Test_impl(PyTypeObject *type, Py_ssize_t nargs, PyObject *const *args); static PyObject * Test(PyTypeObject *type, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; PyTypeObject *base_tp = TestType; - PyObject *__clinic_args = NULL; + PyObject *const *__clinic_args = NULL; if ((type == base_tp || type->tp_init == base_tp->tp_init) && !_PyArg_NoKeywords("Test", kwargs)) { @@ -4992,17 +4987,16 @@ Test(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (!_PyArg_CheckPositional("Test", PyTuple_GET_SIZE(args), 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_GetSlice(0, -1); - return_value = Test_impl(type, __clinic_args); + __clinic_args = _PyTuple_CAST(args)->ob_item; + return_value = Test_impl(type, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } static PyObject * -Test_impl(PyTypeObject *type, PyObject *args) -/*[clinic end generated code: output=8b219f6633e2a2e9 input=70ad829df3dd9b84]*/ +Test_impl(PyTypeObject *type, Py_ssize_t nargs, PyObject *const *args) +/*[clinic end generated code: output=bf22f942407383a5 input=70ad829df3dd9b84]*/ /*[clinic input] diff --git a/Lib/test/support/strace_helper.py b/Lib/test/support/strace_helper.py new file mode 100644 index 00000000000000..90d4b5bccb6fa3 --- /dev/null +++ b/Lib/test/support/strace_helper.py @@ -0,0 +1,170 @@ +import re +import sys +import textwrap +import unittest +from dataclasses import dataclass +from functools import cache +from test import support +from test.support.script_helper import run_python_until_end + +_strace_binary = "/usr/bin/strace" +_syscall_regex = re.compile( + r"(?P[^(]*)\((?P[^)]*)\)\s*[=]\s*(?P.+)") +_returncode_regex = re.compile( + br"\+\+\+ exited with (?P\d+) \+\+\+") + + +@dataclass +class StraceEvent: + syscall: str + args: list[str] + returncode: str + + +@dataclass +class StraceResult: + strace_returncode: int + python_returncode: int + + """The event messages generated by strace. This is very similar to the + stderr strace produces with returncode marker section removed.""" + event_bytes: bytes + stdout: bytes + stderr: bytes + + def events(self): + """Parse event_bytes data into system calls for easier processing. + + This assumes the program under inspection doesn't print any non-utf8 + strings which would mix into the strace output.""" + decoded_events = self.event_bytes.decode('utf-8') + matches = [ + _syscall_regex.match(event) + for event in decoded_events.splitlines() + ] + return [ + StraceEvent(match["syscall"], + [arg.strip() for arg in (match["args"].split(","))], + match["returncode"]) for match in matches if match + ] + + def sections(self): + """Find all "MARK " writes and use them to make groups of events. + + This is useful to avoid variable / overhead events, like those at + interpreter startup or when opening a file so a test can verify just + the small case under study.""" + current_section = "__startup" + sections = {current_section: []} + for event in self.events(): + if event.syscall == 'write' and len( + event.args) > 2 and event.args[1].startswith("\"MARK "): + # Found a new section, don't include the write in the section + # but all events until next mark should be in that section + current_section = event.args[1].split( + " ", 1)[1].removesuffix('\\n"') + if current_section not in sections: + sections[current_section] = list() + else: + sections[current_section].append(event) + + return sections + + +@support.requires_subprocess() +def strace_python(code, strace_flags, check=True): + """Run strace and return the trace. + + Sets strace_returncode and python_returncode to `-1` on error.""" + res = None + + def _make_error(reason, details): + return StraceResult( + strace_returncode=-1, + python_returncode=-1, + event_bytes=f"error({reason},details={details}) = -1".encode('utf-8'), + stdout=res.out if res else b"", + stderr=res.err if res else b"") + + # Run strace, and get out the raw text + try: + res, cmd_line = run_python_until_end( + "-c", + textwrap.dedent(code), + __run_using_command=[_strace_binary] + strace_flags) + except OSError as err: + return _make_error("Caught OSError", err) + + if check and res.rc: + res.fail(cmd_line) + + # Get out program returncode + stripped = res.err.strip() + output = stripped.rsplit(b"\n", 1) + if len(output) != 2: + return _make_error("Expected strace events and exit code line", + stripped[-50:]) + + returncode_match = _returncode_regex.match(output[1]) + if not returncode_match: + return _make_error("Expected to find returncode in last line.", + output[1][:50]) + + python_returncode = int(returncode_match["returncode"]) + if check and python_returncode: + res.fail(cmd_line) + + return StraceResult(strace_returncode=res.rc, + python_returncode=python_returncode, + event_bytes=output[0], + stdout=res.out, + stderr=res.err) + + +def get_events(code, strace_flags, prelude, cleanup): + # NOTE: The flush is currently required to prevent the prints from getting + # buffered and done all at once at exit + prelude = textwrap.dedent(prelude) + code = textwrap.dedent(code) + cleanup = textwrap.dedent(cleanup) + to_run = f""" +print("MARK prelude", flush=True) +{prelude} +print("MARK code", flush=True) +{code} +print("MARK cleanup", flush=True) +{cleanup} +print("MARK __shutdown", flush=True) + """ + trace = strace_python(to_run, strace_flags) + all_sections = trace.sections() + return all_sections['code'] + + +def get_syscalls(code, strace_flags, prelude="", cleanup=""): + """Get the syscalls which a given chunk of python code generates""" + events = get_events(code, strace_flags, prelude=prelude, cleanup=cleanup) + return [ev.syscall for ev in events] + + +# Moderately expensive (spawns a subprocess), so share results when possible. +@cache +def _can_strace(): + res = strace_python("import sys; sys.exit(0)", [], check=False) + assert res.events(), "Should have parsed multiple calls" + + return res.strace_returncode == 0 and res.python_returncode == 0 + + +def requires_strace(): + if sys.platform != "linux": + return unittest.skip("Linux only, requires strace.") + + if support.check_sanitizer(address=True, memory=True): + return unittest.skip("LeakSanitizer does not work under ptrace (strace, gdb, etc)") + + return unittest.skipUnless(_can_strace(), "Requires working strace") + + +__all__ = ["get_events", "get_syscalls", "requires_strace", "strace_python", + "StraceEvent", "StraceResult"] diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index a1013ab803348d..9d2d356631b42c 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -2688,6 +2688,28 @@ def test_get_context(self): finally: loop.close() + def test_proper_refcounts(self): + # see: https://github.com/python/cpython/issues/126083 + class Break: + def __str__(self): + raise RuntimeError("break") + + obj = object() + initial_refcount = sys.getrefcount(obj) + + coro = coroutine_function() + loop = asyncio.new_event_loop() + task = asyncio.Task.__new__(asyncio.Task) + + for _ in range(5): + with self.assertRaisesRegex(RuntimeError, 'break'): + task.__init__(coro, loop=loop, context=obj, name=Break()) + + coro.close() + del task + + self.assertEqual(sys.getrefcount(obj), initial_refcount) + def add_subclass_tests(cls): BaseTask = cls.Task diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index 85491a89947318..a557e35e68915d 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -747,6 +747,49 @@ def test_codec_stream_writer(self): class CAPICodecErrors(unittest.TestCase): + @classmethod + def _generate_exception_args(cls): + for objlen in range(5): + maxind = 2 * max(2, objlen) + for start in range(-maxind, maxind + 1): + for end in range(-maxind, maxind + 1): + yield objlen, start, end + + @classmethod + def generate_encode_errors(cls): + return tuple( + UnicodeEncodeError('utf-8', '0' * objlen, start, end, 'why') + for objlen, start, end in cls._generate_exception_args() + ) + + @classmethod + def generate_decode_errors(cls): + return tuple( + UnicodeDecodeError('utf-8', b'0' * objlen, start, end, 'why') + for objlen, start, end in cls._generate_exception_args() + ) + + @classmethod + def generate_translate_errors(cls): + return tuple( + UnicodeTranslateError('0' * objlen, start, end, 'why') + for objlen, start, end in cls._generate_exception_args() + ) + + @classmethod + def setUpClass(cls): + cls.unicode_encode_errors = cls.generate_encode_errors() + cls.unicode_decode_errors = cls.generate_decode_errors() + cls.unicode_translate_errors = cls.generate_translate_errors() + cls.all_unicode_errors = ( + cls.unicode_encode_errors + + cls.unicode_decode_errors + + cls.unicode_translate_errors + ) + cls.bad_unicode_errors = ( + ValueError(), + ) + def test_codec_register_error(self): # for cleaning up between tests from _codecs import _unregister_error as _codecs_unregister_error @@ -780,33 +823,82 @@ def test_codec_lookup_error(self): self.assertIs(codec_lookup_error('ignore'), codecs.ignore_errors) self.assertIs(codec_lookup_error('replace'), codecs.replace_errors) self.assertIs(codec_lookup_error('xmlcharrefreplace'), codecs.xmlcharrefreplace_errors) + self.assertIs(codec_lookup_error('backslashreplace'), codecs.backslashreplace_errors) self.assertIs(codec_lookup_error('namereplace'), codecs.namereplace_errors) self.assertRaises(LookupError, codec_lookup_error, 'unknown') - def test_codec_error_handlers(self): - exceptions = [ - # A UnicodeError with an empty message currently crashes: - # See: https://github.com/python/cpython/issues/123378 - # UnicodeEncodeError('bad', '', 0, 1, 'reason'), - UnicodeEncodeError('bad', 'x', 0, 1, 'reason'), - UnicodeEncodeError('bad', 'xyz123', 0, 1, 'reason'), - UnicodeEncodeError('bad', 'xyz123', 1, 4, 'reason'), - ] - - strict_handler = _testcapi.codec_strict_errors + def test_codec_strict_errors_handler(self): + handler = _testcapi.codec_strict_errors + for exc in self.all_unicode_errors + self.bad_unicode_errors: + with self.subTest(handler=handler, exc=exc): + self.assertRaises(type(exc), handler, exc) + + def test_codec_ignore_errors_handler(self): + handler = _testcapi.codec_ignore_errors + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) + + def test_codec_replace_errors_handler(self): + handler = _testcapi.codec_replace_errors + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) + + def test_codec_xmlcharrefreplace_errors_handler(self): + handler = _testcapi.codec_xmlcharrefreplace_errors + self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) + + def test_codec_backslashreplace_errors_handler(self): + handler = _testcapi.codec_backslashreplace_errors + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) + + def test_codec_namereplace_errors_handler(self): + handler = _testlimitedcapi.codec_namereplace_errors + self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) + + def do_test_codec_errors_handler(self, handler, exceptions): + at_least_one = False for exc in exceptions: - with self.subTest(handler=strict_handler, exc=exc): - self.assertRaises(UnicodeEncodeError, strict_handler, exc) - - for handler in [ - _testcapi.codec_ignore_errors, - _testcapi.codec_replace_errors, - _testcapi.codec_xmlcharrefreplace_errors, - _testlimitedcapi.codec_namereplace_errors, - ]: - for exc in exceptions: - with self.subTest(handler=handler, exc=exc): - self.assertIsInstance(handler(exc), tuple) + # See https://github.com/python/cpython/issues/123378 and related + # discussion and issues for details. + if self._exception_may_crash(exc): + continue + + at_least_one = True + with self.subTest(handler=handler, exc=exc): + # test that the handler does not crash + self.assertIsInstance(handler(exc), tuple) + + if exceptions: + self.assertTrue(at_least_one, "all exceptions are crashing") + + for bad_exc in ( + self.bad_unicode_errors + + tuple(e for e in self.all_unicode_errors if e not in exceptions) + ): + with self.subTest('bad type', handler=handler, exc=bad_exc): + self.assertRaises(TypeError, handler, bad_exc) + + @classmethod + def _exception_may_crash(cls, exc): + """Indicate whether a Unicode exception might currently crash + the interpreter when used by a built-in codecs error handler. + + Until gh-123378 is fixed, we skip the tests for these exceptions. + + This should only be used by "do_test_codec_errors_handler". + """ + message, start, end = exc.object, exc.start, exc.end + match exc: + case UnicodeEncodeError(): + return end < start or (end - start) >= len(message) + case UnicodeDecodeError(): + # The case "end - start >= len(message)" does not crash. + return end < start + case UnicodeTranslateError(): + # Test "end <= start" because PyCodec_ReplaceErrors checks + # the Unicode kind of a 0-length string which by convention + # is PyUnicode_1BYTE_KIND and not PyUnicode_2BYTE_KIND as + # the handler currently expects. + return end <= start or (end - start) >= len(message) + return False if __name__ == "__main__": diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index b88c4d16ba4ef4..7720cf157fa9ae 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -503,6 +503,56 @@ def __eq__(self, other): return 1 self.assertRaises(TypeError, hash, C2()) + def testPredefinedAttrs(self): + o = object() + + class Custom: + pass + + c = Custom() + + methods = ( + '__class__', '__delattr__', '__dir__', '__eq__', '__format__', + '__ge__', '__getattribute__', '__getstate__', '__gt__', '__hash__', + '__init__', '__init_subclass__', '__le__', '__lt__', '__ne__', + '__new__', '__reduce__', '__reduce_ex__', '__repr__', + '__setattr__', '__sizeof__', '__str__', '__subclasshook__' + ) + for name in methods: + with self.subTest(name): + self.assertTrue(callable(getattr(object, name, None))) + self.assertTrue(callable(getattr(o, name, None))) + self.assertTrue(callable(getattr(Custom, name, None))) + self.assertTrue(callable(getattr(c, name, None))) + + not_defined = [ + '__abs__', '__aenter__', '__aexit__', '__aiter__', '__anext__', + '__await__', '__bool__', '__bytes__', '__ceil__', + '__complex__', '__contains__', '__del__', '__delete__', + '__delitem__', '__divmod__', '__enter__', '__exit__', + '__float__', '__floor__', '__get__', '__getattr__', '__getitem__', + '__index__', '__int__', '__invert__', '__iter__', '__len__', + '__length_hint__', '__missing__', '__neg__', '__next__', + '__objclass__', '__pos__', '__rdivmod__', '__reversed__', + '__round__', '__set__', '__setitem__', '__trunc__' + ] + augment = ( + 'add', 'and', 'floordiv', 'lshift', 'matmul', 'mod', 'mul', 'pow', + 'rshift', 'sub', 'truediv', 'xor' + ) + not_defined.extend(map("__{}__".format, augment)) + not_defined.extend(map("__r{}__".format, augment)) + not_defined.extend(map("__i{}__".format, augment)) + for name in not_defined: + with self.subTest(name): + self.assertFalse(hasattr(object, name)) + self.assertFalse(hasattr(o, name)) + self.assertFalse(hasattr(Custom, name)) + self.assertFalse(hasattr(c, name)) + + # __call__() is defined on the metaclass but not the class + self.assertFalse(hasattr(o, "__call__")) + self.assertFalse(hasattr(c, "__call__")) def testSFBug532646(self): # Test for SF bug 532646 diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 402106194f169f..d492ea1d76aa3a 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -3381,8 +3381,8 @@ def test_keyword_only_parameter(self): def test_varpos(self): # fn(*args) fn = ac_tester.varpos - self.assertEqual(fn(), ()) - self.assertEqual(fn(1, 2), (1, 2)) + self.assertEqual(fn(), ((),)) + self.assertEqual(fn(1, 2), ((1, 2),)) def test_posonly_varpos(self): # fn(a, b, /, *args) diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index b2595eccc82f70..65720871d5c5f0 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -30,6 +30,22 @@ def test_bad_counter_during_dealloc(self): self.assertEqual(cm.unraisable.exc_type, TypeError) + def test_crash_with_not_enough_args(self): + # gh-126220 + import _lsprof + + for profile in [_lsprof.Profiler(), cProfile.Profile()]: + for method in [ + "_pystart_callback", + "_pyreturn_callback", + "_ccall_callback", + "_creturn_callback", + ]: + with self.subTest(profile=profile, method=method): + method_obj = getattr(profile, method) + with self.assertRaises(TypeError): + method_obj() # should not crash + def test_evil_external_timer(self): # gh-120289 # Disabling profiler in external timer should not crash diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py index fa5144a3ca01bb..eb77d6d7782478 100644 --- a/Lib/test/test_ctypes/test_c_simple_type_meta.py +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -85,3 +85,68 @@ class Sub(CtBase): self.assertIsInstance(POINTER(Sub), p_meta) self.assertTrue(issubclass(POINTER(Sub), Sub)) + + def test_creating_pointer_in_dunder_init_1(self): + class ct_meta(type): + def __init__(self, name, bases, namespace): + super().__init__(name, bases, namespace) + + # Avoid recursion. + # (See test_creating_pointer_in_dunder_new_1) + if bases == (c_void_p,): + return + if issubclass(self, PtrBase): + return + if bases == (object,): + ptr_bases = (self, PtrBase) + else: + ptr_bases = (self, POINTER(bases[0])) + p = p_meta(f"POINTER({self.__name__})", ptr_bases, {}) + ctypes._pointer_type_cache[self] = p + + class p_meta(PyCSimpleType, ct_meta): + pass + + class PtrBase(c_void_p, metaclass=p_meta): + pass + + class CtBase(object, metaclass=ct_meta): + pass + + class Sub(CtBase): + pass + + class Sub2(Sub): + pass + + self.assertIsInstance(POINTER(Sub2), p_meta) + self.assertTrue(issubclass(POINTER(Sub2), Sub2)) + self.assertTrue(issubclass(POINTER(Sub2), POINTER(Sub))) + self.assertTrue(issubclass(POINTER(Sub), POINTER(CtBase))) + + def test_creating_pointer_in_dunder_init_2(self): + class ct_meta(type): + def __init__(self, name, bases, namespace): + super().__init__(name, bases, namespace) + + # Avoid recursion. + # (See test_creating_pointer_in_dunder_new_2) + if isinstance(self, p_meta): + return + p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {}) + ctypes._pointer_type_cache[self] = p + + class p_meta(PyCSimpleType, ct_meta): + pass + + class Core(object): + pass + + class CtBase(Core, metaclass=ct_meta): + pass + + class Sub(CtBase): + pass + + self.assertIsInstance(POINTER(Sub), p_meta) + self.assertTrue(issubclass(POINTER(Sub), Sub)) diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index 171412cb7cb08c..b1e165fe16b54f 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -2410,25 +2410,6 @@ def test_DocFileSuite(): >>> suite.run(unittest.TestResult()) - Support for using a package's __loader__.get_data() is also - provided. - - >>> import unittest, pkgutil, test - >>> added_loader = False - >>> if not hasattr(test, '__loader__'): - ... test.__loader__ = pkgutil.get_loader(test) - ... added_loader = True - >>> try: - ... suite = doctest.DocFileSuite('test_doctest.txt', - ... 'test_doctest2.txt', - ... 'test_doctest4.txt', - ... package='test.test_doctest') - ... suite.run(unittest.TestResult()) - ... finally: - ... if added_loader: - ... del test.__loader__ - - '/' should be used as a path separator. It will be converted to a native separator at run time: diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 0695ce29a20bee..bf861ef06ee2d3 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1,5 +1,6 @@ # Run the tests in Programs/_testembed.c (tests for the CPython embedding APIs) from test import support +from test.libregrtest.utils import get_build_info from test.support import import_helper, os_helper, threading_helper, MS_WINDOWS import unittest @@ -1781,8 +1782,10 @@ def test_initconfig_api(self): 'perf_profiling': 2, } config_dev_mode(preconfig, config) + # Temporarily enable ignore_stderr=True to ignore warnings on JIT builds + # See gh-126255 for more information self.check_all_configs("test_initconfig_api", config, preconfig, - api=API_ISOLATED) + api=API_ISOLATED, ignore_stderr=True) def test_initconfig_get_api(self): self.run_embedded_interpreter("test_initconfig_get_api") diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index 0611d1749f41c1..d60aabcdf1ae22 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -11,7 +11,7 @@ from test.support import ( cpython_only, swap_attr, gc_collect, is_emscripten, is_wasi, - infinite_recursion, + infinite_recursion, strace_helper ) from test.support.os_helper import ( TESTFN, TESTFN_ASCII, TESTFN_UNICODE, make_bad_fd, @@ -24,6 +24,9 @@ import _pyio # Python implementation of io +_strace_flags=["--trace=%file,%desc"] + + class AutoFileTests: # file tests for which a test file is automatically set up @@ -359,6 +362,139 @@ def testErrnoOnClosedReadinto(self, f): a = array('b', b'x'*10) f.readinto(a) + @strace_helper.requires_strace() + def test_syscalls_read(self): + """Check that the set of system calls produced by the I/O stack is what + is expected for various read cases. + + It's expected as bits of the I/O implementation change, this will need + to change. The goal is to catch changes that unintentionally add + additional systemcalls (ex. additional calls have been looked at in + bpo-21679 and gh-120754). + """ + self.f.write(b"Hello, World!") + self.f.close() + + + def check_readall(name, code, prelude="", cleanup="", + extra_checks=None): + with self.subTest(name=name): + syscalls = strace_helper.get_events(code, _strace_flags, + prelude=prelude, + cleanup=cleanup) + + # The first call should be an open that returns a + # file descriptor (fd). Afer that calls may vary. Once the file + # is opened, check calls refer to it by fd as the filename + # could be removed from the filesystem, renamed, etc. See: + # Time-of-check time-of-use (TOCTOU) software bug class. + # + # There are a number of related but distinct open system calls + # so not checking precise name here. + self.assertGreater( + len(syscalls), + 1, + f"Should have had at least an open call|calls={syscalls}") + fd_str = syscalls[0].returncode + + # All other calls should contain the fd in their argument set. + for ev in syscalls[1:]: + self.assertIn( + fd_str, + ev.args, + f"Looking for file descriptor in arguments|ev={ev}" + ) + + # There are a number of related syscalls used to implement + # behaviors in a libc (ex. fstat, newfstatat, statx, open, openat). + # Allow any that use the same substring. + def count_similarname(name): + return len([ev for ev in syscalls if name in ev.syscall]) + + checks = [ + # Should open and close the file exactly once + ("open", 1), + ("close", 1), + # There should no longer be an isatty call (All files being + # tested are block devices / not character devices). + ('ioctl', 0), + # Should only have one fstat (bpo-21679, gh-120754) + # note: It's important this uses a fd rather than filename, + # That is validated by the `fd` check above. + # note: fstat, newfstatat, and statx have all been observed + # here in the underlying C library implementations. + ("stat", 1) + ] + + if extra_checks: + checks += extra_checks + + for call, count in checks: + self.assertEqual( + count_similarname(call), + count, + msg=f"call={call}|count={count}|syscalls={syscalls}" + ) + + # "open, read, close" file using different common patterns. + check_readall( + "open builtin with default options", + f""" + f = open('{TESTFN}') + f.read() + f.close() + """ + ) + + check_readall( + "open in binary mode", + f""" + f = open('{TESTFN}', 'rb') + f.read() + f.close() + """ + ) + + check_readall( + "open in text mode", + f""" + f = open('{TESTFN}', 'rt') + f.read() + f.close() + """, + # GH-122111: read_text uses BufferedIO which requires looking up + # position in file. `read_bytes` disables that buffering and avoids + # these calls which is tested the `pathlib read_bytes` case. + extra_checks=[("seek", 1)] + ) + + check_readall( + "pathlib read_bytes", + "p.read_bytes()", + prelude=f"""from pathlib import Path; p = Path("{TESTFN}")""", + # GH-122111: Buffering is disabled so these calls are avoided. + extra_checks=[("seek", 0)] + ) + + check_readall( + "pathlib read_text", + "p.read_text()", + prelude=f"""from pathlib import Path; p = Path("{TESTFN}")""" + ) + + # Focus on just `read()`. + calls = strace_helper.get_syscalls( + prelude=f"f = open('{TESTFN}')", + code="f.read()", + cleanup="f.close()", + strace_flags=_strace_flags + ) + # One to read all the bytes + # One to read the EOF and get a size 0 return. + self.assertEqual(calls.count("read"), 2) + + + class CAutoFileTests(AutoFileTests, unittest.TestCase): FileIO = _io.FileIO modulename = '_io' diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 173e405b785ddc..ff9a52b7adac8a 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1429,6 +1429,39 @@ def test_instruction_size_macro(self): with self.assertRaisesRegex(SyntaxError, "All instructions containing a uop"): self.run_cases_test(input, output) + def test_escaping_call_next_to_cmacro(self): + input = """ + inst(OP, (--)) { + #ifdef Py_GIL_DISABLED + escaping_call(); + #else + another_escaping_call(); + #endif + yet_another_escaping_call(); + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + #else + _PyFrame_SetStackPointer(frame, stack_pointer); + another_escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + #endif + _PyFrame_SetStackPointer(frame, stack_pointer); + yet_another_escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + class TestGeneratedAbstractCases(unittest.TestCase): def setUp(self) -> None: diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 2250b7e76dac01..a4430a868676e2 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -5708,8 +5708,8 @@ def test_faulthandler_module_has_signatures(self): self._test_module_has_signatures(faulthandler, unsupported_signature=unsupported_signature) def test_functools_module_has_signatures(self): - no_signature = {'reduce'} - self._test_module_has_signatures(functools, no_signature) + unsupported_signature = {"reduce"} + self._test_module_has_signatures(functools, unsupported_signature=unsupported_signature) def test_gc_module_has_signatures(self): import gc diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 5e3d7a052bae91..a9befbba64daa0 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -54,6 +54,9 @@ def test_in_main(self): self.assertIsInstance(interp, interpreters.Interpreter) self.assertIn(interp, interpreters.list_all()) + # GH-126221: Passing an invalid Unicode character used to cause a SystemError + self.assertRaises(UnicodeEncodeError, _interpreters.create, '\udc80') + def test_in_thread(self): lock = threading.Lock() interp = None diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 08355a71453807..4ab804850e9c3e 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1,4 +1,5 @@ import collections +import contextlib import io import os import errno @@ -1424,6 +1425,24 @@ def close(self): 'st_mode st_ino st_dev st_nlink st_uid st_gid st_size st_atime st_mtime st_ctime') +class DummyDirEntry: + """ + Minimal os.DirEntry-like object. Returned from DummyPath.scandir(). + """ + __slots__ = ('name', '_is_symlink', '_is_dir') + + def __init__(self, name, is_symlink, is_dir): + self.name = name + self._is_symlink = is_symlink + self._is_dir = is_dir + + def is_symlink(self): + return self._is_symlink + + def is_dir(self, *, follow_symlinks=True): + return self._is_dir and (follow_symlinks or not self._is_symlink) + + class DummyPath(PathBase): """ Simple implementation of PathBase that keeps files and directories in @@ -1491,14 +1510,25 @@ def open(self, mode='r', buffering=-1, encoding=None, stream = io.TextIOWrapper(stream, encoding=encoding, errors=errors, newline=newline) return stream - def iterdir(self): - path = str(self.resolve()) - if path in self._files: - raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) - elif path in self._directories: - return iter([self / name for name in self._directories[path]]) + @contextlib.contextmanager + def scandir(self): + path = self.resolve() + path_str = str(path) + if path_str in self._files: + raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path_str) + elif path_str in self._directories: + yield iter([path.joinpath(name)._dir_entry for name in self._directories[path_str]]) else: - raise FileNotFoundError(errno.ENOENT, "File not found", path) + raise FileNotFoundError(errno.ENOENT, "File not found", path_str) + + @property + def _dir_entry(self): + path_str = str(self) + is_symlink = path_str in self._symlinks + is_directory = (path_str in self._directories + if not is_symlink + else self._symlinks[path_str][1]) + return DummyDirEntry(self.name, is_symlink, is_directory) def mkdir(self, mode=0o777, parents=False, exist_ok=False): path = str(self.parent.resolve() / self.name) @@ -1602,9 +1632,11 @@ def setUp(self): if self.can_symlink: p.joinpath('linkA').symlink_to('fileA') p.joinpath('brokenLink').symlink_to('non-existing') - p.joinpath('linkB').symlink_to('dirB') - p.joinpath('dirA', 'linkC').symlink_to(parser.join('..', 'dirB')) - p.joinpath('dirB', 'linkD').symlink_to(parser.join('..', 'dirB')) + p.joinpath('linkB').symlink_to('dirB', target_is_directory=True) + p.joinpath('dirA', 'linkC').symlink_to( + parser.join('..', 'dirB'), target_is_directory=True) + p.joinpath('dirB', 'linkD').symlink_to( + parser.join('..', 'dirB'), target_is_directory=True) p.joinpath('brokenLinkLoop').symlink_to('brokenLinkLoop') def tearDown(self): @@ -1919,7 +1951,7 @@ def ordered_walk(path): if self.can_symlink: # Add some symlinks source.joinpath('linkC').symlink_to('fileC') - source.joinpath('linkD').symlink_to('dirD') + source.joinpath('linkD').symlink_to('dirD', target_is_directory=True) # Perform the copy target = base / 'copyC' @@ -2187,6 +2219,23 @@ def test_iterdir_nodir(self): self.assertIn(cm.exception.errno, (errno.ENOTDIR, errno.ENOENT, errno.EINVAL)) + def test_scandir(self): + p = self.cls(self.base) + with p.scandir() as entries: + self.assertTrue(list(entries)) + with p.scandir() as entries: + for entry in entries: + child = p / entry.name + self.assertIsNotNone(entry) + self.assertEqual(entry.name, child.name) + self.assertEqual(entry.is_symlink(), + child.is_symlink()) + self.assertEqual(entry.is_dir(follow_symlinks=False), + child.is_dir(follow_symlinks=False)) + if entry.name != 'brokenLinkLoop': + self.assertEqual(entry.is_dir(), child.is_dir()) + + def test_glob_common(self): def _check(glob, expected): self.assertEqual(set(glob), { P(self.base, q) for q in expected }) @@ -2432,7 +2481,7 @@ def test_glob_permissions(self): if i % 2: link.symlink_to(P(self.base, "dirE", "nonexistent")) else: - link.symlink_to(P(self.base, "dirC")) + link.symlink_to(P(self.base, "dirC"), target_is_directory=True) self.assertEqual(len(set(base.glob("*"))), 100) self.assertEqual(len(set(base.glob("*/"))), 50) @@ -2920,7 +2969,7 @@ def setUpWalk(self): f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n") if self.can_symlink: - self.link_path.symlink_to(t2_path) + self.link_path.symlink_to(t2_path, target_is_directory=True) broken_link_path.symlink_to('broken') broken_link2_path.symlink_to(self.cls('tmp3', 'broken')) self.sub2_tree = (self.sub2_path, [], ["broken_link", "broken_link2", "link", "tmp3"]) @@ -3038,7 +3087,7 @@ class DummyPathWithSymlinks(DummyPath): def readlink(self): path = str(self.parent.resolve() / self.name) if path in self._symlinks: - return self.with_segments(self._symlinks[path]) + return self.with_segments(self._symlinks[path][0]) elif path in self._files or path in self._directories: raise OSError(errno.EINVAL, "Not a symlink", path) else: @@ -3050,7 +3099,7 @@ def symlink_to(self, target, target_is_directory=False): if path in self._symlinks: raise FileExistsError(errno.EEXIST, "File exists", path) self._directories[parent].add(self.name) - self._symlinks[path] = str(target) + self._symlinks[path] = str(target), target_is_directory class DummyPathWithSymlinksTest(DummyPathTest): diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index b55d441759eb69..1e74990878007a 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -210,14 +210,14 @@ def test_sys_api_with_existing_trampoline(self): sys.activate_stack_trampoline("perf") sys.activate_stack_trampoline("perf") """ - assert_python_ok("-c", code) + assert_python_ok("-c", code, PYTHON_JIT="0") def test_sys_api_with_invalid_trampoline(self): code = """if 1: import sys sys.activate_stack_trampoline("invalid") """ - rc, out, err = assert_python_failure("-c", code) + rc, out, err = assert_python_failure("-c", code, PYTHON_JIT="0") self.assertIn("invalid backend: invalid", err.decode()) def test_sys_api_get_status(self): @@ -228,7 +228,7 @@ def test_sys_api_get_status(self): sys.deactivate_stack_trampoline() assert sys.is_stack_trampoline_active() is False """ - assert_python_ok("-c", code) + assert_python_ok("-c", code, PYTHON_JIT="0") def is_unwinding_reliable_with_frame_pointers(): diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index ca6927554b053c..736b83711def03 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -607,73 +607,6 @@ class ImportlibMigrationTests(unittest.TestCase): # PEP 302 emulation in this module is in the process of being # deprecated in favour of importlib proper - @unittest.skipIf(__name__ == '__main__', 'not compatible with __main__') - @ignore_warnings(category=DeprecationWarning) - def test_get_loader_handles_missing_loader_attribute(self): - global __loader__ - this_loader = __loader__ - del __loader__ - try: - self.assertIsNotNone(pkgutil.get_loader(__name__)) - finally: - __loader__ = this_loader - - @ignore_warnings(category=DeprecationWarning) - def test_get_loader_handles_missing_spec_attribute(self): - name = 'spam' - mod = type(sys)(name) - del mod.__spec__ - with CleanImport(name): - try: - sys.modules[name] = mod - loader = pkgutil.get_loader(name) - finally: - sys.modules.pop(name, None) - self.assertIsNone(loader) - - @ignore_warnings(category=DeprecationWarning) - def test_get_loader_handles_spec_attribute_none(self): - name = 'spam' - mod = type(sys)(name) - mod.__spec__ = None - with CleanImport(name): - try: - sys.modules[name] = mod - loader = pkgutil.get_loader(name) - finally: - sys.modules.pop(name, None) - self.assertIsNone(loader) - - @ignore_warnings(category=DeprecationWarning) - def test_get_loader_None_in_sys_modules(self): - name = 'totally bogus' - sys.modules[name] = None - try: - loader = pkgutil.get_loader(name) - finally: - del sys.modules[name] - self.assertIsNone(loader) - - def test_get_loader_is_deprecated(self): - with check_warnings( - (r".*\bpkgutil.get_loader\b.*", DeprecationWarning), - ): - res = pkgutil.get_loader("sys") - self.assertIsNotNone(res) - - def test_find_loader_is_deprecated(self): - with check_warnings( - (r".*\bpkgutil.find_loader\b.*", DeprecationWarning), - ): - res = pkgutil.find_loader("sys") - self.assertIsNotNone(res) - - @ignore_warnings(category=DeprecationWarning) - def test_find_loader_missing_module(self): - name = 'totally bogus' - loader = pkgutil.find_loader(name) - self.assertIsNone(loader) - def test_get_importer_avoids_emulation(self): # We use an illegal path so *none* of the path hooks should fire with check_warnings() as w: diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index f065b9c9bb1c2c..e45701dfe033a6 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -4,6 +4,7 @@ from test.support import check_sanitizer from test.support import import_helper from test.support import os_helper +from test.support import strace_helper from test.support import warnings_helper from test.support.script_helper import assert_python_ok import subprocess @@ -3415,7 +3416,7 @@ def __del__(self): @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), "vfork() not enabled by configure.") - @unittest.skipIf(sys.platform != "linux", "Linux only, requires strace.") + @strace_helper.requires_strace() @mock.patch("subprocess._USE_POSIX_SPAWN", new=False) def test_vfork_used_when_expected(self): # This is a performance regression test to ensure we default to using @@ -3423,36 +3424,25 @@ def test_vfork_used_when_expected(self): # Technically this test could pass when posix_spawn is used as well # because libc tends to implement that internally using vfork. But # that'd just be testing a libc+kernel implementation detail. - strace_binary = "/usr/bin/strace" - # The only system calls we are interested in. - strace_filter = "--trace=clone,clone2,clone3,fork,vfork,exit,exit_group" - true_binary = "/bin/true" - strace_command = [strace_binary, strace_filter] - try: - does_strace_work_process = subprocess.run( - strace_command + [true_binary], - stderr=subprocess.PIPE, - stdout=subprocess.DEVNULL, - ) - rc = does_strace_work_process.returncode - stderr = does_strace_work_process.stderr - except OSError: - rc = -1 - stderr = "" - if rc or (b"+++ exited with 0 +++" not in stderr): - self.skipTest("strace not found or not working as expected.") + # Are intersted in the system calls: + # clone,clone2,clone3,fork,vfork,exit,exit_group + # Unfortunately using `--trace` with that list to strace fails because + # not all are supported on all platforms (ex. clone2 is ia64 only...) + # So instead use `%process` which is recommended by strace, and contains + # the above. + true_binary = "/bin/true" + strace_args = ["--trace=%process"] with self.subTest(name="default_is_vfork"): - vfork_result = assert_python_ok( - "-c", - textwrap.dedent(f"""\ - import subprocess - subprocess.check_call([{true_binary!r}])"""), - __run_using_command=strace_command, + vfork_result = strace_helper.strace_python( + f"""\ + import subprocess + subprocess.check_call([{true_binary!r}])""", + strace_args ) # Match both vfork() and clone(..., flags=...|CLONE_VFORK|...) - self.assertRegex(vfork_result.err, br"(?i)vfork") + self.assertRegex(vfork_result.event_bytes, br"(?i)vfork") # Do NOT check that fork() or other clones did not happen. # If the OS denys the vfork it'll fallback to plain fork(). @@ -3465,9 +3455,8 @@ def test_vfork_used_when_expected(self): ("setgroups", "", "extra_groups=[]", True), ): with self.subTest(name=sub_name): - non_vfork_result = assert_python_ok( - "-c", - textwrap.dedent(f"""\ + non_vfork_result = strace_helper.strace_python( + f"""\ import subprocess {preamble} try: @@ -3475,11 +3464,11 @@ def test_vfork_used_when_expected(self): [{true_binary!r}], **dict({sp_kwarg})) except PermissionError: if not {expect_permission_error}: - raise"""), - __run_using_command=strace_command, + raise""", + strace_args ) # Ensure neither vfork() or clone(..., flags=...|CLONE_VFORK|...). - self.assertNotRegex(non_vfork_result.err, br"(?i)vfork") + self.assertNotRegex(non_vfork_result.event_bytes, br"(?i)vfork") @unittest.skipUnless(mswindows, "Windows specific tests") diff --git a/Lib/test/test_tools/i18n_data/docstrings.pot b/Lib/test/test_tools/i18n_data/docstrings.pot new file mode 100644 index 00000000000000..5af1d41422ff62 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/docstrings.pot @@ -0,0 +1,40 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: docstrings.py:7 +#, docstring +msgid "" +msgstr "" + +#: docstrings.py:18 +#, docstring +msgid "" +"multiline\n" +" docstring\n" +" " +msgstr "" + +#: docstrings.py:25 +#, docstring +msgid "docstring1" +msgstr "" + +#: docstrings.py:30 +#, docstring +msgid "Hello, {}!" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/docstrings.py b/Lib/test/test_tools/i18n_data/docstrings.py new file mode 100644 index 00000000000000..85d7f159d37775 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/docstrings.py @@ -0,0 +1,41 @@ +# Test docstring extraction +from gettext import gettext as _ + + +# Empty docstring +def test(x): + """""" + + +# Leading empty line +def test2(x): + + """docstring""" # XXX This should be extracted but isn't. + + +# XXX Multiline docstrings should be cleaned with `inspect.cleandoc`. +def test3(x): + """multiline + docstring + """ + + +# Multiple docstrings - only the first should be extracted +def test4(x): + """docstring1""" + """docstring2""" + + +def test5(x): + """Hello, {}!""".format("world!") # XXX This should not be extracted. + + +# Nested docstrings +def test6(x): + def inner(y): + """nested docstring""" # XXX This should be extracted but isn't. + + +class Outer: + class Inner: + "nested class docstring" # XXX This should be extracted but isn't. diff --git a/Lib/test/test_tools/i18n_data/fileloc.pot b/Lib/test/test_tools/i18n_data/fileloc.pot new file mode 100644 index 00000000000000..dbd28687a73556 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/fileloc.pot @@ -0,0 +1,35 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: fileloc.py:5 fileloc.py:6 +msgid "foo" +msgstr "" + +#: fileloc.py:9 +msgid "bar" +msgstr "" + +#: fileloc.py:14 fileloc.py:18 +#, docstring +msgid "docstring" +msgstr "" + +#: fileloc.py:22 fileloc.py:26 +#, docstring +msgid "baz" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/fileloc.py b/Lib/test/test_tools/i18n_data/fileloc.py new file mode 100644 index 00000000000000..c5d4d0595fea52 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/fileloc.py @@ -0,0 +1,26 @@ +# Test file locations +from gettext import gettext as _ + +# Duplicate strings +_('foo') +_('foo') + +# Duplicate strings on the same line should only add one location to the output +_('bar'), _('bar') + + +# Duplicate docstrings +class A: + """docstring""" + + +def f(): + """docstring""" + + +# Duplicate message and docstring +_('baz') + + +def g(): + """baz""" diff --git a/Lib/test/test_tools/i18n_data/messages.pot b/Lib/test/test_tools/i18n_data/messages.pot new file mode 100644 index 00000000000000..ddfbd18349ef4f --- /dev/null +++ b/Lib/test/test_tools/i18n_data/messages.pot @@ -0,0 +1,67 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: messages.py:5 +msgid "" +msgstr "" + +#: messages.py:8 messages.py:9 +msgid "parentheses" +msgstr "" + +#: messages.py:12 +msgid "Hello, world!" +msgstr "" + +#: messages.py:15 +msgid "" +"Hello,\n" +" multiline!\n" +msgstr "" + +#: messages.py:29 +msgid "Hello, {}!" +msgstr "" + +#: messages.py:33 +msgid "1" +msgstr "" + +#: messages.py:33 +msgid "2" +msgstr "" + +#: messages.py:34 messages.py:35 +msgid "A" +msgstr "" + +#: messages.py:34 messages.py:35 +msgid "B" +msgstr "" + +#: messages.py:36 +msgid "set" +msgstr "" + +#: messages.py:42 +msgid "nested string" +msgstr "" + +#: messages.py:47 +msgid "baz" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/messages.py b/Lib/test/test_tools/i18n_data/messages.py new file mode 100644 index 00000000000000..f220294b8d5c67 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/messages.py @@ -0,0 +1,64 @@ +# Test message extraction +from gettext import gettext as _ + +# Empty string +_("") + +# Extra parentheses +(_("parentheses")) +((_("parentheses"))) + +# Multiline strings +_("Hello, " + "world!") + +_("""Hello, + multiline! +""") + +# Invalid arguments +_() +_(None) +_(1) +_(False) +_(x="kwargs are not allowed") +_("foo", "bar") +_("something", x="something else") + +# .format() +_("Hello, {}!").format("world") # valid +_("Hello, {}!".format("world")) # invalid + +# Nested structures +_("1"), _("2") +arr = [_("A"), _("B")] +obj = {'a': _("A"), 'b': _("B")} +{{{_('set')}}} + + +# Nested functions and classes +def test(): + _("nested string") # XXX This should be extracted but isn't. + [_("nested string")] + + +class Foo: + def bar(self): + return _("baz") + + +def bar(x=_('default value')): # XXX This should be extracted but isn't. + pass + + +def baz(x=[_('default value')]): # XXX This should be extracted but isn't. + pass + + +# Shadowing _() +def _(x): + pass + + +def _(x="don't extract me"): + pass diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index c083a04475e726..21dead8f943bb7 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -1,9 +1,11 @@ """Tests to cover the Tools/i18n package""" import os +import re import sys import unittest from textwrap import dedent +from pathlib import Path from test.support.script_helper import assert_python_ok from test.test_tools import skip_if_missing, toolsdir @@ -12,20 +14,47 @@ skip_if_missing() +DATA_DIR = Path(__file__).resolve().parent / 'i18n_data' + + +def normalize_POT_file(pot): + """Normalize the POT creation timestamp, charset and + file locations to make the POT file easier to compare. + + """ + # Normalize the creation date. + date_pattern = re.compile(r'"POT-Creation-Date: .+?\\n"') + header = r'"POT-Creation-Date: 2000-01-01 00:00+0000\\n"' + pot = re.sub(date_pattern, header, pot) + + # Normalize charset to UTF-8 (currently there's no way to specify the output charset). + charset_pattern = re.compile(r'"Content-Type: text/plain; charset=.+?\\n"') + charset = r'"Content-Type: text/plain; charset=UTF-8\\n"' + pot = re.sub(charset_pattern, charset, pot) + + # Normalize file location path separators in case this test is + # running on Windows (which uses '\'). + fileloc_pattern = re.compile(r'#:.+') + + def replace(match): + return match[0].replace(os.sep, "/") + pot = re.sub(fileloc_pattern, replace, pot) + return pot + class Test_pygettext(unittest.TestCase): """Tests for the pygettext.py tool""" - script = os.path.join(toolsdir,'i18n', 'pygettext.py') + script = Path(toolsdir, 'i18n', 'pygettext.py') def get_header(self, data): """ utility: return the header of a .po file as a dictionary """ headers = {} for line in data.split('\n'): - if not line or line.startswith(('#', 'msgid','msgstr')): + if not line or line.startswith(('#', 'msgid', 'msgstr')): continue line = line.strip('"') - key, val = line.split(':',1) + key, val = line.split(':', 1) headers[key] = val.strip() return headers @@ -53,13 +82,18 @@ def get_msgids(self, data): return msgids + def assert_POT_equal(self, expected, actual): + """Check if two POT files are equal""" + self.maxDiff = None + self.assertEqual(normalize_POT_file(expected), normalize_POT_file(actual)) + def extract_docstrings_from_str(self, module_content): """ utility: return all msgids extracted from module_content """ filename = 'test_docstrings.py' with temp_cwd(None) as cwd: with open(filename, 'w', encoding='utf-8') as fp: fp.write(module_content) - assert_python_ok(self.script, '-D', filename) + assert_python_ok('-Xutf8', self.script, '-D', filename) with open('messages.pot', encoding='utf-8') as fp: data = fp.read() return self.get_msgids(data) @@ -69,7 +103,7 @@ def test_header(self): http://www.gnu.org/software/gettext/manual/gettext.html#Header-Entry """ with temp_cwd(None) as cwd: - assert_python_ok(self.script) + assert_python_ok('-Xutf8', self.script) with open('messages.pot', encoding='utf-8') as fp: data = fp.read() header = self.get_header(data) @@ -96,7 +130,7 @@ def test_POT_Creation_Date(self): """ Match the date format from xgettext for POT-Creation-Date """ from datetime import datetime with temp_cwd(None) as cwd: - assert_python_ok(self.script) + assert_python_ok('-Xutf8', self.script) with open('messages.pot', encoding='utf-8') as fp: data = fp.read() header = self.get_header(data) @@ -310,6 +344,20 @@ def test_calls_in_fstring_with_partially_wrong_expression(self): self.assertNotIn('foo', msgids) self.assertIn('bar', msgids) + def test_pygettext_output(self): + """Test that the pygettext output exactly matches snapshots.""" + for input_file in DATA_DIR.glob('*.py'): + output_file = input_file.with_suffix('.pot') + with self.subTest(input_file=f'i18n_data/{input_file}'): + contents = input_file.read_text(encoding='utf-8') + with temp_cwd(None): + Path(input_file.name).write_text(contents) + assert_python_ok('-Xutf8', self.script, '--docstrings', input_file.name) + output = Path('messages.pot').read_text(encoding='utf-8') + + expected = output_file.read_text(encoding='utf-8') + self.assert_POT_equal(expected, output) + def test_files_list(self): """Make sure the directories are inspected for source files bpo-31920 @@ -318,21 +366,41 @@ def test_files_list(self): text2 = 'Text to translate2' text3 = 'Text to ignore' with temp_cwd(None), temp_dir(None) as sdir: - os.mkdir(os.path.join(sdir, 'pypkg')) - with open(os.path.join(sdir, 'pypkg', 'pymod.py'), 'w', - encoding='utf-8') as sfile: - sfile.write(f'_({text1!r})') - os.mkdir(os.path.join(sdir, 'pkg.py')) - with open(os.path.join(sdir, 'pkg.py', 'pymod2.py'), 'w', - encoding='utf-8') as sfile: - sfile.write(f'_({text2!r})') - os.mkdir(os.path.join(sdir, 'CVS')) - with open(os.path.join(sdir, 'CVS', 'pymod3.py'), 'w', - encoding='utf-8') as sfile: - sfile.write(f'_({text3!r})') - assert_python_ok(self.script, sdir) - with open('messages.pot', encoding='utf-8') as fp: - data = fp.read() + pymod = Path(sdir, 'pypkg', 'pymod.py') + pymod.parent.mkdir() + pymod.write_text(f'_({text1!r})', encoding='utf-8') + + pymod2 = Path(sdir, 'pkg.py', 'pymod2.py') + pymod2.parent.mkdir() + pymod2.write_text(f'_({text2!r})', encoding='utf-8') + + pymod3 = Path(sdir, 'CVS', 'pymod3.py') + pymod3.parent.mkdir() + pymod3.write_text(f'_({text3!r})', encoding='utf-8') + + assert_python_ok('-Xutf8', self.script, sdir) + data = Path('messages.pot').read_text(encoding='utf-8') self.assertIn(f'msgid "{text1}"', data) self.assertIn(f'msgid "{text2}"', data) self.assertNotIn(text3, data) + + +def update_POT_snapshots(): + for input_file in DATA_DIR.glob('*.py'): + output_file = input_file.with_suffix('.pot') + contents = input_file.read_bytes() + with temp_cwd(None): + Path(input_file.name).write_bytes(contents) + assert_python_ok('-Xutf8', Test_pygettext.script, '--docstrings', input_file.name) + output = Path('messages.pot').read_text(encoding='utf-8') + + output = normalize_POT_file(output) + output_file.write_text(output, encoding='utf-8') + + +if __name__ == '__main__': + # To regenerate POT files + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + update_POT_snapshots() + sys.exit(0) + unittest.main() diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index dc852c8f02758c..3ee17f96b817e1 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -1524,7 +1524,7 @@ def test_pathname2url_win(self): # Test special prefixes are correctly handled in pathname2url() fn = urllib.request.pathname2url self.assertEqual(fn('\\\\?\\C:\\dir'), '///C:/dir') - self.assertEqual(fn('\\\\?\\unc\\server\\share\\dir'), '/server/share/dir') + self.assertEqual(fn('\\\\?\\unc\\server\\share\\dir'), '//server/share/dir') self.assertEqual(fn("C:"), '///C:') self.assertEqual(fn("C:\\"), '///C:') self.assertEqual(fn('C:\\a\\b.c'), '///C:/a/b.c') @@ -1535,14 +1535,14 @@ def test_pathname2url_win(self): self.assertRaises(IOError, fn, "XX:\\") # No drive letter self.assertEqual(fn("\\folder\\test\\"), '/folder/test/') - self.assertEqual(fn("\\\\folder\\test\\"), '////folder/test/') - self.assertEqual(fn("\\\\\\folder\\test\\"), '/////folder/test/') - self.assertEqual(fn('\\\\some\\share\\'), '////some/share/') - self.assertEqual(fn('\\\\some\\share\\a\\b.c'), '////some/share/a/b.c') - self.assertEqual(fn('\\\\some\\share\\a\\b%#c\xe9'), '////some/share/a/b%25%23c%C3%A9') + self.assertEqual(fn("\\\\folder\\test\\"), '//folder/test/') + self.assertEqual(fn("\\\\\\folder\\test\\"), '///folder/test/') + self.assertEqual(fn('\\\\some\\share\\'), '//some/share/') + self.assertEqual(fn('\\\\some\\share\\a\\b.c'), '//some/share/a/b.c') + self.assertEqual(fn('\\\\some\\share\\a\\b%#c\xe9'), '//some/share/a/b%25%23c%C3%A9') # Round-tripping urls = ['///C:', - '/////folder/test/', + '///folder/test/', '///C:/foo/bar/spam.foo'] for url in urls: self.assertEqual(fn(urllib.request.url2pathname(url)), url) diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 8b4bb8750f8f5c..4d97fe56f3a094 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -505,20 +505,16 @@ def test_flushes(self): for sync in sync_opt: for level in range(10): - try: + with self.subTest(sync=sync, level=level): obj = zlib.compressobj( level ) a = obj.compress( data[:3000] ) b = obj.flush( sync ) c = obj.compress( data[3000:] ) d = obj.flush() - except: - print("Error for flush mode={}, level={}" - .format(sync, level)) - raise - self.assertEqual(zlib.decompress(b''.join([a,b,c,d])), - data, ("Decompress failed: flush " - "mode=%i, level=%i") % (sync, level)) - del obj + self.assertEqual(zlib.decompress(b''.join([a,b,c,d])), + data, ("Decompress failed: flush " + "mode=%i, level=%i") % (sync, level)) + del obj @unittest.skipUnless(hasattr(zlib, 'Z_SYNC_FLUSH'), 'requires zlib.Z_SYNC_FLUSH') diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index e2aaf8bab4913d..08c83cfb760250 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -309,7 +309,7 @@ def _EndRecData(fpin): fpin.seek(-sizeEndCentDir, 2) except OSError: return None - data = fpin.read() + data = fpin.read(sizeEndCentDir) if (len(data) == sizeEndCentDir and data[0:4] == stringEndArchive and data[-2:] == b"\000\000"): @@ -329,9 +329,9 @@ def _EndRecData(fpin): # record signature. The comment is the last item in the ZIP file and may be # up to 64K long. It is assumed that the "end of central directory" magic # number does not appear in the comment. - maxCommentStart = max(filesize - (1 << 16) - sizeEndCentDir, 0) + maxCommentStart = max(filesize - ZIP_MAX_COMMENT - sizeEndCentDir, 0) fpin.seek(maxCommentStart, 0) - data = fpin.read() + data = fpin.read(ZIP_MAX_COMMENT + sizeEndCentDir) start = data.rfind(stringEndArchive) if start >= 0: # found the magic number; attempt to unpack and interpret diff --git a/Makefile.pre.in b/Makefile.pre.in index 719525e5775d31..c650ecaf7be137 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -905,7 +905,7 @@ coverage-report: regen-token regen-frozen # Run "Argument Clinic" over all source files .PHONY: clinic clinic: check-clean-src - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --exclude Lib/test/clinic.test.c --srcdir $(srcdir) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --force --make --exclude Lib/test/clinic.test.c --srcdir $(srcdir) .PHONY: clinic-tests clinic-tests: check-clean-src $(srcdir)/Lib/test/clinic.test.c @@ -2547,6 +2547,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_tomllib/data/valid/dates-and-times \ test/test_tomllib/data/valid/multiline-basic-str \ test/test_tools \ + test/test_tools/i18n_data \ test/test_ttk \ test/test_unittest \ test/test_unittest/namespace_test_pkg \ diff --git a/Misc/ACKS b/Misc/ACKS index a1769d9601a2ea..5e36eda554af0f 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1164,6 +1164,7 @@ Grzegorz Makarewicz David Malcolm Greg Malcolm William Mallard +Cody Maloney Ken Manheimer Vladimir Marangozov Colin Marc diff --git a/Misc/NEWS.d/3.12.0b1.rst b/Misc/NEWS.d/3.12.0b1.rst index 7126e08a20c7fd..3a3870ac9fe621 100644 --- a/Misc/NEWS.d/3.12.0b1.rst +++ b/Misc/NEWS.d/3.12.0b1.rst @@ -1750,7 +1750,7 @@ Remove the long-deprecated ``imp`` module. .. nonce: N46coo .. section: Library -Deprecate :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` in +Deprecate :func:`!pkgutil.find_loader` and :func:`!pkgutil.get_loader` in favor of :func:`importlib.util.find_spec`. .. diff --git a/Misc/NEWS.d/next/Build/2024-10-31-15-37-05.gh-issue-126206.oC6z2i.rst b/Misc/NEWS.d/next/Build/2024-10-31-15-37-05.gh-issue-126206.oC6z2i.rst new file mode 100644 index 00000000000000..24b172e1747403 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-10-31-15-37-05.gh-issue-126206.oC6z2i.rst @@ -0,0 +1,2 @@ +``make clinic`` now runs Argument Clinic using the ``--force`` option, +thus forcefully regenerating generated code. diff --git a/Misc/NEWS.d/next/C_API/2024-06-04-13-38-44.gh-issue-120026.uhEvJ9.rst b/Misc/NEWS.d/next/C_API/2024-06-04-13-38-44.gh-issue-120026.uhEvJ9.rst new file mode 100644 index 00000000000000..d43a138724ad33 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-06-04-13-38-44.gh-issue-120026.uhEvJ9.rst @@ -0,0 +1 @@ +The :c:macro:`!Py_HUGE_VAL` macro is :term:`soft deprecated`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-14-20-09-39.gh-issue-123714.o1mbe4.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-14-20-09-39.gh-issue-123714.o1mbe4.rst new file mode 100644 index 00000000000000..03a2ef63238073 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-14-20-09-39.gh-issue-123714.o1mbe4.rst @@ -0,0 +1 @@ +Update JIT compilation to use LLVM 19 diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-22-04-18-53.gh-issue-125498.cFjPIn.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-22-04-18-53.gh-issue-125498.cFjPIn.rst new file mode 100644 index 00000000000000..807c2e07210e7f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-22-04-18-53.gh-issue-125498.cFjPIn.rst @@ -0,0 +1,4 @@ +The JIT has been updated to leverage Clang 19’s new ``preserve_none`` attribute, +which supports more platforms and is more useful than LLVM's existing ``ghccc`` +calling convention. This also removes the need to manually patch the calling +convention in LLVM IR, simplifying the JIT compilation process. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-20-31-43.gh-issue-90370.IP_W3a.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-20-31-43.gh-issue-90370.IP_W3a.rst new file mode 100644 index 00000000000000..b6a19c06a228ca --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-20-31-43.gh-issue-90370.IP_W3a.rst @@ -0,0 +1,2 @@ +Avoid temporary tuple creation for vararg in argument passing with Argument +Clinic generated code (if arguments either vararg or positional-only). diff --git a/Misc/NEWS.d/next/Documentation/2023-03-28-22-24-45.gh-issue-60712.So5uad.rst b/Misc/NEWS.d/next/Documentation/2023-03-28-22-24-45.gh-issue-60712.So5uad.rst new file mode 100644 index 00000000000000..e401cc2535e389 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2023-03-28-22-24-45.gh-issue-60712.So5uad.rst @@ -0,0 +1,2 @@ +Include the :class:`object` type in the lists of documented types. +Change by Furkan Onder and Martin Panter. diff --git a/Misc/NEWS.d/next/Library/2024-05-28-14-35-23.gh-issue-97850.dCtjel.rst b/Misc/NEWS.d/next/Library/2024-05-28-14-35-23.gh-issue-97850.dCtjel.rst new file mode 100644 index 00000000000000..bb94f7d8ad124d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-28-14-35-23.gh-issue-97850.dCtjel.rst @@ -0,0 +1 @@ +Remove deprecated :func:`!pkgutil.get_loader` and :func:`!pkgutil.find_loader`. diff --git a/Misc/NEWS.d/next/Library/2024-07-02-15-56-42.gh-issue-121267.yFBWkh.rst b/Misc/NEWS.d/next/Library/2024-07-02-15-56-42.gh-issue-121267.yFBWkh.rst new file mode 100644 index 00000000000000..9e52405c15a82d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-02-15-56-42.gh-issue-121267.yFBWkh.rst @@ -0,0 +1,2 @@ +Improve the performance of :mod:`tarfile` when writing files, by caching user names +and group names. diff --git a/Misc/NEWS.d/next/Library/2024-07-23-02-24-50.gh-issue-120754.nHb5mG.rst b/Misc/NEWS.d/next/Library/2024-07-23-02-24-50.gh-issue-120754.nHb5mG.rst new file mode 100644 index 00000000000000..6c33e7b7ec7716 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-23-02-24-50.gh-issue-120754.nHb5mG.rst @@ -0,0 +1 @@ +Update unbounded ``read`` calls in :mod:`zipfile` to specify an explicit ``size`` putting a limit on how much data they may read. This also updates handling around ZIP max comment size to match the standard instead of reading comments that are one byte too long. diff --git a/Misc/NEWS.d/next/Library/2024-10-28-01-24-52.gh-issue-125413.Jat5kq.rst b/Misc/NEWS.d/next/Library/2024-10-28-01-24-52.gh-issue-125413.Jat5kq.rst new file mode 100644 index 00000000000000..ddf1f9725d9695 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-28-01-24-52.gh-issue-125413.Jat5kq.rst @@ -0,0 +1,3 @@ +Add :meth:`pathlib.Path.scandir` method to efficiently fetch directory +children and their file attributes. This is a trivial wrapper of +:func:`os.scandir`. diff --git a/Misc/NEWS.d/next/Library/2024-10-28-22-35-22.gh-issue-126083.TuI--n.rst b/Misc/NEWS.d/next/Library/2024-10-28-22-35-22.gh-issue-126083.TuI--n.rst new file mode 100644 index 00000000000000..d64b7dd2fedbd6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-28-22-35-22.gh-issue-126083.TuI--n.rst @@ -0,0 +1 @@ +Fixed a reference leak in :class:`asyncio.Task` objects when reinitializing the same object with a non-``None`` context. Patch by Nico Posada. diff --git a/Misc/NEWS.d/next/Library/2024-10-29-10-38-28.gh-issue-126080.qKRBuo.rst b/Misc/NEWS.d/next/Library/2024-10-29-10-38-28.gh-issue-126080.qKRBuo.rst new file mode 100644 index 00000000000000..e54ac17b217c92 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-29-10-38-28.gh-issue-126080.qKRBuo.rst @@ -0,0 +1,3 @@ +Fix a use-after-free crash on :class:`asyncio.Task` objects for which the +underlying event loop implements an evil :meth:`~object.__getattribute__`. +Reported by Nico-Posada. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-10-30-00-12-22.gh-issue-126156.BOSqv0.rst b/Misc/NEWS.d/next/Library/2024-10-30-00-12-22.gh-issue-126156.BOSqv0.rst new file mode 100644 index 00000000000000..4fe18275ab9384 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-30-00-12-22.gh-issue-126156.BOSqv0.rst @@ -0,0 +1 @@ +Improved performances of creating :py:class:`~http.cookies.Morsel` objects by a factor of 3.8x. diff --git a/Misc/NEWS.d/next/Library/2024-10-30-20-45-17.gh-issue-126205.CHEmtx.rst b/Misc/NEWS.d/next/Library/2024-10-30-20-45-17.gh-issue-126205.CHEmtx.rst new file mode 100644 index 00000000000000..c92ffb75056606 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-30-20-45-17.gh-issue-126205.CHEmtx.rst @@ -0,0 +1,2 @@ +Fix issue where :func:`urllib.request.pathname2url` generated URLs beginning +with four slashes (rather than two) when given a Windows UNC path. diff --git a/Misc/NEWS.d/next/Library/2024-10-30-23-42-44.gh-issue-126223.k2qooc.rst b/Misc/NEWS.d/next/Library/2024-10-30-23-42-44.gh-issue-126223.k2qooc.rst new file mode 100644 index 00000000000000..fee391c030b941 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-30-23-42-44.gh-issue-126223.k2qooc.rst @@ -0,0 +1,2 @@ +Raise a :exc:`UnicodeEncodeError` instead of a :exc:`SystemError` upon +calling :func:`!_interpreters.create` with an invalid Unicode character. diff --git a/Misc/NEWS.d/next/Library/2024-10-31-14-06-28.gh-issue-126220.uJAJCU.rst b/Misc/NEWS.d/next/Library/2024-10-31-14-06-28.gh-issue-126220.uJAJCU.rst new file mode 100644 index 00000000000000..555f2f3bafbf33 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-31-14-06-28.gh-issue-126220.uJAJCU.rst @@ -0,0 +1,2 @@ +Fix crash in :class:`!cProfile.Profile` and :class:`!_lsprof.Profiler` when their +callbacks were directly called with 0 arguments. diff --git a/Misc/NEWS.d/next/Library/2024-11-01-14-31-41.gh-issue-126138.yTniOG.rst b/Misc/NEWS.d/next/Library/2024-11-01-14-31-41.gh-issue-126138.yTniOG.rst new file mode 100644 index 00000000000000..459eebc82bd42a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-01-14-31-41.gh-issue-126138.yTniOG.rst @@ -0,0 +1,3 @@ +Fix a use-after-free crash on :class:`asyncio.Task` objects +whose underlying coroutine yields an object that implements +an evil :meth:`~object.__getattribute__`. Patch by Nico Posada. diff --git a/Misc/NEWS.d/next/Library/2024-11-03-09-42-42.gh-issue-126313.EFP6Dl.rst b/Misc/NEWS.d/next/Library/2024-11-03-09-42-42.gh-issue-126313.EFP6Dl.rst new file mode 100644 index 00000000000000..dad348d8898f13 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-03-09-42-42.gh-issue-126313.EFP6Dl.rst @@ -0,0 +1,2 @@ +Fix an issue in :func:`curses.napms` when :func:`curses.initscr` has not yet +been called. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Windows/2024-10-29-19-48-03.gh-issue-125315.jdB9qN.rst b/Misc/NEWS.d/next/Windows/2024-10-29-19-48-03.gh-issue-125315.jdB9qN.rst new file mode 100644 index 00000000000000..3d813248766a5b --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-10-29-19-48-03.gh-issue-125315.jdB9qN.rst @@ -0,0 +1,2 @@ +Avoid crashing in :mod:`platform` due to slow WMI calls on some Windows +machines. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index d4135f04e56575..214c18e966c4c1 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2120,7 +2120,7 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, return -1; } } else { - self->task_context = Py_NewRef(context); + Py_XSETREF(self->task_context, Py_NewRef(context)); } Py_CLEAR(self->task_fut_waiter); @@ -2738,7 +2738,11 @@ task_call_step_soon(asyncio_state *state, TaskObj *task, PyObject *arg) return -1; } - int ret = call_soon(state, task->task_loop, cb, NULL, task->task_context); + // Beware: An evil call_soon could alter task_context. + // See: https://github.com/python/cpython/issues/126080. + PyObject *task_context = Py_NewRef(task->task_context); + int ret = call_soon(state, task->task_loop, cb, NULL, task_context); + Py_DECREF(task_context); Py_DECREF(cb); return ret; } @@ -2963,8 +2967,17 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu if (task->task_must_cancel) { PyObject *r; int is_true; + + // Beware: An evil `__getattribute__` could + // prematurely delete task->task_cancel_msg before the + // task is cancelled, thereby causing a UAF crash. + // + // See https://github.com/python/cpython/issues/126138 + PyObject *task_cancel_msg = Py_NewRef(task->task_cancel_msg); r = PyObject_CallMethodOneArg(result, &_Py_ID(cancel), - task->task_cancel_msg); + task_cancel_msg); + Py_DECREF(task_cancel_msg); + if (r == NULL) { return NULL; } @@ -3056,8 +3069,17 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu if (task->task_must_cancel) { PyObject *r; int is_true; + + // Beware: An evil `__getattribute__` could + // prematurely delete task->task_cancel_msg before the + // task is cancelled, thereby causing a UAF crash. + // + // See https://github.com/python/cpython/issues/126138 + PyObject *task_cancel_msg = Py_NewRef(task->task_cancel_msg); r = PyObject_CallMethodOneArg(result, &_Py_ID(cancel), - task->task_cancel_msg); + task_cancel_msg); + Py_DECREF(task_cancel_msg); + if (r == NULL) { return NULL; } diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 27d5df08de933e..040ffa81153ebe 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -3815,8 +3815,11 @@ static int _curses_napms_impl(PyObject *module, int ms) /*[clinic end generated code: output=5f292a6a724491bd input=c6d6e01f2f1df9f7]*/ { - PyCursesStatefulInitialised(module); - + if (!_PyCursesStatefulCheckFunction(module, + curses_initscr_called, + "initscr")) { + return -1; + } return napms(ms); } diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index da4e088e54621e..d2afe1a1bea018 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -932,15 +932,31 @@ _functools_cmp_to_key_impl(PyObject *module, PyObject *mycmp) /* reduce (used to be a builtin) ********************************************/ -// Not converted to argument clinic, because of `args` in-place modification. -// AC will affect performance. +/*[clinic input] +_functools.reduce + + function as func: object + iterable as seq: object + initial as result: object = NULL + / + +Apply a function of two arguments cumulatively to the items of an iterable, from left to right. + +This effectively reduces the iterable to a single value. If initial is present, +it is placed before the items of the iterable in the calculation, and serves as +a default when the iterable is empty. + +For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) +calculates ((((1 + 2) + 3) + 4) + 5). +[clinic start generated code]*/ + static PyObject * -functools_reduce(PyObject *self, PyObject *args) +_functools_reduce_impl(PyObject *module, PyObject *func, PyObject *seq, + PyObject *result) +/*[clinic end generated code: output=30d898fe1267c79d input=d233c2670cba7f66]*/ { - PyObject *seq, *func, *result = NULL, *it; + PyObject *args, *it; - if (!PyArg_UnpackTuple(args, "reduce", 2, 3, &func, &seq, &result)) - return NULL; if (result != NULL) Py_INCREF(result); @@ -1006,18 +1022,6 @@ functools_reduce(PyObject *self, PyObject *args) return NULL; } -PyDoc_STRVAR(functools_reduce_doc, -"reduce(function, iterable[, initial], /) -> value\n\ -\n\ -Apply a function of two arguments cumulatively to the items of an iterable, from left to right.\n\ -\n\ -This effectively reduces the iterable to a single value. If initial is present,\n\ -it is placed before the items of the iterable in the calculation, and serves as\n\ -a default when the iterable is empty.\n\ -\n\ -For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5])\n\ -calculates ((((1 + 2) + 3) + 4) + 5)."); - /* lru_cache object **********************************************************/ /* There are four principal algorithmic differences from the pure python version: @@ -1722,7 +1726,7 @@ PyDoc_STRVAR(_functools_doc, "Tools that operate on functions."); static PyMethodDef _functools_methods[] = { - {"reduce", functools_reduce, METH_VARARGS, functools_reduce_doc}, + _FUNCTOOLS_REDUCE_METHODDEF _FUNCTOOLS_CMP_TO_KEY_METHODDEF {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 6f3392fe6ea28d..63f2bb38768511 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -402,7 +402,11 @@ config_from_object(PyObject *configobj, PyInterpreterConfig *config) } } else if (PyUnicode_Check(configobj)) { - if (init_named_config(config, PyUnicode_AsUTF8(configobj)) < 0) { + const char *utf8name = PyUnicode_AsUTF8(configobj); + if (utf8name == NULL) { + return -1; + } + if (init_named_config(config, utf8name) < 0) { return -1; } } diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index f374592eb95967..cf0f1d671b507a 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -131,6 +131,8 @@ internal_close(fileio *self) _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS } + PyMem_Free(self->stat_atopen); + self->stat_atopen = NULL; if (err < 0) { errno = save_errno; PyErr_SetFromErrno(PyExc_OSError); @@ -268,8 +270,9 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, if (self->fd >= 0) { if (self->closefd) { /* Have to close the existing file first. */ - if (internal_close(self) < 0) + if (internal_close(self) < 0) { return -1; + } } else self->fd = -1; @@ -523,10 +526,8 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, internal_close(self); _PyErr_ChainExceptions1(exc); } - if (self->stat_atopen != NULL) { - PyMem_Free(self->stat_atopen); - self->stat_atopen = NULL; - } + PyMem_Free(self->stat_atopen); + self->stat_atopen = NULL; done: #ifdef MS_WINDOWS diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 8b6906234bdc25..06958a078509d9 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -608,6 +608,12 @@ setBuiltins(ProfilerObject *pObj, int nvalue) PyObject* pystart_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) { + if (size < 2) { + PyErr_Format(PyExc_TypeError, + "_pystart_callback expected 2 arguments, got %zd", + size); + return NULL; + } PyObject* code = args[0]; ptrace_enter_call((PyObject*)self, (void *)code, (PyObject *)code); @@ -616,6 +622,12 @@ PyObject* pystart_callback(ProfilerObject* self, PyObject *const *args, Py_ssize PyObject* pyreturn_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) { + if (size < 3) { + PyErr_Format(PyExc_TypeError, + "_pyreturn_callback expected 3 arguments, got %zd", + size); + return NULL; + } PyObject* code = args[0]; ptrace_leave_call((PyObject*)self, (void *)code); @@ -651,6 +663,12 @@ PyObject* get_cfunc_from_callable(PyObject* callable, PyObject* self_arg, PyObje PyObject* ccall_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) { + if (size < 4) { + PyErr_Format(PyExc_TypeError, + "_ccall_callback expected 4 arguments, got %zd", + size); + return NULL; + } if (self->flags & POF_BUILTINS) { PyObject* callable = args[2]; PyObject* self_arg = args[3]; @@ -669,6 +687,12 @@ PyObject* ccall_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t PyObject* creturn_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) { + if (size < 4) { + PyErr_Format(PyExc_TypeError, + "_creturn_callback expected 4 arguments, got %zd", + size); + return NULL; + } if (self->flags & POF_BUILTINS) { PyObject* callable = args[2]; PyObject* self_arg = args[3]; diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index ca884af1aa29b8..e3c8ba9b0b5074 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -58,6 +58,20 @@ pack_arguments_newref(int argc, ...) return tuple; } +static PyObject * +pack_varargs_to_tuple(Py_ssize_t varargssize, PyObject *const *args) +{ + assert(!PyErr_Occurred()); + PyObject *tuple = PyTuple_New(varargssize); + if (!tuple) { + return NULL; + } + for (Py_ssize_t i = 0; i < varargssize; i++) { + PyTuple_SET_ITEM(tuple, i, Py_NewRef(args[i])); + } + return tuple; +} + /* Pack arguments to a tuple. * `wrapper` is function which converts primitive type to PyObject. * `arg_type` is type that arguments should be converted to before wrapped. */ @@ -970,10 +984,16 @@ varpos [clinic start generated code]*/ static PyObject * -varpos_impl(PyObject *module, PyObject *args) -/*[clinic end generated code: output=7b0b9545872bdca4 input=f87cd674145d394c]*/ +varpos_impl(PyObject *module, Py_ssize_t nargs, PyObject *const *args) +/*[clinic end generated code: output=b65096f423fb5dcc input=f87cd674145d394c]*/ { - return Py_NewRef(args); + PyObject *vararg_tuple = pack_varargs_to_tuple(nargs, args); + if (!vararg_tuple) { + return NULL; + } + PyObject *result = pack_arguments_newref(1, vararg_tuple); + Py_DECREF(vararg_tuple); + return result; } @@ -989,10 +1009,16 @@ posonly_varpos static PyObject * posonly_varpos_impl(PyObject *module, PyObject *a, PyObject *b, - PyObject *args) -/*[clinic end generated code: output=5dae5eb2a0d623cd input=c9fd7895cfbaabba]*/ + Py_ssize_t nargs, PyObject *const *args) +/*[clinic end generated code: output=d10d43d86d117ab3 input=c9fd7895cfbaabba]*/ { - return pack_arguments_newref(3, a, b, args); + PyObject *vararg_tuple = pack_varargs_to_tuple(nargs, args); + if (!vararg_tuple) { + return NULL; + } + PyObject *result = pack_arguments_newref(3, a, b, vararg_tuple); + Py_DECREF(vararg_tuple); + return result; } @@ -1157,8 +1183,9 @@ Proof-of-concept of GH-99233 refcount error bug. [clinic start generated code]*/ static PyObject * -gh_99233_refcount_impl(PyObject *module, PyObject *args) -/*[clinic end generated code: output=585855abfbca9a7f input=eecfdc2092d90dc3]*/ +gh_99233_refcount_impl(PyObject *module, Py_ssize_t nargs, + PyObject *const *args) +/*[clinic end generated code: output=b570007e61e5c670 input=eecfdc2092d90dc3]*/ { Py_RETURN_NONE; } diff --git a/Modules/clinic/_functoolsmodule.c.h b/Modules/clinic/_functoolsmodule.c.h index e98984dc4d3a09..0564921034be47 100644 --- a/Modules/clinic/_functoolsmodule.c.h +++ b/Modules/clinic/_functoolsmodule.c.h @@ -67,6 +67,50 @@ _functools_cmp_to_key(PyObject *module, PyObject *const *args, Py_ssize_t nargs, return return_value; } +PyDoc_STRVAR(_functools_reduce__doc__, +"reduce($module, function, iterable, initial=, /)\n" +"--\n" +"\n" +"Apply a function of two arguments cumulatively to the items of an iterable, from left to right.\n" +"\n" +"This effectively reduces the iterable to a single value. If initial is present,\n" +"it is placed before the items of the iterable in the calculation, and serves as\n" +"a default when the iterable is empty.\n" +"\n" +"For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5])\n" +"calculates ((((1 + 2) + 3) + 4) + 5)."); + +#define _FUNCTOOLS_REDUCE_METHODDEF \ + {"reduce", _PyCFunction_CAST(_functools_reduce), METH_FASTCALL, _functools_reduce__doc__}, + +static PyObject * +_functools_reduce_impl(PyObject *module, PyObject *func, PyObject *seq, + PyObject *result); + +static PyObject * +_functools_reduce(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *func; + PyObject *seq; + PyObject *result = NULL; + + if (!_PyArg_CheckPositional("reduce", nargs, 2, 3)) { + goto exit; + } + func = args[0]; + seq = args[1]; + if (nargs < 3) { + goto skip_optional; + } + result = args[2]; +skip_optional: + return_value = _functools_reduce_impl(module, func, seq, result); + +exit: + return return_value; +} + PyDoc_STRVAR(_functools__lru_cache_wrapper_cache_info__doc__, "cache_info($self, /)\n" "--\n" @@ -114,4 +158,4 @@ _functools__lru_cache_wrapper_cache_clear(PyObject *self, PyObject *Py_UNUSED(ig return return_value; } -/*[clinic end generated code: output=755265bb6d5ea751 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=214d6c6307cfcd91 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 1988c06971087d..7e29998c7db520 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -2530,28 +2530,22 @@ PyDoc_STRVAR(varpos__doc__, {"varpos", _PyCFunction_CAST(varpos), METH_FASTCALL, varpos__doc__}, static PyObject * -varpos_impl(PyObject *module, PyObject *args); +varpos_impl(PyObject *module, Py_ssize_t nargs, PyObject *const *args); static PyObject * varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("varpos", nargs, 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = varpos_impl(module, __clinic_args); + __clinic_args = args + 0; + return_value = varpos_impl(module, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -2565,32 +2559,26 @@ PyDoc_STRVAR(posonly_varpos__doc__, static PyObject * posonly_varpos_impl(PyObject *module, PyObject *a, PyObject *b, - PyObject *args); + Py_ssize_t nargs, PyObject *const *args); static PyObject * posonly_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; + Py_ssize_t nvararg = nargs - 2; PyObject *a; PyObject *b; - PyObject *__clinic_args = NULL; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("posonly_varpos", nargs, 2, PY_SSIZE_T_MAX)) { goto exit; } a = args[0]; b = args[1]; - __clinic_args = PyTuple_New(nargs - 2); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 2; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[2 + i])); - } - return_value = posonly_varpos_impl(module, a, b, __clinic_args); + __clinic_args = args + 2; + return_value = posonly_varpos_impl(module, a, b, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -3136,28 +3124,23 @@ PyDoc_STRVAR(gh_99233_refcount__doc__, {"gh_99233_refcount", _PyCFunction_CAST(gh_99233_refcount), METH_FASTCALL, gh_99233_refcount__doc__}, static PyObject * -gh_99233_refcount_impl(PyObject *module, PyObject *args); +gh_99233_refcount_impl(PyObject *module, Py_ssize_t nargs, + PyObject *const *args); static PyObject * gh_99233_refcount(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("gh_99233_refcount", nargs, 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = gh_99233_refcount_impl(module, __clinic_args); + __clinic_args = args + 0; + return_value = gh_99233_refcount_impl(module, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -3693,4 +3676,4 @@ _testclinic_TestClass_defclass_posonly_varpos(PyObject *self, PyTypeObject *cls, Py_XDECREF(__clinic_args); return return_value; } -/*[clinic end generated code: output=76ecbb38c632bde8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7662d07e7d29cbeb input=a9049054013a1b77]*/ diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index 9fff4da616ba00..be3bd35b4ffd44 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -312,28 +312,23 @@ PyDoc_STRVAR(gc_get_referrers__doc__, {"get_referrers", _PyCFunction_CAST(gc_get_referrers), METH_FASTCALL, gc_get_referrers__doc__}, static PyObject * -gc_get_referrers_impl(PyObject *module, PyObject *args); +gc_get_referrers_impl(PyObject *module, Py_ssize_t nargs, + PyObject *const *args); static PyObject * gc_get_referrers(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("get_referrers", nargs, 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = gc_get_referrers_impl(module, __clinic_args); + __clinic_args = args + 0; + return_value = gc_get_referrers_impl(module, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -347,28 +342,23 @@ PyDoc_STRVAR(gc_get_referents__doc__, {"get_referents", _PyCFunction_CAST(gc_get_referents), METH_FASTCALL, gc_get_referents__doc__}, static PyObject * -gc_get_referents_impl(PyObject *module, PyObject *args); +gc_get_referents_impl(PyObject *module, Py_ssize_t nargs, + PyObject *const *args); static PyObject * gc_get_referents(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("get_referents", nargs, 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = gc_get_referents_impl(module, __clinic_args); + __clinic_args = args + 0; + return_value = gc_get_referents_impl(module, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -585,4 +575,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=0a7e91917adcb937 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f488a0d4d6bd3687 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index 81eec310ddb21d..7d0b98d5502267 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -8,6 +8,64 @@ preserve #endif #include "pycore_modsupport.h" // _PyArg_CheckPositional() +PyDoc_STRVAR(math_gcd__doc__, +"gcd($module, /, *integers)\n" +"--\n" +"\n" +"Greatest Common Divisor."); + +#define MATH_GCD_METHODDEF \ + {"gcd", _PyCFunction_CAST(math_gcd), METH_FASTCALL, math_gcd__doc__}, + +static PyObject * +math_gcd_impl(PyObject *module, Py_ssize_t nargs, PyObject *const *args); + +static PyObject * +math_gcd(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("gcd", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = args + 0; + return_value = math_gcd_impl(module, nvararg, __clinic_args); + +exit: + return return_value; +} + +PyDoc_STRVAR(math_lcm__doc__, +"lcm($module, /, *integers)\n" +"--\n" +"\n" +"Least Common Multiple."); + +#define MATH_LCM_METHODDEF \ + {"lcm", _PyCFunction_CAST(math_lcm), METH_FASTCALL, math_lcm__doc__}, + +static PyObject * +math_lcm_impl(PyObject *module, Py_ssize_t nargs, PyObject *const *args); + +static PyObject * +math_lcm(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("lcm", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = args + 0; + return_value = math_lcm_impl(module, nvararg, __clinic_args); + +exit: + return return_value; +} + PyDoc_STRVAR(math_ceil__doc__, "ceil($module, x, /)\n" "--\n" @@ -351,6 +409,46 @@ math_dist(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(math_hypot__doc__, +"hypot($module, /, *coordinates)\n" +"--\n" +"\n" +"Multidimensional Euclidean distance from the origin to a point.\n" +"\n" +"Roughly equivalent to:\n" +" sqrt(sum(x**2 for x in coordinates))\n" +"\n" +"For a two dimensional point (x, y), gives the hypotenuse\n" +"using the Pythagorean theorem: sqrt(x*x + y*y).\n" +"\n" +"For example, the hypotenuse of a 3/4/5 right triangle is:\n" +"\n" +" >>> hypot(3.0, 4.0)\n" +" 5.0"); + +#define MATH_HYPOT_METHODDEF \ + {"hypot", _PyCFunction_CAST(math_hypot), METH_FASTCALL, math_hypot__doc__}, + +static PyObject * +math_hypot_impl(PyObject *module, Py_ssize_t nargs, PyObject *const *args); + +static PyObject * +math_hypot(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("hypot", nargs, 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = args + 0; + return_value = math_hypot_impl(module, nvararg, __clinic_args); + +exit: + return return_value; +} + PyDoc_STRVAR(math_sumprod__doc__, "sumprod($module, p, q, /)\n" "--\n" @@ -1011,4 +1109,4 @@ math_ulp(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=755da3b1dbd9e45f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ee0a2f6bd1220061 input=a9049054013a1b77]*/ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index e07c2dbd262354..81cbf0d554de3c 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -150,7 +150,7 @@ special_type(double d) #define P14 0.25*Py_MATH_PI #define P12 0.5*Py_MATH_PI #define P34 0.75*Py_MATH_PI -#define INF Py_HUGE_VAL +#define INF Py_INFINITY #define N Py_NAN #define U -9.5426319407711027e33 /* unlikely value, used as placeholder */ diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 57e4aae9ed557e..f5fea5aa4dde08 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -8,6 +8,7 @@ #include "pycore_gc.h" #include "pycore_object.h" // _PyObject_IS_GC() #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_tuple.h" // _PyTuple_FromArray() typedef struct _gc_runtime_state GCState; @@ -221,15 +222,25 @@ Return the list of objects that directly refer to any of 'objs'. [clinic start generated code]*/ static PyObject * -gc_get_referrers_impl(PyObject *module, PyObject *args) -/*[clinic end generated code: output=296a09587f6a86b5 input=bae96961b14a0922]*/ +gc_get_referrers_impl(PyObject *module, Py_ssize_t nargs, + PyObject *const *args) +/*[clinic end generated code: output=1d44a7695ea25c40 input=bae96961b14a0922]*/ { - if (PySys_Audit("gc.get_referrers", "(O)", args) < 0) { + PyObject *varargs = _PyTuple_FromArray(args, nargs); + + if (!varargs) { + return NULL; + } + if (PySys_Audit("gc.get_referrers", "(O)", varargs) < 0) { + Py_DECREF(varargs); return NULL; } PyInterpreterState *interp = _PyInterpreterState_GET(); - return _PyGC_GetReferrers(interp, args); + PyObject *result = _PyGC_GetReferrers(interp, varargs); + + Py_DECREF(varargs); + return result; } /* Append obj to list; return true if error (out of memory), false if OK. */ @@ -269,27 +280,37 @@ Return the list of objects that are directly referred to by 'objs'. [clinic start generated code]*/ static PyObject * -gc_get_referents_impl(PyObject *module, PyObject *args) -/*[clinic end generated code: output=d47dc02cefd06fe8 input=b3ceab0c34038cbf]*/ +gc_get_referents_impl(PyObject *module, Py_ssize_t nargs, + PyObject *const *args) +/*[clinic end generated code: output=e459f3e8c0d19311 input=b3ceab0c34038cbf]*/ { - if (PySys_Audit("gc.get_referents", "(O)", args) < 0) { + PyObject *varargs = _PyTuple_FromArray(args, nargs); + + if (!varargs) { + return NULL; + } + if (PySys_Audit("gc.get_referents", "(O)", varargs) < 0) { + Py_DECREF(varargs); return NULL; } PyInterpreterState *interp = _PyInterpreterState_GET(); PyObject *result = PyList_New(0); - if (result == NULL) + if (result == NULL) { + Py_DECREF(varargs); return NULL; + } // NOTE: stop the world is a no-op in default build _PyEval_StopTheWorld(interp); - int err = append_referrents(result, args); + int err = append_referrents(result, varargs); _PyEval_StartTheWorld(interp); if (err < 0) { Py_CLEAR(result); } + Py_DECREF(varargs); return result; } diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 058f57770755aa..7e8d8b3f5bafa2 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -438,7 +438,7 @@ m_tgamma(double x) } else { errno = ERANGE; - return Py_HUGE_VAL; + return Py_INFINITY; } } @@ -502,14 +502,14 @@ m_lgamma(double x) if (isnan(x)) return x; /* lgamma(nan) = nan */ else - return Py_HUGE_VAL; /* lgamma(+-inf) = +inf */ + return Py_INFINITY; /* lgamma(+-inf) = +inf */ } /* integer arguments */ if (x == floor(x) && x <= 2.0) { if (x <= 0.0) { errno = EDOM; /* lgamma(n) = inf, divide-by-zero for */ - return Py_HUGE_VAL; /* integers n <= 0 */ + return Py_INFINITY; /* integers n <= 0 */ } else { return 0.0; /* lgamma(1) = lgamma(2) = 0.0 */ @@ -645,7 +645,7 @@ m_log(double x) return log(x); errno = EDOM; if (x == 0.0) - return -Py_HUGE_VAL; /* log(0) = -inf */ + return -Py_INFINITY; /* log(0) = -inf */ else return Py_NAN; /* log(-ve) = nan */ } @@ -688,7 +688,7 @@ m_log2(double x) } else if (x == 0.0) { errno = EDOM; - return -Py_HUGE_VAL; /* log2(0) = -inf, divide-by-zero */ + return -Py_INFINITY; /* log2(0) = -inf, divide-by-zero */ } else { errno = EDOM; @@ -704,7 +704,7 @@ m_log10(double x) return log10(x); errno = EDOM; if (x == 0.0) - return -Py_HUGE_VAL; /* log10(0) = -inf */ + return -Py_INFINITY; /* log10(0) = -inf */ else return Py_NAN; /* log10(-ve) = nan */ } @@ -719,8 +719,17 @@ m_log10(double x) } +/*[clinic input] +math.gcd + + *integers as args: object + +Greatest Common Divisor. +[clinic start generated code]*/ + static PyObject * -math_gcd(PyObject *module, PyObject * const *args, Py_ssize_t nargs) +math_gcd_impl(PyObject *module, Py_ssize_t nargs, PyObject *const *args) +/*[clinic end generated code: output=b57687fcf431c1b8 input=94e675b7ceeaf0c9]*/ { // Fast-path for the common case: gcd(int, int) if (nargs == 2 && PyLong_CheckExact(args[0]) && PyLong_CheckExact(args[1])) @@ -763,12 +772,6 @@ math_gcd(PyObject *module, PyObject * const *args, Py_ssize_t nargs) return res; } -PyDoc_STRVAR(math_gcd_doc, -"gcd($module, *integers)\n" -"--\n" -"\n" -"Greatest Common Divisor."); - static PyObject * long_lcm(PyObject *a, PyObject *b) @@ -798,8 +801,17 @@ long_lcm(PyObject *a, PyObject *b) } +/*[clinic input] +math.lcm + + *integers as args: object + +Least Common Multiple. +[clinic start generated code]*/ + static PyObject * -math_lcm(PyObject *module, PyObject * const *args, Py_ssize_t nargs) +math_lcm_impl(PyObject *module, Py_ssize_t nargs, PyObject *const *args) +/*[clinic end generated code: output=f3eff0c25e4d7030 input=e64c33e85f4c47c6]*/ { PyObject *res, *x; Py_ssize_t i; @@ -839,13 +851,6 @@ math_lcm(PyObject *module, PyObject * const *args, Py_ssize_t nargs) } -PyDoc_STRVAR(math_lcm_doc, -"lcm($module, *integers)\n" -"--\n" -"\n" -"Least Common Multiple."); - - /* Call is_error when errno != 0, and where x is the result libm * returned. is_error will usually set up an exception and return * true (1), but may return false (0) without setting up an exception. @@ -2121,7 +2126,7 @@ math_ldexp_impl(PyObject *module, double x, PyObject *i) errno = 0; } else if (exp > INT_MAX) { /* overflow */ - r = copysign(Py_HUGE_VAL, x); + r = copysign(Py_INFINITY, x); errno = ERANGE; } else if (exp < INT_MIN) { /* underflow to +-0 */ @@ -2621,9 +2626,28 @@ math_dist_impl(PyObject *module, PyObject *p, PyObject *q) return NULL; } -/* AC: cannot convert yet, waiting for *args support */ +/*[clinic input] +math.hypot + + *coordinates as args: object + +Multidimensional Euclidean distance from the origin to a point. + +Roughly equivalent to: + sqrt(sum(x**2 for x in coordinates)) + +For a two dimensional point (x, y), gives the hypotenuse +using the Pythagorean theorem: sqrt(x*x + y*y). + +For example, the hypotenuse of a 3/4/5 right triangle is: + + >>> hypot(3.0, 4.0) + 5.0 +[clinic start generated code]*/ + static PyObject * -math_hypot(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +math_hypot_impl(PyObject *module, Py_ssize_t nargs, PyObject *const *args) +/*[clinic end generated code: output=dcb6d4b7a1102ee1 input=5c0061a2d11235ed]*/ { Py_ssize_t i; PyObject *item; @@ -2664,22 +2688,6 @@ math_hypot(PyObject *self, PyObject *const *args, Py_ssize_t nargs) #undef NUM_STACK_ELEMS -PyDoc_STRVAR(math_hypot_doc, - "hypot(*coordinates) -> value\n\n\ -Multidimensional Euclidean distance from the origin to a point.\n\ -\n\ -Roughly equivalent to:\n\ - sqrt(sum(x**2 for x in coordinates))\n\ -\n\ -For a two dimensional point (x, y), gives the hypotenuse\n\ -using the Pythagorean theorem: sqrt(x*x + y*y).\n\ -\n\ -For example, the hypotenuse of a 3/4/5 right triangle is:\n\ -\n\ - >>> hypot(3.0, 4.0)\n\ - 5.0\n\ -"); - /** sumprod() ***************************************************************/ /* Forward declaration */ @@ -4112,14 +4120,14 @@ static PyMethodDef math_methods[] = { MATH_FREXP_METHODDEF MATH_FSUM_METHODDEF {"gamma", math_gamma, METH_O, math_gamma_doc}, - {"gcd", _PyCFunction_CAST(math_gcd), METH_FASTCALL, math_gcd_doc}, - {"hypot", _PyCFunction_CAST(math_hypot), METH_FASTCALL, math_hypot_doc}, + MATH_GCD_METHODDEF + MATH_HYPOT_METHODDEF MATH_ISCLOSE_METHODDEF MATH_ISFINITE_METHODDEF MATH_ISINF_METHODDEF MATH_ISNAN_METHODDEF MATH_ISQRT_METHODDEF - {"lcm", _PyCFunction_CAST(math_lcm), METH_FASTCALL, math_lcm_doc}, + MATH_LCM_METHODDEF MATH_LDEXP_METHODDEF {"lgamma", math_lgamma, METH_O, math_lgamma_doc}, {"log", _PyCFunction_CAST(math_log), METH_FASTCALL, math_log_doc}, diff --git a/Objects/clinic/setobject.c.h b/Objects/clinic/setobject.c.h index 3853ce3bce685b..d6e381a9975050 100644 --- a/Objects/clinic/setobject.c.h +++ b/Objects/clinic/setobject.c.h @@ -41,28 +41,22 @@ PyDoc_STRVAR(set_update__doc__, {"update", _PyCFunction_CAST(set_update), METH_FASTCALL, set_update__doc__}, static PyObject * -set_update_impl(PySetObject *so, PyObject *args); +set_update_impl(PySetObject *so, Py_ssize_t nargs, PyObject *const *args); static PyObject * set_update(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("update", nargs, 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = set_update_impl(so, __clinic_args); + __clinic_args = args + 0; + return_value = set_update_impl(so, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -148,28 +142,22 @@ PyDoc_STRVAR(set_union__doc__, {"union", _PyCFunction_CAST(set_union), METH_FASTCALL, set_union__doc__}, static PyObject * -set_union_impl(PySetObject *so, PyObject *args); +set_union_impl(PySetObject *so, Py_ssize_t nargs, PyObject *const *args); static PyObject * set_union(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("union", nargs, 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = set_union_impl(so, __clinic_args); + __clinic_args = args + 0; + return_value = set_union_impl(so, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -183,28 +171,23 @@ PyDoc_STRVAR(set_intersection_multi__doc__, {"intersection", _PyCFunction_CAST(set_intersection_multi), METH_FASTCALL, set_intersection_multi__doc__}, static PyObject * -set_intersection_multi_impl(PySetObject *so, PyObject *args); +set_intersection_multi_impl(PySetObject *so, Py_ssize_t nargs, + PyObject *const *args); static PyObject * set_intersection_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("intersection", nargs, 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = set_intersection_multi_impl(so, __clinic_args); + __clinic_args = args + 0; + return_value = set_intersection_multi_impl(so, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -218,28 +201,23 @@ PyDoc_STRVAR(set_intersection_update_multi__doc__, {"intersection_update", _PyCFunction_CAST(set_intersection_update_multi), METH_FASTCALL, set_intersection_update_multi__doc__}, static PyObject * -set_intersection_update_multi_impl(PySetObject *so, PyObject *args); +set_intersection_update_multi_impl(PySetObject *so, Py_ssize_t nargs, + PyObject *const *args); static PyObject * set_intersection_update_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("intersection_update", nargs, 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = set_intersection_update_multi_impl(so, __clinic_args); + __clinic_args = args + 0; + return_value = set_intersection_update_multi_impl(so, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -277,28 +255,23 @@ PyDoc_STRVAR(set_difference_update__doc__, {"difference_update", _PyCFunction_CAST(set_difference_update), METH_FASTCALL, set_difference_update__doc__}, static PyObject * -set_difference_update_impl(PySetObject *so, PyObject *args); +set_difference_update_impl(PySetObject *so, Py_ssize_t nargs, + PyObject *const *args); static PyObject * set_difference_update(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("difference_update", nargs, 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = set_difference_update_impl(so, __clinic_args); + __clinic_args = args + 0; + return_value = set_difference_update_impl(so, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -312,28 +285,23 @@ PyDoc_STRVAR(set_difference_multi__doc__, {"difference", _PyCFunction_CAST(set_difference_multi), METH_FASTCALL, set_difference_multi__doc__}, static PyObject * -set_difference_multi_impl(PySetObject *so, PyObject *args); +set_difference_multi_impl(PySetObject *so, Py_ssize_t nargs, + PyObject *const *args); static PyObject * set_difference_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + Py_ssize_t nvararg = nargs - 0; + PyObject *const *__clinic_args = NULL; if (!_PyArg_CheckPositional("difference", nargs, 0, PY_SSIZE_T_MAX)) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = set_difference_multi_impl(so, __clinic_args); + __clinic_args = args + 0; + return_value = set_difference_multi_impl(so, nvararg, __clinic_args); exit: - Py_XDECREF(__clinic_args); return return_value; } @@ -568,4 +536,4 @@ set___sizeof__(PySetObject *so, PyObject *Py_UNUSED(ignored)) return return_value; } -/*[clinic end generated code: output=de4ee725bd29f758 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9d4b41191b2c602f input=a9049054013a1b77]*/ diff --git a/Objects/floatobject.c b/Objects/floatobject.c index d66863febe8c86..7e14a8ad959590 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -2390,7 +2390,7 @@ PyFloat_Unpack2(const char *data, int le) if (e == 0x1f) { if (f == 0) { /* Infinity */ - return sign ? -Py_HUGE_VAL : Py_HUGE_VAL; + return sign ? -Py_INFINITY : Py_INFINITY; } else { /* NaN */ diff --git a/Objects/setobject.c b/Objects/setobject.c index 9f40e085f06fa6..66d7fc730c555c 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1060,13 +1060,13 @@ Update the set, adding elements from all others. [clinic start generated code]*/ static PyObject * -set_update_impl(PySetObject *so, PyObject *args) -/*[clinic end generated code: output=34f6371704974c8a input=df4fe486e38cd337]*/ +set_update_impl(PySetObject *so, Py_ssize_t nargs, PyObject *const *args) +/*[clinic end generated code: output=050e2a21f8d7d16a input=df4fe486e38cd337]*/ { Py_ssize_t i; - for (i=0 ; iquery); + if (!bstrQuery) { + hr = HRESULT_FROM_WIN32(ERROR_NOT_ENOUGH_MEMORY); + } + + if (SUCCEEDED(hr)) { + hr = CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED); + } + if (FAILED(hr)) { CloseHandle(data->writePipe); + if (bstrQuery) { + SysFreeString(bstrQuery); + } return (DWORD)hr; } @@ -101,12 +115,6 @@ _query_thread(LPVOID param) NULL, EOAC_NONE ); } - if (SUCCEEDED(hr)) { - bstrQuery = SysAllocString(data->query); - if (!bstrQuery) { - hr = HRESULT_FROM_WIN32(ERROR_NOT_ENOUGH_MEMORY); - } - } if (SUCCEEDED(hr)) { hr = services->ExecQuery( bstr_t("WQL"), diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 24e573ac1cc89b..a930cd0b0b10c6 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -920,15 +920,6 @@ Modules - - Modules - - - Modules - - - Modules - Modules diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index cb21777f566189..5ac1dd7813689c 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1128,6 +1128,9 @@ expr_ty _PyPegen_collect_call_seqs(Parser *p, asdl_expr_seq *a, asdl_seq *b, } asdl_expr_seq *args = _Py_asdl_expr_seq_new(total_len, arena); + if (args == NULL) { + return NULL; + } Py_ssize_t i = 0; for (i = 0; i < args_len; i++) { @@ -1298,6 +1301,9 @@ unpack_top_level_joined_strs(Parser *p, asdl_expr_seq *raw_expressions) } asdl_expr_seq *expressions = _Py_asdl_expr_seq_new(req_size, p->arena); + if (expressions == NULL) { + return NULL; + } Py_ssize_t raw_index, req_index = 0; for (raw_index = 0; raw_index < raw_size; raw_index++) { @@ -1490,6 +1496,9 @@ expr_ty _PyPegen_formatted_value(Parser *p, expr_ty expression, Token *debug, Re } asdl_expr_seq *values = _Py_asdl_expr_seq_new(2, arena); + if (values == NULL) { + return NULL; + } asdl_seq_SET(values, 0, debug_text); asdl_seq_SET(values, 1, formatted_value); return _PyAST_JoinedStr(values, lineno, col_offset, debug_end_line, debug_end_offset, p->arena); diff --git a/Python/codegen.c b/Python/codegen.c index d79aee4859e51b..d6ba85887e3860 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -4087,9 +4087,12 @@ codegen_call_helper(compiler *c, location loc, return codegen_call_helper_impl(c, loc, n, args, NULL, keywords); } -/* List and set comprehensions and generator expressions work by creating a - nested function to perform the actual iteration. This means that the - iteration variables don't leak into the current scope. +/* List and set comprehensions work by being inlined at the location where + they are defined. The isolation of iteration variables is provided by + pushing/popping clashing locals on the stack. Generator expressions work + by creating a nested function to perform the actual iteration. + This means that the iteration variables don't leak into the current scope. + See https://peps.python.org/pep-0709/ for additional information. The defined function is called immediately following its definition, with the result of that call being the result of the expression. The LC/SC version returns the populated container, while the GE version is diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 2efaa9db7d7d58..23882d083844ac 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1310,12 +1310,17 @@ init_interp_main(PyThreadState *tstate) enabled = *env != '0'; } if (enabled) { +#ifdef _Py_JIT + // perf profiler works fine with tier 2 interpreter, so + // only checking for a "real JIT". if (config->perf_profiling > 0) { (void)PyErr_WarnEx( PyExc_RuntimeWarning, "JIT deactivated as perf profiling support is active", 0); - } else { + } else +#endif + { PyObject *opt = _PyOptimizer_NewUOpOptimizer(); if (opt == NULL) { return _PyStatus_ERR("can't initialize optimizer"); diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 2f2b588bd147d8..7b74f613ed563b 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -43,7 +43,7 @@ _Py_parse_inf_or_nan(const char *p, char **endptr) s += 3; if (case_insensitive_match(s, "inity")) s += 5; - retval = negate ? -Py_HUGE_VAL : Py_HUGE_VAL; + retval = negate ? -Py_INFINITY : Py_INFINITY; } else if (case_insensitive_match(s, "nan")) { s += 3; @@ -286,7 +286,7 @@ _PyOS_ascii_strtod(const char *nptr, char **endptr) string, -1.0 is returned and again ValueError is raised. On overflow (e.g., when trying to convert '1e500' on an IEEE 754 machine), - if overflow_exception is NULL then +-Py_HUGE_VAL is returned, and no Python + if overflow_exception is NULL then +-Py_INFINITY is returned, and no Python exception is raised. Otherwise, overflow_exception should point to a Python exception, this exception will be raised, -1.0 will be returned, and *endptr will point just past the end of the converted value. diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 9235a294e44c8b..a086bb979efa9c 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2295,6 +2295,7 @@ sys_activate_stack_trampoline_impl(PyObject *module, const char *backend) #ifdef _Py_JIT _PyOptimizerObject* optimizer = _Py_GetOptimizer(); if (optimizer != NULL) { + Py_DECREF(optimizer); PyErr_SetString(PyExc_ValueError, "Cannot activate the perf trampoline if the JIT is active"); return NULL; } diff --git a/Tools/c-analyzer/TODO b/Tools/c-analyzer/TODO index 3d599538510bd9..e81ceb29c64bf0 100644 --- a/Tools/c-analyzer/TODO +++ b/Tools/c-analyzer/TODO @@ -562,8 +562,6 @@ Objects/unicodeobject.c:static_strings static _Py_Iden # PyTypeObject (311) Modules/_abc.c:_abc_data_type static PyTypeObject _abc_data_type -Modules/_blake2/blake2b_impl.c:PyBlake2_BLAKE2bType PyTypeObject PyBlake2_BLAKE2bType -Modules/_blake2/blake2s_impl.c:PyBlake2_BLAKE2sType PyTypeObject PyBlake2_BLAKE2sType Modules/_collectionsmodule.c:defdict_type static PyTypeObject defdict_type Modules/_collectionsmodule.c:deque_type static PyTypeObject deque_type Modules/_collectionsmodule.c:dequeiter_type static PyTypeObject dequeiter_type diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 2605825d3d0078..686f3935d91bda 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -629,8 +629,6 @@ Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput ELEMENTTREE_PARSEWHOLE_INITI Include/internal/pycore_importdl.h - _PyImport_DynLoadFiletab - Include/py_curses.h - PyCurses_API - Include/pydecimal.h - _decimal_api - -Modules/_blake2/blake2module.c - blake2b_type_spec - -Modules/_blake2/blake2module.c - blake2s_type_spec - Modules/_io/fileio.c - _Py_open_cloexec_works - Modules/_io/_iomodule.h - PyIOBase_Type - Modules/_io/_iomodule.h - PyRawIOBase_Type - diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 66ead741b87a2b..a725ec10d4e52a 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -637,7 +637,7 @@ def find_stmt_start(node: parser.InstDef, idx: int) -> lexer.Token: assert idx < len(node.block.tokens) while True: tkn = node.block.tokens[idx-1] - if tkn.kind in {"SEMI", "LBRACE", "RBRACE"}: + if tkn.kind in {"SEMI", "LBRACE", "RBRACE", "CMACRO"}: break idx -= 1 assert idx > 0 diff --git a/Tools/clinic/libclinic/clanguage.py b/Tools/clinic/libclinic/clanguage.py index 73d47833d97294..32aba81ab8a850 100644 --- a/Tools/clinic/libclinic/clanguage.py +++ b/Tools/clinic/libclinic/clanguage.py @@ -15,7 +15,7 @@ Module, Class, Function, Parameter, permute_optional_groups, GETTER, SETTER, METHOD_INIT) -from libclinic.converters import self_converter +from libclinic.converters import defining_class_converter, self_converter from libclinic.parse_args import ParseArgsCodeGen if TYPE_CHECKING: from libclinic.app import Clinic @@ -396,6 +396,12 @@ def render_function( first_optional = len(selfless) positional = selfless and selfless[-1].is_positional_only() has_option_groups = False + requires_defining_class = (len(selfless) + and isinstance(selfless[0].converter, + defining_class_converter)) + pass_vararg_directly = (all(p.is_positional_only() or p.is_vararg() + for p in selfless) + and not requires_defining_class) # offset i by -1 because first_optional needs to ignore self for i, p in enumerate(parameters, -1): @@ -404,7 +410,7 @@ def render_function( if (i != -1) and (p.default is not unspecified): first_optional = min(first_optional, i) - if p.is_vararg(): + if p.is_vararg() and not pass_vararg_directly: data.cleanup.append(f"Py_XDECREF({c.parser_name});") # insert group variable @@ -418,6 +424,11 @@ def render_function( data.impl_parameters.append("int " + group_name) has_option_groups = True + if p.is_vararg() and pass_vararg_directly: + data.impl_arguments.append('nvararg') + data.impl_parameters.append('Py_ssize_t nargs') + p.converter.type = 'PyObject *const *' + c.render(p, data) if has_option_groups and (not positional): diff --git a/Tools/clinic/libclinic/parse_args.py b/Tools/clinic/libclinic/parse_args.py index 96c9b919bff811..559d4fbdd09c57 100644 --- a/Tools/clinic/libclinic/parse_args.py +++ b/Tools/clinic/libclinic/parse_args.py @@ -469,7 +469,11 @@ def parse_pos_only(self) -> None: nargs = 'PyTuple_GET_SIZE(args)' argname_fmt = 'PyTuple_GET_ITEM(args, %d)' - left_args = f"{nargs} - {self.max_pos}" + if self.vararg != NO_VARARG: + self.declarations = f"Py_ssize_t nvararg = {nargs} - {self.max_pos};" + else: + self.declarations = "" + max_args = NO_VARARG if (self.vararg != NO_VARARG) else self.max_pos if self.limited_capi: parser_code = [] @@ -518,30 +522,13 @@ def parse_pos_only(self) -> None: use_parser_code = True for i, p in enumerate(self.parameters): if p.is_vararg(): + var = p.converter.parser_name if self.fastcall: - parser_code.append(libclinic.normalize_snippet(""" - %s = PyTuple_New(%s); - if (!%s) {{ - goto exit; - }} - for (Py_ssize_t i = 0; i < %s; ++i) {{ - PyTuple_SET_ITEM(%s, i, Py_NewRef(args[%d + i])); - }} - """ % ( - p.converter.parser_name, - left_args, - p.converter.parser_name, - left_args, - p.converter.parser_name, - self.max_pos - ), indent=4)) + code = f"{var} = args + {self.vararg};" else: - parser_code.append(libclinic.normalize_snippet(""" - %s = PyTuple_GetSlice(%d, -1); - """ % ( - p.converter.parser_name, - self.max_pos - ), indent=4)) + code = f"{var} = _PyTuple_CAST(args)->ob_item;" + formatted_code = libclinic.normalize_snippet(code, indent=4) + parser_code.append(formatted_code) continue displayname = p.get_displayname(i+1) @@ -588,7 +575,7 @@ def parse_pos_only(self) -> None: goto exit; }} """, indent=4)] - self.parser_body(*parser_code) + self.parser_body(*parser_code, declarations=self.declarations) def parse_general(self, clang: CLanguage) -> None: parsearg: str | None diff --git a/Tools/jit/README.md b/Tools/jit/README.md index bc6f793b296f12..801c64e4059ccc 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -7,49 +7,46 @@ This version of CPython can be built with an experimental just-in-time compiler[ The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 18 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-18`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 19 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. It's easy to install all of the required tools: ### Linux -Install LLVM 18 on Ubuntu/Debian: +Install LLVM 19 on Ubuntu/Debian: ```sh wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh -sudo ./llvm.sh 18 +sudo ./llvm.sh 19 ``` -Install LLVM 18 on Fedora Linux 40 or newer: +Install LLVM 19 on Fedora Linux 40 or newer: ```sh -sudo dnf install 'clang(major) = 18' 'llvm(major) = 18' +sudo dnf install 'clang(major) = 19' 'llvm(major) = 19' ``` ### macOS -Install LLVM 18 with [Homebrew](https://brew.sh): +Install LLVM 19 with [Homebrew](https://brew.sh): ```sh -brew install llvm@18 +brew install llvm@19 ``` Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. ### Windows -Install LLVM 18 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=18), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** +Install LLVM 19 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=19), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** Alternatively, you can use [chocolatey](https://chocolatey.org): ```sh -choco install llvm --version=18.1.6 +choco install llvm --version=19.1.0 ``` -### Dev Containers - -If you are working CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no need to install LLVM as the Fedora 40 base image includes LLVM 18 out of the box. ## Building diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 606f280a14d974..925b56ac669aa0 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -8,7 +8,7 @@ import subprocess import typing -_LLVM_VERSION = 18 +_LLVM_VERSION = 19 _LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+") _P = typing.ParamSpec("_P") diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index e4b2bf6e4702b3..2cd051b0a77b8d 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -2,6 +2,7 @@ import dataclasses import enum +import sys import typing import _schema @@ -132,8 +133,18 @@ class Hole: def __post_init__(self) -> None: self.func = _PATCH_FUNCS[self.kind] - def fold(self, other: typing.Self) -> typing.Self | None: + def fold(self, other: typing.Self, body: bytes) -> typing.Self | None: """Combine two holes into a single hole, if possible.""" + instruction_a = int.from_bytes( + body[self.offset : self.offset + 4], byteorder=sys.byteorder + ) + instruction_b = int.from_bytes( + body[other.offset : other.offset + 4], byteorder=sys.byteorder + ) + reg_a = instruction_a & 0b11111 + reg_b1 = instruction_b & 0b11111 + reg_b2 = (instruction_b >> 5) & 0b11111 + if ( self.offset + 4 == other.offset and self.value == other.value @@ -141,6 +152,7 @@ def fold(self, other: typing.Self) -> typing.Self | None: and self.addend == other.addend and self.func == "patch_aarch64_21rx" and other.func == "patch_aarch64_12x" + and reg_a == reg_b1 == reg_b2 ): # These can *only* be properly relaxed when they appear together and # patch the same value: diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 5eb316e782fda8..634208da3c8157 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -26,7 +26,6 @@ PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h" TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c" - _S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection) _R = typing.TypeVar( "_R", _schema.COFFRelocation, _schema.ELFRelocation, _schema.MachORelocation @@ -39,7 +38,6 @@ class _Target(typing.Generic[_S, _R]): _: dataclasses.KW_ONLY alignment: int = 1 args: typing.Sequence[str] = () - ghccc: bool = False prefix: str = "" stable: bool = False debug: bool = False @@ -88,11 +86,7 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: sections: list[dict[typing.Literal["Section"], _S]] = json.loads(output) for wrapped_section in sections: self._handle_section(wrapped_section["Section"], group) - # The trampoline's entry point is just named "_ENTRY", since on some - # platforms we later assume that any function starting with "_JIT_" uses - # the GHC calling convention: - entry_symbol = "_JIT_ENTRY" if "_JIT_ENTRY" in group.symbols else "_ENTRY" - assert group.symbols[entry_symbol] == (_stencils.HoleValue.CODE, 0) + assert group.symbols["_JIT_ENTRY"] == (_stencils.HoleValue.CODE, 0) if group.data.body: line = f"0: {str(bytes(group.data.body)).removeprefix('b')}" group.data.disassembly.append(line) @@ -112,9 +106,6 @@ def _handle_relocation( async def _compile( self, opname: str, c: pathlib.Path, tempdir: pathlib.Path ) -> _stencils.StencilGroup: - # "Compile" the trampoline to an empty stencil group if it's not needed: - if opname == "trampoline" and not self.ghccc: - return _stencils.StencilGroup() o = tempdir / f"{opname}.o" args = [ f"--target={self.triple}", @@ -128,6 +119,7 @@ async def _compile( f"-I{CPYTHON / 'Include' / 'internal'}", f"-I{CPYTHON / 'Include' / 'internal' / 'mimalloc'}", f"-I{CPYTHON / 'Python'}", + f"-I{CPYTHON / 'Tools' / 'jit'}", "-O3", "-c", # This debug info isn't necessary, and bloats out the JIT'ed code. @@ -143,44 +135,12 @@ async def _compile( # Don't call stack-smashing canaries that we can't find or patch: "-fno-stack-protector", "-std=c11", + "-o", + f"{o}", + f"{c}", *self.args, ] - if self.ghccc: - # This is a bit of an ugly workaround, but it makes the code much - # smaller and faster, so it's worth it. We want to use the GHC - # calling convention, but Clang doesn't support it. So, we *first* - # compile the code to LLVM IR, perform some text replacements on the - # IR to change the calling convention(!), and then compile *that*. - # Once we have access to Clang 19, we can get rid of this and use - # __attribute__((preserve_none)) directly in the C code instead: - ll = tempdir / f"{opname}.ll" - args_ll = args + [ - # -fomit-frame-pointer is necessary because the GHC calling - # convention uses RBP to pass arguments: - "-S", - "-emit-llvm", - "-fomit-frame-pointer", - "-o", - f"{ll}", - f"{c}", - ] - await _llvm.run("clang", args_ll, echo=self.verbose) - ir = ll.read_text() - # This handles declarations, definitions, and calls to named symbols - # starting with "_JIT_": - ir = re.sub( - r"(((noalias|nonnull|noundef) )*ptr @_JIT_\w+\()", r"ghccc \1", ir - ) - # This handles calls to anonymous callees, since anything with - # "musttail" needs to use the same calling convention: - ir = ir.replace("musttail call", "musttail call ghccc") - # Sometimes *both* replacements happen at the same site, so fix it: - ir = ir.replace("ghccc ghccc", "ghccc") - ll.write_text(ir) - args_o = args + ["-Wno-unused-command-line-argument", "-o", f"{o}", f"{ll}"] - else: - args_o = args + ["-o", f"{o}", f"{c}"] - await _llvm.run("clang", args_o, echo=self.verbose) + await _llvm.run("clang", args, echo=self.verbose) return await self._parse(o) async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: @@ -519,7 +479,6 @@ def _handle_relocation( def get_target(host: str) -> _COFF | _ELF | _MachO: """Build a _Target for the given host "triple" and options.""" - # ghccc currently crashes Clang when combined with musttail on aarch64. :( target: _COFF | _ELF | _MachO if re.fullmatch(r"aarch64-apple-darwin.*", host): target = _MachO(host, alignment=8, prefix="_") @@ -535,16 +494,20 @@ def get_target(host: str) -> _COFF | _ELF | _MachO: ] target = _ELF(host, alignment=8, args=args) elif re.fullmatch(r"i686-pc-windows-msvc", host): - args = ["-DPy_NO_ENABLE_SHARED"] - target = _COFF(host, args=args, ghccc=True, prefix="_") + args = [ + "-DPy_NO_ENABLE_SHARED", + # __attribute__((preserve_none)) is not supported + "-Wno-ignored-attributes", + ] + target = _COFF(host, args=args, prefix="_") elif re.fullmatch(r"x86_64-apple-darwin.*", host): - target = _MachO(host, ghccc=True, prefix="_") + target = _MachO(host, prefix="_") elif re.fullmatch(r"x86_64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll"] - target = _COFF(host, args=args, ghccc=True) + target = _COFF(host, args=args) elif re.fullmatch(r"x86_64-.*-linux-gnu", host): args = ["-fpic"] - target = _ELF(host, args=args, ghccc=True) + target = _ELF(host, args=args) else: raise ValueError(host) return target diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py index 4e7f614b0e9d23..f33d8ef322f073 100644 --- a/Tools/jit/_writer.py +++ b/Tools/jit/_writer.py @@ -65,7 +65,7 @@ def _dump_stencil(opname: str, group: _stencils.StencilGroup) -> typing.Iterator if skip: skip = False continue - if pair and (folded := hole.fold(pair)): + if pair and (folded := hole.fold(pair, stencil.body)): skip = True hole = folded yield f" {hole.as_c(part)}" diff --git a/Tools/jit/jit.h b/Tools/jit/jit.h new file mode 100644 index 00000000000000..47da64cb12bd24 --- /dev/null +++ b/Tools/jit/jit.h @@ -0,0 +1,4 @@ +// To use preserve_none in JIT builds, we need to declare a separate function +// pointer with __attribute__((preserve_none)), since this attribute may not be +// supported by the compiler used to build the rest of the interpreter. +typedef jit_func __attribute__((preserve_none)) jit_func_preserve_none; diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 6cf15085f79933..57c1006ab423e9 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -21,6 +21,8 @@ #include "ceval_macros.h" +#include "jit.h" + #undef CURRENT_OPARG #define CURRENT_OPARG() (_oparg) @@ -49,7 +51,7 @@ do { \ OPT_STAT_INC(traces_executed); \ __attribute__((musttail)) \ - return ((jit_func)((EXECUTOR)->jit_side_entry))(frame, stack_pointer, tstate); \ + return ((jit_func_preserve_none)((EXECUTOR)->jit_side_entry))(frame, stack_pointer, tstate); \ } while (0) #undef GOTO_TIER_ONE @@ -72,7 +74,7 @@ do { \ do { \ PyAPI_DATA(void) ALIAS; \ __attribute__((musttail)) \ - return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); \ + return ((jit_func_preserve_none)&ALIAS)(frame, stack_pointer, tstate); \ } while (0) #undef JUMP_TO_JUMP_TARGET @@ -86,7 +88,7 @@ do { \ #define TIER_TWO 2 -_Py_CODEUNIT * +__attribute__((preserve_none)) _Py_CODEUNIT * _JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) { // Locals that the instruction implementations expect to exist: diff --git a/Tools/jit/trampoline.c b/Tools/jit/trampoline.c index a0a963f2a49656..f0cffa2f049d26 100644 --- a/Tools/jit/trampoline.c +++ b/Tools/jit/trampoline.c @@ -4,11 +4,10 @@ #include "pycore_frame.h" #include "pycore_jit.h" -// This is where the calling convention changes, on platforms that require it. -// The actual change is patched in while the JIT compiler is being built, in -// Tools/jit/_targets.py. On other platforms, this function compiles to nothing. +#include "jit.h" + _Py_CODEUNIT * -_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) +_JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) { // This is subtle. The actual trace will return to us once it exits, so we // need to make sure that we stay alive until then. If our trace side-exits @@ -19,7 +18,7 @@ _ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *ts Py_INCREF(executor); // Note that this is *not* a tail call: PyAPI_DATA(void) _JIT_CONTINUE; - _Py_CODEUNIT *target = ((jit_func)&_JIT_CONTINUE)(frame, stack_pointer, tstate); + _Py_CODEUNIT *target = ((jit_func_preserve_none)&_JIT_CONTINUE)(frame, stack_pointer, tstate); Py_SETREF(tstate->previous_executor, executor); return target; } diff --git a/configure b/configure index 5be57e39309b91..1097747e055179 100755 --- a/configure +++ b/configure @@ -28091,7 +28091,6 @@ done SRCDIRS="\ Modules \ - Modules/_blake2 \ Modules/_ctypes \ Modules/_decimal \ Modules/_decimal/libmpdec \ @@ -29254,7 +29253,7 @@ case $ac_sys_system in #( py_cv_module__scproxy=n/a ;; #( - Emscripten|WASI) : + Emscripten) : @@ -29276,9 +29275,6 @@ case $ac_sys_system in #( py_cv_module_syslog=n/a py_cv_module_=n/a - case $ac_sys_system/$ac_sys_emscripten_target in #( - Emscripten/browser*) : - py_cv_module_fcntl=n/a @@ -29286,10 +29282,28 @@ case $ac_sys_system in #( py_cv_module_termios=n/a py_cv_module_=n/a - ;; #( - Emscripten/node*) : - ;; #( - WASI/*) : + ;; #( + WASI) : + + + + py_cv_module__curses=n/a + py_cv_module__curses_panel=n/a + py_cv_module__dbm=n/a + py_cv_module__gdbm=n/a + py_cv_module__multiprocessing=n/a + py_cv_module__posixshmem=n/a + py_cv_module__posixsubprocess=n/a + py_cv_module__scproxy=n/a + py_cv_module__tkinter=n/a + py_cv_module__interpreters=n/a + py_cv_module__interpchannels=n/a + py_cv_module__interpqueues=n/a + py_cv_module_grp=n/a + py_cv_module_pwd=n/a + py_cv_module_resource=n/a + py_cv_module_syslog=n/a + py_cv_module_=n/a @@ -29305,11 +29319,6 @@ case $ac_sys_system in #( py_cv_module_xxlimited_35=n/a py_cv_module_=n/a - - ;; #( - *) : - ;; -esac ;; #( *) : diff --git a/configure.ac b/configure.ac index caf0244cc7b54b..6d514705e91ce5 100644 --- a/configure.ac +++ b/configure.ac @@ -7074,7 +7074,6 @@ done AC_SUBST([SRCDIRS]) SRCDIRS="\ Modules \ - Modules/_blake2 \ Modules/_ctypes \ Modules/_decimal \ Modules/_decimal/libmpdec \ @@ -7622,12 +7621,11 @@ AS_CASE([$ac_sys_system], [CYGWIN*], [PY_STDLIB_MOD_SET_NA([_scproxy])], [QNX*], [PY_STDLIB_MOD_SET_NA([_scproxy])], [FreeBSD*], [PY_STDLIB_MOD_SET_NA([_scproxy])], - [Emscripten|WASI], [ + [Emscripten], [ dnl subprocess and multiprocessing are not supported (no fork syscall). dnl curses and tkinter user interface are not available. dnl dbm and gdbm aren't available, too. - dnl Emscripten and WASI provide only stubs for pwd, grp APIs. - dnl resource functions (get/setrusage) are stubs, too. + dnl pwd, grp APIs, and resource functions (get/setrusage) are stubs. PY_STDLIB_MOD_SET_NA( [_curses], [_curses_panel], @@ -7646,33 +7644,50 @@ AS_CASE([$ac_sys_system], [resource], [syslog], ) - AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], - [Emscripten/browser*], [ - dnl These modules are not particularly useful in browsers. - PY_STDLIB_MOD_SET_NA( - [fcntl], - [readline], - [termios], - ) - ], - [Emscripten/node*], [], - [WASI/*], [ - dnl WASI SDK 15.0 does not support file locking, mmap, and more. - dnl Test modules that must be compiled as shared libraries are not supported - dnl (see Modules/Setup.stdlib.in). - PY_STDLIB_MOD_SET_NA( - [_ctypes_test], - [_testexternalinspection], - [_testimportmultiple], - [_testmultiphase], - [_testsinglephase], - [fcntl], - [mmap], - [termios], - [xxlimited], - [xxlimited_35], - ) - ] + dnl fcntl, readline, and termios are not particularly useful in browsers. + PY_STDLIB_MOD_SET_NA( + [fcntl], + [readline], + [termios], + ) + ], + [WASI], [ + dnl subprocess and multiprocessing are not supported (no fork syscall). + dnl curses and tkinter user interface are not available. + dnl dbm and gdbm aren't available, too. + dnl pwd, grp APIs, and resource functions (get/setrusage) are stubs. + PY_STDLIB_MOD_SET_NA( + [_curses], + [_curses_panel], + [_dbm], + [_gdbm], + [_multiprocessing], + [_posixshmem], + [_posixsubprocess], + [_scproxy], + [_tkinter], + [_interpreters], + [_interpchannels], + [_interpqueues], + [grp], + [pwd], + [resource], + [syslog], + ) + dnl WASI SDK 15.0 does not support file locking, mmap, and more. + dnl Test modules that must be compiled as shared libraries are not supported + dnl (see Modules/Setup.stdlib.in). + PY_STDLIB_MOD_SET_NA( + [_ctypes_test], + [_testexternalinspection], + [_testimportmultiple], + [_testmultiphase], + [_testsinglephase], + [fcntl], + [mmap], + [termios], + [xxlimited], + [xxlimited_35], ) ], [PY_STDLIB_MOD_SET_NA([_scproxy])]