diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 1fab3f577f2f89..567b5f6855a48a 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -394,8 +394,7 @@ Initializing and finalizing the interpreter Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of Python/C API functions, and destroy all sub-interpreters (see :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since - the last call to :c:func:`Py_Initialize`. Ideally, this frees all memory - allocated by the Python interpreter. This is a no-op when called for a second + the last call to :c:func:`Py_Initialize`. This is a no-op when called for a second time (without calling :c:func:`Py_Initialize` again first). Since this is the reverse of :c:func:`Py_Initialize`, it should be called @@ -407,6 +406,12 @@ Initializing and finalizing the interpreter If there were errors during finalization (flushing buffered data), ``-1`` is returned. + Note that Python will do a best effort at freeing all memory allocated by the Python + interpreter. Therefore, any C-Extension should make sure to correctly clean up all + of the preveiously allocated PyObjects before using them in subsequent calls to + :c:func:`Py_Initialize`. Otherwise it could introduce vulnerabilities and incorrect + behavior. + This function is provided for a number of reasons. An embedding application might want to restart Python without having to restart the application itself. An application that has loaded the Python interpreter from a dynamically @@ -421,10 +426,11 @@ Initializing and finalizing the interpreter loaded extension modules loaded by Python are not unloaded. Small amounts of memory allocated by the Python interpreter may not be freed (if you find a leak, please report it). Memory tied up in circular references between objects is not - freed. Some memory allocated by extension modules may not be freed. Some - extensions may not work properly if their initialization routine is called more - than once; this can happen if an application calls :c:func:`Py_Initialize` and - :c:func:`Py_FinalizeEx` more than once. + freed. Interned strings will all be deallocated regarldess of their reference count. + Some memory allocated by extension modules may not be freed. Some extensions may not + work properly if their initialization routine is called more than once; this can + happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` + more than once. .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx diff --git a/Doc/conf.py b/Doc/conf.py index 3860d146a27e85..dc6ea6ad2b1bb6 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -546,6 +546,8 @@ r'https://msdn.microsoft.com/.*': 'https://learn.microsoft.com/.*', r'https://docs.microsoft.com/.*': 'https://learn.microsoft.com/.*', r'https://go.microsoft.com/fwlink/\?LinkID=\d+': 'https://learn.microsoft.com/.*', + # Debian's man page redirects to its current stable version + r'https://manpages.debian.org/\w+\(\d(\w+)?\)': r'https://manpages.debian.org/\w+/[\w/\-\.]*\.\d(\w+)?\.en\.html', # Language redirects r'https://toml.io': 'https://toml.io/en/', r'https://www.redhat.com': 'https://www.redhat.com/en', diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 8b3f397ea862f4..0a4246158a5488 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -570,6 +570,8 @@ Data Types >>> len(white) 3 + .. versionadded:: 3.11 + .. method:: __bool__(self): Returns *True* if any members in flag, *False* otherwise:: diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index b8faf756698097..da664f2f360ff1 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -14,8 +14,8 @@ There are two variants of the interactive :term:`REPL`. The classic basic interpreter is supported on all platforms with minimal line control capabilities. -On Unix-like systems (e.g. Linux or macOS) with :mod:`curses` and -:mod:`readline` support, a new interactive shell is used by default. +On Windows, or Unix-like systems with :mod:`curses` support, +a new interactive shell is used by default. This one supports color, multiline editing, history browsing, and paste mode. To disable color, see :ref:`using-on-controlling-color` for details. Function keys provide some additional functionality. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index bd3c45c8ab503b..97b3e17351cbd4 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -88,7 +88,7 @@ Several legacy standard library modules have now `been removed This article doesn't attempt to provide a complete specification of all new features, but instead gives a convenient overview. For full details refer to the documentation, -such as the :ref:`Library Reference ` +such as the :ref:`Library Reference ` and :ref:`Language Reference `. To understand the complete implementation and design rationale for a change, refer to the PEP for a particular new feature; @@ -173,10 +173,10 @@ New typing features: Platform support: -* :pep:`730`: Apple's iOS is now an officially supported platform, - at :pep:`tier 3 <11#tier-3>`. +* :pep:`730`: Apple's iOS is now an :ref:`officially supported platform + `, at :pep:`tier 3 <11#tier-3>`. Official Android support (:pep:`738`) is in the works as well. -* ``wasm32-wasi`` is now a supported as a :pep:`tier 2 <11#tier-2>` platform. +* ``wasm32-wasi`` is now supported as a :pep:`tier 2 <11#tier-2>` platform. * ``wasm32-emscripten`` is no longer an officially supported platform. Important removals: @@ -187,7 +187,7 @@ Important removals: :mod:`!crypt`, :mod:`!imghdr`, :mod:`!mailcap`, :mod:`!msilib`, :mod:`!nis`, :mod:`!nntplib`, :mod:`!ossaudiodev`, :mod:`!pipes`, :mod:`!sndhdr`, :mod:`!spwd`, :mod:`!sunau`, :mod:`!telnetlib`, :mod:`!uu` and :mod:`!xdrlib`. -* Remove the :program:`!2to3` tool and :mod:`!lib2to3` module +* Remove the :program:`2to3` tool and :mod:`!lib2to3` module (deprecated in Python 3.11). * Remove the :mod:`!tkinter.tix` module (deprecated in Python 3.6). * Remove :func:`!locale.resetlocale()`. @@ -209,51 +209,50 @@ This updated policy means that: New Features ============ + .. _whatsnew313-better-interactive-interpreter: -A Better Interactive Interpreter +A better interactive interpreter -------------------------------- -On Unix-like systems like Linux or macOS as well as Windows, Python now -uses a new :term:`interactive` shell. When the user starts the -:term:`REPL` from an interactive terminal the interactive shell now -supports the following new features: +Python now uses a new :term:`interactive` shell by default. +When the user starts the :term:`REPL` from an interactive terminal, +the following new features are now supported: -* Colorized prompts. * Multiline editing with history preservation. +* Direct support for REPL-specific commands like :kbd:`help`, :kbd:`exit`, + and :kbd:`quit`, without the need to call them as functions. +* Prompts and tracebacks with :ref:`color enabled by default + `. * Interactive help browsing using :kbd:`F1` with a separate command history. * History browsing using :kbd:`F2` that skips output as well as the :term:`>>>` and :term:`...` prompts. * "Paste mode" with :kbd:`F3` that makes pasting larger blocks of code easier (press :kbd:`F3` again to return to the regular prompt). -* The ability to issue REPL-specific commands like :kbd:`help`, :kbd:`exit`, - and :kbd:`quit` without the need to use call parentheses after the command - name. - -If the new interactive shell is not desired, it can be disabled via -the :envvar:`PYTHON_BASIC_REPL` environment variable. - -The new shell requires :mod:`curses` on Unix-like systems. +To disable the new interactive shell, +set the :envvar:`PYTHON_BASIC_REPL` environment variable. For more on interactive mode, see :ref:`tut-interac`. (Contributed by Pablo Galindo Salgado, Łukasz Langa, and Lysandros Nikolaou in :gh:`111201` based on code from the PyPy project. Windows support contributed by Dino Viehland and Anthony Shaw.) + .. _whatsnew313-improved-error-messages: -Improved Error Messages +Improved error messages ----------------------- -* The interpreter now colorizes error messages when displaying tracebacks by default. - This feature can be controlled via the new :envvar:`PYTHON_COLORS` environment - variable as well as the canonical |NO_COLOR|_ and |FORCE_COLOR|_ environment - variables. See also :ref:`using-on-controlling-color`. +* The interpreter now uses color by default when displaying tracebacks in the + terminal. This feature :ref:`can be controlled ` + via the new :envvar:`PYTHON_COLORS` environment variable as well as + the canonical |NO_COLOR|_ and |FORCE_COLOR|_ environment variables. (Contributed by Pablo Galindo Salgado in :gh:`112730`.) .. Apparently this how you hack together a formatted link: + (https://www.docutils.org/docs/ref/rst/directives.html#replacement-text) .. |FORCE_COLOR| replace:: ``FORCE_COLOR`` .. _FORCE_COLOR: https://force-color.org/ @@ -265,223 +264,242 @@ Improved Error Messages standard library module. When this results in errors, we now display a more helpful error message: - .. code-block:: shell-session + .. code-block:: pytb - $ python random.py - Traceback (most recent call last): - File "/home/random.py", line 1, in - import random; print(random.randint(5)) - ^^^^^^^^^^^^^ - File "/home/random.py", line 1, in - import random; print(random.randint(5)) - ^^^^^^^^^^^^^^ - AttributeError: module 'random' has no attribute 'randint' (consider renaming '/home/random.py' since it has the same name as the standard library module named 'random' and the import system gives it precedence) + $ python random.py + Traceback (most recent call last): + File "/home/me/random.py", line 1, in + import random + File "/home/me/random.py", line 3, in + print(random.randint(5)) + ^^^^^^^^^^^^^^ + AttributeError: module 'random' has no attribute 'randint' (consider renaming '/home/me/random.py' since it has the same name as the standard library module named 'random' and the import system gives it precedence) Similarly, if a script has the same name as a third-party - module it attempts to import, and this results in errors, + module that it attempts to import and this results in errors, we also display a more helpful error message: - .. code-block:: shell-session + .. code-block:: pytb - $ python numpy.py - Traceback (most recent call last): - File "/home/numpy.py", line 1, in - import numpy as np; np.array([1,2,3]) - ^^^^^^^^^^^^^^^^^^ - File "/home/numpy.py", line 1, in - import numpy as np; np.array([1,2,3]) - ^^^^^^^^ - AttributeError: module 'numpy' has no attribute 'array' (consider renaming '/home/numpy.py' if it has the same name as a third-party module you intended to import) + $ python numpy.py + Traceback (most recent call last): + File "/home/me/numpy.py", line 1, in + import numpy as np + File "/home/me/numpy.py", line 3, in + np.array([1, 2, 3]) + ^^^^^^^^ + AttributeError: module 'numpy' has no attribute 'array' (consider renaming '/home/me/numpy.py' if it has the same name as a third-party module you intended to import) (Contributed by Shantanu Jain in :gh:`95754`.) -* When an incorrect keyword argument is passed to a function, the error message - now potentially suggests the correct keyword argument. - (Contributed by Pablo Galindo Salgado and Shantanu Jain in :gh:`107944`.) +* The error message now tries to suggest the correct keyword argument + when an incorrect keyword argument is passed to a function. - >>> "better error messages!".split(max_split=1) - Traceback (most recent call last): - File "", line 1, in - "better error messages!".split(max_split=1) - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^ - TypeError: split() got an unexpected keyword argument 'max_split'. Did you mean 'maxsplit'? + .. code-block:: pycon -* Classes have a new :attr:`~class.__static_attributes__` attribute, populated by the compiler, - with a tuple of names of attributes of this class which are assigned - through ``self.X`` from any function in its body. (Contributed by Irit Katriel - in :gh:`115775`.) + >>> "Better error messages!".split(max_split=1) + Traceback (most recent call last): + File "", line 1, in + "Better error messages!".split(max_split=1) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^ + TypeError: split() got an unexpected keyword argument 'max_split'. Did you mean 'maxsplit'? -.. _whatsnew313-locals-semantics: + (Contributed by Pablo Galindo Salgado and Shantanu Jain in :gh:`107944`.) -Defined mutation semantics for :py:func:`locals` ------------------------------------------------- -Historically, the expected result of mutating the return value of :func:`locals` -has been left to individual Python implementations to define. - -Through :pep:`667`, Python 3.13 standardises the historical behaviour of CPython -for most code execution scopes, but changes -:term:`optimized scopes ` (functions, generators, coroutines, -comprehensions, and generator expressions) to explicitly return independent -snapshots of the currently assigned local variables, including locally -referenced nonlocal variables captured in closures. - -This change to the semantics of :func:`locals` in optimized scopes also affects the default -behaviour of code execution functions that implicitly target ``locals()`` if no explicit -namespace is provided (such as :func:`exec` and :func:`eval`). In previous versions, whether -or not changes could be accessed by calling ``locals()`` after calling the code execution -function was implementation dependent. In CPython specifically, such code would typically -appear to work as desired, but could sometimes fail in optimized scopes based on other code -(including debuggers and code execution tracing tools) potentially resetting the shared -snapshot in that scope. Now, the code will always run against an independent snapshot of the -local variables in optimized scopes, and hence the changes will never be visible in -subsequent calls to ``locals()``. To access the changes made in these cases, an explicit -namespace reference must now be passed to the relevant function. Alternatively, it may make -sense to update affected code to use a higher level code execution API that returns the -resulting code execution namespace (e.g. :func:`runpy.run_path` when executing Python -files from disk). +.. _whatsnew313-free-threaded-cpython: -To ensure debuggers and similar tools can reliably update local variables in -scopes affected by this change, :attr:`FrameType.f_locals ` now -returns a write-through proxy to the frame's local and locally referenced -nonlocal variables in these scopes, rather than returning an inconsistently -updated shared ``dict`` instance with undefined runtime semantics. +Free-threaded CPython +--------------------- -See :pep:`667` for more details, including related C API changes and deprecations. Porting -notes are also provided below for the affected :ref:`Python APIs ` -and :ref:`C APIs `. +CPython now has experimental support for running in a free-threaded mode, +with the :term:`global interpreter lock` (GIL) disabled. +This is an experimental feature and therefore is not enabled by default. +The free-threaded mode requires a different executable, +usually called ``python3.13t`` or ``python3.13t.exe``. +Pre-built binaries marked as *free-threaded* can be installed as part of +the official :ref:`Windows ` +and :ref:`macOS ` installers, +or CPython can be built from source with the :option:`--disable-gil` option. -(PEP and implementation contributed by Mark Shannon and Tian Gao in -:gh:`74929`. Documentation updates provided by Guido van Rossum and -Alyssa Coghlan.) - -Incremental Garbage Collection ------------------------------- +.. better macOS link pending + https://github.com/python/cpython/issues/109975#issuecomment-2286391179 -* The cycle garbage collector is now incremental. - This means that maximum pause times are reduced - by an order of magnitude or more for larger heaps. +Free-threaded execution allows for full utilization of the available +processing power by running threads in parallel on available CPU cores. +While not all software will benefit from this automatically, programs +designed with threading in mind will run faster on multi-core hardware. +**The free-threaded mode is experimental** and work is ongoing to improve it: +expect some bugs and a substantial single-threaded performance hit. +Free-threaded builds of CPython support optionally running with the GIL +enabled at runtime using the environment variable :envvar:`PYTHON_GIL` or +the command-line option :option:`-X gil`. -Support For Mobile Platforms ----------------------------- +To check if the current interpreter supports free-threading, :option:`python -VV <-V>` +and :attr:`sys.version` contain "experimental free-threading build". +The new :func:`!sys._is_gil_enabled` function can be used to check whether +the GIL is actually disabled in the running process. -* iOS is now a :pep:`11` supported platform. ``arm64-apple-ios`` - (iPhone and iPad devices released after 2013) and - ``arm64-apple-ios-simulator`` (Xcode iOS simulator running on Apple Silicon - hardware) are now tier 3 platforms. +C-API extension modules need to be built specifically for the free-threaded +build. Extensions that support running with the :term:`GIL` disabled should +use the :c:data:`Py_mod_gil` slot. Extensions using single-phase init should +use :c:func:`PyUnstable_Module_SetGIL` to indicate whether they support +running with the GIL disabled. Importing C extensions that don't use these +mechanisms will cause the GIL to be enabled, unless the GIL was explicitly +disabled with the :envvar:`PYTHON_GIL` environment variable or the +:option:`-X gil=0` option. +pip 24.1 or newer is required to install packages with C extensions in the +free-threaded build. - ``x86_64-apple-ios-simulator`` (Xcode iOS simulator running on older x86_64 - hardware) is not a tier 3 supported platform, but will be supported on a - best-effort basis. +.. seealso:: - See :pep:`730`: for more details. + :pep:`703` "Making the Global Interpreter Lock Optional in CPython" + contains rationale and information surrounding this work. - (PEP written and implementation contributed by Russell Keith-Magee in - :gh:`114099`.) .. _whatsnew313-jit-compiler: -Experimental JIT Compiler -------------------------- +An experimental just-in-time (JIT) compiler +------------------------------------------- -When CPython is configured using the ``--enable-experimental-jit`` option, -a just-in-time compiler is added which may speed up some Python programs. +When CPython is configured and built using +the :option:`!--enable-experimental-jit` option, +a just-in-time (JIT) compiler is added which may speed up some Python programs. +On Windows, use ``PCbuild/build.bat --experimental-jit`` to enable the JIT +or ``--experimental-jit-interpreter`` to enable the Tier 2 interpreter. +Build requirements and further supporting information `are contained at`__ +:file:`Tools/jit/README.md`. -The internal architecture is roughly as follows. +__ https://github.com/python/cpython/blob/main/Tools/jit/README.md -* We start with specialized *Tier 1 bytecode*. - See :ref:`What's new in 3.11 ` for details. +The :option:`!--enable-experimental-jit` option takes these (optional) values, +defaulting to ``yes`` if :option:`!--enable-experimental-jit` is present +without the optional value. -* When the Tier 1 bytecode gets hot enough, it gets translated - to a new, purely internal *Tier 2 IR*, a.k.a. micro-ops ("uops"). +* ``no``: Disable the entire Tier 2 and JIT pipeline. +* ``yes``: Enable the JIT. + To disable the JIT at runtime, pass the environment variable ``PYTHON_JIT=0``. +* ``yes-off``: Build the JIT but disable it by default. + To enable the JIT at runtime, pass the environment variable ``PYTHON_JIT=1``. +* ``interpreter``: Enable the Tier 2 interpreter but disable the JIT. + The interpreter can be disabled by running with ``PYTHON_JIT=0``. -* The Tier 2 IR uses the same stack-based VM as Tier 1, but the - instruction format is better suited to translation to machine code. +The internal architecture is roughly as follows: +* We start with specialized *Tier 1 bytecode*. + See :ref:`What's new in 3.11 ` for details. +* When the Tier 1 bytecode gets hot enough, it gets translated + to a new purely internal intermediate representation (IR), + called the *Tier 2 IR*, and sometimes referred to as micro-ops ("uops"). +* The Tier 2 IR uses the same stack-based virtual machine as Tier 1, + but the instruction format is better suited to translation to machine code. * We have several optimization passes for Tier 2 IR, which are applied before it is interpreted or translated to machine code. - * There is a Tier 2 interpreter, but it is mostly intended for debugging the earlier stages of the optimization pipeline. The Tier 2 interpreter can be enabled by configuring Python with ``--enable-experimental-jit=interpreter``. - * When the JIT is enabled, the optimized Tier 2 IR is translated to machine code, which is then executed. - * The machine code translation process uses a technique called *copy-and-patch*. It has no runtime dependencies, but there is a new build-time dependency on LLVM. -The ``--enable-experimental-jit`` flag has the following optional values: +.. seealso:: :pep:`744` -* ``no`` (default) -- Disable the entire Tier 2 and JIT pipeline. +(JIT by Brandt Bucher, inspired by a paper by Haoran Xu and Fredrik Kjolstad. +Tier 2 IR by Mark Shannon and Guido van Rossum. +Tier 2 optimizer by Ken Jin.) -* ``yes`` (default if the flag is present without optional value) - -- Enable the JIT. To disable the JIT at runtime, - pass the environment variable ``PYTHON_JIT=0``. -* ``yes-off`` -- Build the JIT but disable it by default. - To enable the JIT at runtime, pass the environment variable - ``PYTHON_JIT=1``. +.. _whatsnew313-locals-semantics: -* ``interpreter`` -- Enable the Tier 2 interpreter but disable the JIT. - The interpreter can be disabled by running with - ``PYTHON_JIT=0``. +Defined mutation semantics for :py:func:`locals` +------------------------------------------------ -(On Windows, use ``PCbuild/build.bat --experimental-jit`` to enable the JIT -or ``--experimental-jit-interpreter`` to enable the Tier 2 interpreter.) +Historically, the expected result of mutating the return value of +:func:`locals` has been left to individual Python implementations to define. +Starting from Python 3.13, :pep:`667` standardises +the historical behaviour of CPython for most code execution scopes, +but changes :term:`optimized scopes ` +(functions, generators, coroutines, comprehensions, and generator expressions) +to explicitly return independent snapshots of the currently assigned local +variables, including locally referenced nonlocal variables captured in closures. + +This change to the semantics of :func:`locals` in optimized scopes also +affects the default behaviour of code execution functions that implicitly +target :func:`!locals` if no explicit namespace is provided +(such as :func:`exec` and :func:`eval`). +In previous versions, whether or not changes could be accessed by calling +:func:`!locals` after calling the code execution function was +implementation-dependent. In CPython specifically, such code would typically +appear to work as desired, but could sometimes fail in optimized scopes based +on other code (including debuggers and code execution tracing tools) +potentially resetting the shared snapshot in that scope. +Now, the code will always run against an independent snapshot of +the local variables in optimized scopes, and hence the changes will never +be visible in subsequent calls to :func:`!locals`. +To access the changes made in these cases, an explicit namespace reference +must now be passed to the relevant function. +Alternatively, it may make sense to update affected code to use a higher level +code execution API that returns the resulting code execution namespace +(e.g. :func:`runpy.run_path` when executing Python files from disk). -See :pep:`744` for more details. +To ensure debuggers and similar tools can reliably update local variables in +scopes affected by this change, :attr:`FrameType.f_locals ` now +returns a write-through proxy to the frame's local and locally referenced +nonlocal variables in these scopes, rather than returning an inconsistently +updated shared ``dict`` instance with undefined runtime semantics. -(JIT by Brandt Bucher, inspired by a paper by Haoran Xu and Fredrik Kjolstad. -Tier 2 IR by Mark Shannon and Guido van Rossum. -Tier 2 optimizer by Ken Jin.) +See :pep:`667` for more details, including related C API changes +and deprecations. Porting notes are also provided below for the affected +:ref:`Python APIs ` and :ref:`C APIs +`. -.. _whatsnew313-free-threaded-cpython: +(PEP and implementation contributed by Mark Shannon and Tian Gao in +:gh:`74929`. Documentation updates provided by Guido van Rossum and +Alyssa Coghlan.) -Free-threaded CPython ---------------------- -CPython will run with the :term:`global interpreter lock` (GIL) disabled when -configured using the ``--disable-gil`` option at build time. This is an -experimental feature and therefore isn't used by default. Users need to -either compile their own interpreter, or install one of the experimental -builds that are marked as *free-threaded*. See :pep:`703` "Making the Global -Interpreter Lock Optional in CPython" for more detail. +.. _whatsnew313-platform-support: -Free-threaded execution allows for full utilization of the available -processing power by running threads in parallel on available CPU cores. -While not all software will benefit from this automatically, programs -designed with threading in mind will run faster on multicore hardware. +Support for mobile platforms +---------------------------- -Work is still ongoing: expect some bugs and a substantial single-threaded -performance hit. +:pep:`730`: iOS is now a :pep:`11` supported platform, with the +``arm64-apple-ios`` and ``arm64-apple-ios-simulator`` targets at tier 3 +(iPhone and iPad devices released after 2013 and the Xcode iOS simulator +running on Apple silicon hardware, respectively). +``x86_64-apple-ios-simulator`` +(the Xcode iOS simulator running on older ``x86_64`` hardware) +is not a tier 3 supported platform, but will have best-effort support. +(PEP written and implementation contributed by Russell Keith-Magee in +:gh:`114099`.) -The free-threaded build still supports optionally running with the GIL -enabled at runtime using the environment variable :envvar:`PYTHON_GIL` or -the command line option :option:`-X gil`. +:pep:`738`: Android support is being actively worked on, +but the platform is not yet officially supported. +(PEP written and implementation contributed by Malcolm Smith in +:gh:`116622`.) -To check if the current interpreter is configured with ``--disable-gil``, -use ``sysconfig.get_config_var("Py_GIL_DISABLED")``. To check if the :term:`GIL` -is actually disabled in the running process, the :func:`!sys._is_gil_enabled` -function can be used. +.. seealso:: :pep:`730`, :pep:`738` -C-API extension modules need to be built specifically for the free-threaded -build. Extensions that support running with the :term:`GIL` disabled should -use the :c:data:`Py_mod_gil` slot. Extensions using single-phase init should -use :c:func:`PyUnstable_Module_SetGIL` to indicate whether they support -running with the GIL disabled. Importing C extensions that don't use these -mechanisms will cause the GIL to be enabled, unless the GIL was explicitly -disabled with the :envvar:`PYTHON_GIL` environment variable or the -:option:`-X gil=0` option. -pip 24.1b1 or newer is required to install packages with C extensions in the -free-threaded build. +Incremental garbage collection +------------------------------ + +The cycle garbage collector is now incremental. +This means that maximum pause times are reduced +by an order of magnitude or more for larger heaps. + Other Language Changes ====================== +* Classes have a new :attr:`~class.__static_attributes__` attribute, populated by the compiler, + with a tuple of names of attributes of this class which are assigned + through ``self.X`` from any function in its body. (Contributed by Irit Katriel + in :gh:`115775`.) + * The :func:`exec` and :func:`eval` built-ins now accept their ``globals`` and ``locals`` namespace arguments as keywords. (Contributed by Raphael Gaschignard in :gh:`105879`) @@ -2033,8 +2051,8 @@ Changes in the Python API independent snapshot on each call, and hence no longer implicitly updates previously returned references. Obtaining the legacy CPython behaviour now requires explicit calls to update the initially returned dictionary with the - results of subsequent calls to ``locals()``. Code execution functions that - implicitly target ``locals()`` (such as ``exec`` and ``eval``) must be + results of subsequent calls to :func:`!locals`. Code execution functions that + implicitly target :func:`!locals` (such as ``exec`` and ``eval``) must be passed an explicit namespace to access their results in an optimized scope. (Changed as part of :pep:`667`.) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 267860712b9967..088f70d9e9fad4 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -419,6 +419,16 @@ New Features which has an ambiguous return value. (Contributed by Irit Katriel and Erlend Aasland in :gh:`105201`.) +* :c:func:`Py_Finalize` now deletes all interned strings. This + is backwards incompatible to any C-Extension that holds onto an interned + string after a call to :c:func:`Py_Finalize` and is then reused after a + call to :c:func:`Py_Initialize`. Any issues arising from this behavior will + normally result in crashes during the exectuion of the subsequent call to + :c:func:`Py_Initialize` from accessing uninitialized memory. To fix, use + an address sanitizer to identify any use-after-free coming from + an interned string and deallocate it during module shutdown. + (Contribued by Eddie Elizondo in :gh:`113601`.) + Porting to Python 3.14 ---------------------- diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index d3a5be000fbce7..d3d745574667aa 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -128,6 +128,13 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * // Don't leave a dangling pointer to the old frame when creating generators // and coroutines: dest->previous = NULL; + +#ifdef Py_GIL_DISABLED + PyCodeObject *co = (PyCodeObject *)dest->f_executable; + for (int i = stacktop; i < co->co_nlocalsplus + co->co_stacksize; i++) { + dest->localsplus[i] = PyStackRef_NULL; + } +#endif } /* Consumes reference to func and locals. @@ -153,6 +160,16 @@ _PyFrame_Initialize( for (int i = null_locals_from; i < code->co_nlocalsplus; i++) { frame->localsplus[i] = PyStackRef_NULL; } + +#ifdef Py_GIL_DISABLED + // On GIL disabled, we walk the entire stack in GC. Since stacktop + // is not always in sync with the real stack pointer, we have + // no choice but to traverse the entire stack. + // This just makes sure we don't pass the GC invalid stack values. + for (int i = code->co_nlocalsplus; i < code->co_nlocalsplus + code->co_stacksize; i++) { + frame->localsplus[i] = PyStackRef_NULL; + } +#endif } /* Gets the pointer to the locals array @@ -314,6 +331,13 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->instr_ptr = _PyCode_CODE(code); frame->owner = FRAME_OWNED_BY_THREAD; frame->return_offset = 0; + +#ifdef Py_GIL_DISABLED + assert(code->co_nlocalsplus == 0); + for (int i = 0; i < code->co_stacksize; i++) { + frame->localsplus[i] = PyStackRef_NULL; + } +#endif return frame; } diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 5dd5b0c78d42fa..89f6017bacc525 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -381,6 +381,8 @@ extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); extern void _Py_ScheduleGC(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); +// GC visit callback for tracked interpreter frames +extern int _PyGC_VisitFrameStack(struct _PyInterpreterFrame *frame, visitproc visit, void *arg); #ifdef __cplusplus } diff --git a/Include/internal/pycore_mimalloc.h b/Include/internal/pycore_mimalloc.h index d10b01d5b49b19..d870d01beb702c 100644 --- a/Include/internal/pycore_mimalloc.h +++ b/Include/internal/pycore_mimalloc.h @@ -36,9 +36,18 @@ typedef enum { # define MI_TSAN 1 #endif +#ifdef __cplusplus +extern "C++" { +#endif + #include "mimalloc/mimalloc.h" #include "mimalloc/mimalloc/types.h" #include "mimalloc/mimalloc/internal.h" + +#ifdef __cplusplus +} +#endif + #endif #ifdef Py_GIL_DISABLED diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index d4ffd977940a02..d4291b87261ae0 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -44,6 +44,15 @@ struct _gilstate_runtime_state { /* Runtime audit hook state */ +#define _Py_Debug_Cookie "xdebugpy" + +#ifdef Py_GIL_DISABLED +# define _Py_Debug_gilruntimestate_enabled offsetof(struct _gil_runtime_state, enabled) +# define _Py_Debug_Free_Threaded 1 +#else +# define _Py_Debug_gilruntimestate_enabled 0 +# define _Py_Debug_Free_Threaded 0 +#endif typedef struct _Py_AuditHookEntry { struct _Py_AuditHookEntry *next; Py_AuditHookFunction hookCFunction; @@ -53,6 +62,7 @@ typedef struct _Py_AuditHookEntry { typedef struct _Py_DebugOffsets { char cookie[8]; uint64_t version; + uint64_t free_threaded; // Runtime state offset; struct _runtime_state { uint64_t size; @@ -71,6 +81,8 @@ typedef struct _Py_DebugOffsets { uint64_t sysdict; uint64_t builtins; uint64_t ceval_gil; + uint64_t gil_runtime_state; + uint64_t gil_runtime_state_enabled; uint64_t gil_runtime_state_locked; uint64_t gil_runtime_state_holder; } interpreter_state; @@ -122,20 +134,57 @@ typedef struct _Py_DebugOffsets { struct _type_object { uint64_t size; uint64_t tp_name; + uint64_t tp_repr; + uint64_t tp_flags; } type_object; // PyTuple object offset; struct _tuple_object { uint64_t size; uint64_t ob_item; + uint64_t ob_size; } tuple_object; + // PyList object offset; + struct _list_object { + uint64_t size; + uint64_t ob_item; + uint64_t ob_size; + } list_object; + + // PyDict object offset; + struct _dict_object { + uint64_t size; + uint64_t ma_keys; + uint64_t ma_values; + } dict_object; + + // PyFloat object offset; + struct _float_object { + uint64_t size; + uint64_t ob_fval; + } float_object; + + // PyLong object offset; + struct _long_object { + uint64_t size; + uint64_t lv_tag; + uint64_t ob_digit; + } long_object; + + // PyBytes object offset; + struct _bytes_object { + uint64_t size; + uint64_t ob_size; + uint64_t ob_sval; + } bytes_object; + // Unicode object offset; struct _unicode_object { uint64_t size; uint64_t state; uint64_t length; - size_t asciiobject_size; + uint64_t asciiobject_size; } unicode_object; // GC runtime state offset; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index da2b8d5570de62..1746bbf0eec8b5 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -29,11 +29,12 @@ extern PyTypeObject _PyExc_MemoryError; /* The static initializers defined here should only be used in the runtime init code (in pystate.c and pylifecycle.c). */ -#define _PyRuntimeState_INIT(runtime) \ +#define _PyRuntimeState_INIT(runtime, debug_cookie) \ { \ .debug_offsets = { \ - .cookie = "xdebugpy", \ + .cookie = debug_cookie, \ .version = PY_VERSION_HEX, \ + .free_threaded = _Py_Debug_Free_Threaded, \ .runtime_state = { \ .size = sizeof(_PyRuntimeState), \ .finalizing = offsetof(_PyRuntimeState, _finalizing), \ @@ -49,6 +50,8 @@ extern PyTypeObject _PyExc_MemoryError; .sysdict = offsetof(PyInterpreterState, sysdict), \ .builtins = offsetof(PyInterpreterState, builtins), \ .ceval_gil = offsetof(PyInterpreterState, ceval.gil), \ + .gil_runtime_state = offsetof(PyInterpreterState, _gil), \ + .gil_runtime_state_enabled = _Py_Debug_gilruntimestate_enabled, \ .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \ .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \ }, \ @@ -90,10 +93,37 @@ extern PyTypeObject _PyExc_MemoryError; .type_object = { \ .size = sizeof(PyTypeObject), \ .tp_name = offsetof(PyTypeObject, tp_name), \ + .tp_repr = offsetof(PyTypeObject, tp_repr), \ + .tp_flags = offsetof(PyTypeObject, tp_flags), \ }, \ .tuple_object = { \ .size = sizeof(PyTupleObject), \ .ob_item = offsetof(PyTupleObject, ob_item), \ + .ob_size = offsetof(PyTupleObject, ob_base.ob_size), \ + }, \ + .list_object = { \ + .size = sizeof(PyListObject), \ + .ob_item = offsetof(PyListObject, ob_item), \ + .ob_size = offsetof(PyListObject, ob_base.ob_size), \ + }, \ + .dict_object = { \ + .size = sizeof(PyDictObject), \ + .ma_keys = offsetof(PyDictObject, ma_keys), \ + .ma_values = offsetof(PyDictObject, ma_values), \ + }, \ + .float_object = { \ + .size = sizeof(PyFloatObject), \ + .ob_fval = offsetof(PyFloatObject, ob_fval), \ + }, \ + .long_object = { \ + .size = sizeof(PyLongObject), \ + .lv_tag = offsetof(_PyLongValue, lv_tag), \ + .ob_digit = offsetof(_PyLongValue, ob_digit), \ + }, \ + .bytes_object = { \ + .size = sizeof(PyBytesObject), \ + .ob_size = offsetof(PyBytesObject, ob_base.ob_size), \ + .ob_sval = offsetof(PyBytesObject, ob_sval), \ }, \ .unicode_object = { \ .size = sizeof(PyUnicodeObject), \ diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 1b35a3e3269257..8c9bb1ae8c4908 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -150,8 +150,7 @@ PyStackRef_FromPyObjectNew(PyObject *obj) // Make sure we don't take an already tagged value. assert(((uintptr_t)obj & Py_TAG_BITS) == 0); assert(obj != NULL); - // TODO (gh-117139): Add deferred objects later. - if (_Py_IsImmortal(obj)) { + if (_Py_IsImmortal(obj) || _PyObject_HasDeferredRefcount(obj)) { return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; } else { @@ -220,7 +219,8 @@ PyStackRef_DUP(_PyStackRef stackref) { if (PyStackRef_IsDeferred(stackref)) { assert(PyStackRef_IsNull(stackref) || - _Py_IsImmortal(PyStackRef_AsPyObjectBorrow(stackref))); + _Py_IsImmortal(PyStackRef_AsPyObjectBorrow(stackref)) || + _PyObject_HasDeferredRefcount(PyStackRef_AsPyObjectBorrow(stackref))); return stackref; } Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref)); diff --git a/Lib/test/_test_embed_structseq.py b/Lib/test/_test_embed_structseq.py index 834daa4df55fec..868f9f83e8be77 100644 --- a/Lib/test/_test_embed_structseq.py +++ b/Lib/test/_test_embed_structseq.py @@ -1,31 +1,27 @@ import sys import types +import unittest -# Note: This test file can't import `unittest` since the runtime can't -# currently guarantee that it will not leak memory. Doing so will mark -# the test as passing but with reference leaks. This can safely import -# the `unittest` library once there's a strict guarantee of no leaks -# during runtime shutdown. # bpo-46417: Test that structseq types used by the sys module are still # valid when Py_Finalize()/Py_Initialize() are called multiple times. -class TestStructSeq: +class TestStructSeq(unittest.TestCase): # test PyTypeObject members - def _check_structseq(self, obj_type): + def check_structseq(self, obj_type): # ob_refcnt - assert sys.getrefcount(obj_type) > 1 + self.assertGreaterEqual(sys.getrefcount(obj_type), 1) # tp_base - assert issubclass(obj_type, tuple) + self.assertTrue(issubclass(obj_type, tuple)) # tp_bases - assert obj_type.__bases__ == (tuple,) + self.assertEqual(obj_type.__bases__, (tuple,)) # tp_dict - assert isinstance(obj_type.__dict__, types.MappingProxyType) + self.assertIsInstance(obj_type.__dict__, types.MappingProxyType) # tp_mro - assert obj_type.__mro__ == (obj_type, tuple, object) + self.assertEqual(obj_type.__mro__, (obj_type, tuple, object)) # tp_name - assert isinstance(type.__name__, str) + self.assertIsInstance(type.__name__, str) # tp_subclasses - assert obj_type.__subclasses__() == [] + self.assertEqual(obj_type.__subclasses__(), []) def test_sys_attrs(self): for attr_name in ( @@ -36,23 +32,23 @@ def test_sys_attrs(self): 'thread_info', # ThreadInfoType 'version_info', # VersionInfoType ): - attr = getattr(sys, attr_name) - self._check_structseq(type(attr)) + with self.subTest(attr=attr_name): + attr = getattr(sys, attr_name) + self.check_structseq(type(attr)) def test_sys_funcs(self): func_names = ['get_asyncgen_hooks'] # AsyncGenHooksType if hasattr(sys, 'getwindowsversion'): func_names.append('getwindowsversion') # WindowsVersionType for func_name in func_names: - func = getattr(sys, func_name) - obj = func() - self._check_structseq(type(obj)) + with self.subTest(func=func_name): + func = getattr(sys, func_name) + obj = func() + self.check_structseq(type(obj)) try: - tests = TestStructSeq() - tests.test_sys_attrs() - tests.test_sys_funcs() + unittest.main() except SystemExit as exc: if exc.args[0] != 0: raise diff --git a/Makefile.pre.in b/Makefile.pre.in index 3a6432d406e01d..363d2a24907ba5 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -222,6 +222,7 @@ LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a LIBEXPAT_A= Modules/expat/libexpat.a LIBHACL_SHA2_A= Modules/_hacl/libHacl_Hash_SHA2.a LIBHACL_BLAKE2_A= Modules/_hacl/libHacl_Hash_Blake2.a +LIBHACL_CFLAGS=@LIBHACL_CFLAGS@ LIBHACL_SIMD128_FLAGS=@LIBHACL_SIMD128_FLAGS@ LIBHACL_SIMD256_FLAGS=@LIBHACL_SIMD256_FLAGS@ LIBHACL_SIMD128_OBJS=@LIBHACL_SIMD128_OBJS@ @@ -1368,7 +1369,6 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS) # Build HACL* static libraries for hashlib: libHacl_Hash_SHA2.a, and # libHacl_Blake2.a -- the contents of the latter vary depending on whether we # have the ability to compile vectorized versions -LIBHACL_CFLAGS=-I$(srcdir)/Modules/_hacl -I$(srcdir)/Modules/_hacl/include -D_BSD_SOURCE -D_DEFAULT_SOURCE $(PY_STDMODULE_CFLAGS) $(CCSHARED) Modules/_hacl/Hacl_Hash_SHA2.o: $(srcdir)/Modules/_hacl/Hacl_Hash_SHA2.c $(LIBHACL_SHA2_HEADERS) $(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_SHA2.c diff --git a/Misc/NEWS.d/next/Build/2024-08-14-19-17-34.gh-issue-121634.eOMfHG.rst b/Misc/NEWS.d/next/Build/2024-08-14-19-17-34.gh-issue-121634.eOMfHG.rst new file mode 100644 index 00000000000000..025b6bca809898 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-08-14-19-17-34.gh-issue-121634.eOMfHG.rst @@ -0,0 +1 @@ +Allow for specifying the target compile triple for WASI. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst new file mode 100644 index 00000000000000..4c12870c3df548 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst @@ -0,0 +1 @@ +:c:func:`Py_Finalize` now deletes all interned strings. diff --git a/Misc/NEWS.d/next/Security/2024-08-06-00-06-23.gh-issue-112301.4k4lw6.rst b/Misc/NEWS.d/next/Security/2024-08-06-00-06-23.gh-issue-112301.4k4lw6.rst new file mode 100644 index 00000000000000..0bd2f4d7810a78 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-08-06-00-06-23.gh-issue-112301.4k4lw6.rst @@ -0,0 +1,2 @@ +Add ability to ignore warnings per file with warning count in warning checking tooling. +Patch by Nate Ohlson. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index da9c5857cc3bee..148d3e55bf830e 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -15623,19 +15623,7 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) int shared = 0; switch (PyUnicode_CHECK_INTERNED(s)) { case SSTATE_INTERNED_IMMORTAL: - /* Make immortal interned strings mortal again. - * - * Currently, the runtime is not able to guarantee that it can exit - * without allocations that carry over to a future initialization - * of Python within the same process. i.e: - * ./python -X showrefcount -c 'import itertools' - * [237 refs, 237 blocks] - * - * This should remain disabled (`Py_DEBUG` only) until there is a - * strict guarantee that no memory will be left after - * `Py_Finalize`. - */ -#ifdef Py_DEBUG + /* Make immortal interned strings mortal again. */ // Skip the Immortal Instance check and restore // the two references (key and value) ignored // by PyUnicode_InternInPlace(). @@ -15648,7 +15636,6 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) #ifdef INTERNED_STATS total_length += PyUnicode_GET_LENGTH(s); #endif -#endif // Py_DEBUG break; case SSTATE_INTERNED_IMMORTAL_STATIC: /* It is shared between interpreters, so we should unmark it diff --git a/Python/compile.c b/Python/compile.c index 237b5dba0497fd..93dd9de2248f18 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -5371,38 +5371,28 @@ typedef struct { PyObject *temp_symbols; PyObject *fast_hidden; jump_target_label cleanup; - jump_target_label end; } inlined_comprehension_state; static int -push_inlined_comprehension_state(struct compiler *c, location loc, - PySTEntryObject *entry, - inlined_comprehension_state *state) +compiler_tweak_inlined_comprehension_scopes(struct compiler *c, location loc, + PySTEntryObject *entry, + inlined_comprehension_state *state) { int in_class_block = (SYMTABLE_ENTRY(c)->ste_type == ClassBlock) && !c->u->u_in_inlined_comp; c->u->u_in_inlined_comp++; - // iterate over names bound in the comprehension and ensure we isolate - // them from the outer scope as needed + PyObject *k, *v; Py_ssize_t pos = 0; while (PyDict_Next(entry->ste_symbols, &pos, &k, &v)) { long symbol = PyLong_AsLong(v); - if (symbol == -1 && PyErr_Occurred()) { - return ERROR; - } + assert(symbol >= 0 || PyErr_Occurred()); + RETURN_IF_ERROR(symbol); long scope = SYMBOL_TO_SCOPE(symbol); - PyObject *outv = PyDict_GetItemWithError(SYMTABLE_ENTRY(c)->ste_symbols, k); - if (outv == NULL) { - if (PyErr_Occurred()) { - return ERROR; - } - outv = _PyLong_GetZero(); - } - long outsymbol = PyLong_AsLong(outv); - if (outsymbol == -1 && PyErr_Occurred()) { - return ERROR; - } + + long outsymbol = _PyST_GetSymbol(SYMTABLE_ENTRY(c), k); + RETURN_IF_ERROR(outsymbol); long outsc = SYMBOL_TO_SCOPE(outsymbol); + // If a name has different scope inside than outside the comprehension, // we need to temporarily handle it with the right scope while // compiling the comprehension. If it's free in the comprehension @@ -5422,16 +5412,16 @@ push_inlined_comprehension_state(struct compiler *c, location loc, // update the symbol to the in-comprehension version and save // the outer version; we'll restore it after running the // comprehension - Py_INCREF(outv); if (PyDict_SetItem(SYMTABLE_ENTRY(c)->ste_symbols, k, v) < 0) { - Py_DECREF(outv); return ERROR; } - if (PyDict_SetItem(state->temp_symbols, k, outv) < 0) { - Py_DECREF(outv); + PyObject *outv = PyLong_FromLong(outsymbol); + if (outv == NULL) { return ERROR; } + int res = PyDict_SetItem(state->temp_symbols, k, outv); Py_DECREF(outv); + RETURN_IF_ERROR(res); } // locals handling for names bound in comprehension (DEF_LOCAL | // DEF_NONLOCAL occurs in assignment expression to nonlocal) @@ -5442,9 +5432,8 @@ push_inlined_comprehension_state(struct compiler *c, location loc, if (PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, k, &orig) < 0) { return ERROR; } - int orig_is_true = (orig == Py_True); - Py_XDECREF(orig); - if (!orig_is_true) { + assert(orig == NULL || orig == Py_True || orig == Py_False); + if (orig != Py_True) { if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_True) < 0) { return ERROR; } @@ -5459,6 +5448,33 @@ push_inlined_comprehension_state(struct compiler *c, location loc, } } } + } + } + return SUCCESS; +} + +static int +codegen_push_inlined_comprehension_locals(struct compiler *c, location loc, + PySTEntryObject *comp, + inlined_comprehension_state *state) +{ + int in_class_block = (SYMTABLE_ENTRY(c)->ste_type == ClassBlock) && !c->u->u_in_inlined_comp; + PySTEntryObject *outer = SYMTABLE_ENTRY(c); + // iterate over names bound in the comprehension and ensure we isolate + // them from the outer scope as needed + PyObject *k, *v; + Py_ssize_t pos = 0; + while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) { + long symbol = PyLong_AsLong(v); + assert(symbol >= 0 || PyErr_Occurred()); + RETURN_IF_ERROR(symbol); + long scope = SYMBOL_TO_SCOPE(symbol); + + long outsymbol = _PyST_GetSymbol(outer, k); + RETURN_IF_ERROR(outsymbol); + long outsc = SYMBOL_TO_SCOPE(outsymbol); + + if ((symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) || in_class_block) { // local names bound in comprehension must be isolated from // outer scope; push existing value (which may be NULL if // not defined) on stack @@ -5496,31 +5512,40 @@ push_inlined_comprehension_state(struct compiler *c, location loc, // handler or finally block. NEW_JUMP_TARGET_LABEL(c, cleanup); state->cleanup = cleanup; - NEW_JUMP_TARGET_LABEL(c, end); - state->end = end; // no need to push an fblock for this "virtual" try/finally; there can't // be return/continue/break inside a comprehension ADDOP_JUMP(c, loc, SETUP_FINALLY, cleanup); } + return SUCCESS; +} +static int +push_inlined_comprehension_state(struct compiler *c, location loc, + PySTEntryObject *comp, + inlined_comprehension_state *state) +{ + RETURN_IF_ERROR( + compiler_tweak_inlined_comprehension_scopes(c, loc, comp, state)); + RETURN_IF_ERROR( + codegen_push_inlined_comprehension_locals(c, loc, comp, state)); return SUCCESS; } static int restore_inlined_comprehension_locals(struct compiler *c, location loc, - inlined_comprehension_state state) + inlined_comprehension_state *state) { PyObject *k; // pop names we pushed to stack earlier - Py_ssize_t npops = PyList_GET_SIZE(state.pushed_locals); + Py_ssize_t npops = PyList_GET_SIZE(state->pushed_locals); // Preserve the comprehension result (or exception) as TOS. This - // reverses the SWAP we did in push_inlined_comprehension_state to get - // the outermost iterable to TOS, so we can still just iterate + // reverses the SWAP we did in push_inlined_comprehension_state + // to get the outermost iterable to TOS, so we can still just iterate // pushed_locals in simple reverse order ADDOP_I(c, loc, SWAP, npops + 1); for (Py_ssize_t i = npops - 1; i >= 0; --i) { - k = PyList_GetItem(state.pushed_locals, i); + k = PyList_GetItem(state->pushed_locals, i); if (k == NULL) { return ERROR; } @@ -5530,43 +5555,47 @@ restore_inlined_comprehension_locals(struct compiler *c, location loc, } static int -pop_inlined_comprehension_state(struct compiler *c, location loc, - inlined_comprehension_state state) +codegen_pop_inlined_comprehension_locals(struct compiler *c, location loc, + inlined_comprehension_state *state) { - c->u->u_in_inlined_comp--; - PyObject *k, *v; - Py_ssize_t pos = 0; - if (state.temp_symbols) { - while (PyDict_Next(state.temp_symbols, &pos, &k, &v)) { - if (PyDict_SetItem(SYMTABLE_ENTRY(c)->ste_symbols, k, v)) { - return ERROR; - } - } - Py_CLEAR(state.temp_symbols); - } - if (state.pushed_locals) { + if (state->pushed_locals) { ADDOP(c, NO_LOCATION, POP_BLOCK); - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, state.end); + + NEW_JUMP_TARGET_LABEL(c, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); // cleanup from an exception inside the comprehension - USE_LABEL(c, state.cleanup); + USE_LABEL(c, state->cleanup); // discard incomplete comprehension result (beneath exc on stack) ADDOP_I(c, NO_LOCATION, SWAP, 2); ADDOP(c, NO_LOCATION, POP_TOP); - if (restore_inlined_comprehension_locals(c, loc, state) < 0) { - return ERROR; - } + RETURN_IF_ERROR(restore_inlined_comprehension_locals(c, loc, state)); ADDOP_I(c, NO_LOCATION, RERAISE, 0); - USE_LABEL(c, state.end); - if (restore_inlined_comprehension_locals(c, loc, state) < 0) { - return ERROR; + USE_LABEL(c, end); + RETURN_IF_ERROR(restore_inlined_comprehension_locals(c, loc, state)); + Py_CLEAR(state->pushed_locals); + } + return SUCCESS; +} + +static int +compiler_revert_inlined_comprehension_scopes(struct compiler *c, location loc, + inlined_comprehension_state *state) +{ + if (state->temp_symbols) { + PyObject *k, *v; + Py_ssize_t pos = 0; + while (PyDict_Next(state->temp_symbols, &pos, &k, &v)) { + if (PyDict_SetItem(SYMTABLE_ENTRY(c)->ste_symbols, k, v)) { + return ERROR; + } } - Py_CLEAR(state.pushed_locals); + Py_CLEAR(state->temp_symbols); } - if (state.fast_hidden) { - while (PySet_Size(state.fast_hidden) > 0) { - PyObject *k = PySet_Pop(state.fast_hidden); + if (state->fast_hidden) { + while (PySet_Size(state->fast_hidden) > 0) { + PyObject *k = PySet_Pop(state->fast_hidden); if (k == NULL) { return ERROR; } @@ -5578,11 +5607,21 @@ pop_inlined_comprehension_state(struct compiler *c, location loc, } Py_DECREF(k); } - Py_CLEAR(state.fast_hidden); + Py_CLEAR(state->fast_hidden); } return SUCCESS; } +static int +pop_inlined_comprehension_state(struct compiler *c, location loc, + inlined_comprehension_state *state) +{ + c->u->u_in_inlined_comp--; + RETURN_IF_ERROR(codegen_pop_inlined_comprehension_locals(c, loc, state)); + RETURN_IF_ERROR(compiler_revert_inlined_comprehension_scopes(c, loc, state)); + return SUCCESS; +} + static inline int compiler_comprehension_iter(struct compiler *c, location loc, comprehension_ty comp) @@ -5603,7 +5642,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, expr_ty val) { PyCodeObject *co = NULL; - inlined_comprehension_state inline_state = {NULL, NULL, NULL, NO_LABEL, NO_LABEL}; + inlined_comprehension_state inline_state = {NULL, NULL, NULL, NO_LABEL}; comprehension_ty outermost; #ifndef NDEBUG int scope_type = c->u->u_scope_type; @@ -5671,7 +5710,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, } if (is_inlined) { - if (pop_inlined_comprehension_state(c, loc, inline_state)) { + if (pop_inlined_comprehension_state(c, loc, &inline_state)) { goto error; } return SUCCESS; diff --git a/Python/frame.c b/Python/frame.c index 25fa2824630f9b..3192968a0fb1b5 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -15,15 +15,7 @@ _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg) Py_VISIT(frame->f_locals); Py_VISIT(frame->f_funcobj); Py_VISIT(_PyFrame_GetCode(frame)); - /* locals */ - _PyStackRef *locals = _PyFrame_GetLocalsArray(frame); - _PyStackRef *sp = frame->stackpointer; - /* locals and stack */ - while (sp > locals) { - sp--; - Py_VISIT(PyStackRef_AsPyObjectBorrow(*sp)); - } - return 0; + return _PyGC_VisitFrameStack(frame, visit, arg); } PyFrameObject * diff --git a/Python/gc.c b/Python/gc.c index 38a0da91a97510..923a79299cab03 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -534,6 +534,17 @@ visit_decref(PyObject *op, void *parent) return 0; } +int +_PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg) +{ + _PyStackRef *ref = _PyFrame_GetLocalsArray(frame); + /* locals and stack */ + for (; ref < frame->stackpointer; ref++) { + Py_VISIT(PyStackRef_AsPyObjectBorrow(*ref)); + } + return 0; +} + /* Subtract internal references from gc_refs. After this, gc_refs is >= 0 * for all objects in containers, and is GC_REACHABLE for all tracked gc * objects not in containers. The ones with gc_refs > 0 are directly diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 543bee24652dc9..b95456519dca06 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -164,15 +164,31 @@ gc_decref(PyObject *op) static void disable_deferred_refcounting(PyObject *op) { - if (_PyObject_HasDeferredRefcount(op)) { - op->ob_gc_bits &= ~_PyGC_BITS_DEFERRED; - op->ob_ref_shared -= _Py_REF_SHARED(_Py_REF_DEFERRED, 0); - - if (PyType_Check(op)) { - // Disable thread-local refcounting for heap types - PyTypeObject *type = (PyTypeObject *)op; - if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { - _PyType_ReleaseId((PyHeapTypeObject *)op); + if (!_PyObject_HasDeferredRefcount(op)) { + return; + } + + op->ob_gc_bits &= ~_PyGC_BITS_DEFERRED; + op->ob_ref_shared -= _Py_REF_SHARED(_Py_REF_DEFERRED, 0); + + if (PyType_Check(op)) { + // Disable thread-local refcounting for heap types + PyTypeObject *type = (PyTypeObject *)op; + if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + _PyType_ReleaseId((PyHeapTypeObject *)op); + } + } + else if (PyGen_CheckExact(op) || PyCoro_CheckExact(op) || PyAsyncGen_CheckExact(op)) { + // Ensure any non-refcounted pointers in locals are converted to + // strong references. This ensures that the generator/coroutine is not + // freed before its locals. + PyGenObject *gen = (PyGenObject *)op; + struct _PyInterpreterFrame *frame = &gen->gi_iframe; + assert(frame->stackpointer != NULL); + for (_PyStackRef *ref = frame->localsplus; ref < frame->stackpointer; ref++) { + if (!PyStackRef_IsNull(*ref) && PyStackRef_IsDeferred(*ref)) { + // Convert a deferred reference to a strong reference. + *ref = PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(*ref)); } } } @@ -313,6 +329,41 @@ gc_visit_heaps(PyInterpreterState *interp, mi_block_visit_fun *visitor, return err; } +static inline void +gc_visit_stackref(_PyStackRef stackref) +{ + // Note: we MUST check that it is deferred before checking the rest. + // Otherwise we might read into invalid memory due to non-deferred references + // being dead already. + if (PyStackRef_IsDeferred(stackref) && !PyStackRef_IsNull(stackref)) { + PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref); + if (_PyObject_GC_IS_TRACKED(obj)) { + gc_add_refs(obj, 1); + } + } +} + +// Add 1 to the gc_refs for every deferred reference on each thread's stack. +static void +gc_visit_thread_stacks(PyInterpreterState *interp) +{ + HEAD_LOCK(&_PyRuntime); + for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { + _PyInterpreterFrame *f = p->current_frame; + while (f != NULL) { + if (f->f_executable != NULL && PyCode_Check(f->f_executable)) { + PyCodeObject *co = (PyCodeObject *)f->f_executable; + int max_stack = co->co_nlocalsplus + co->co_stacksize; + for (int i = 0; i < max_stack; i++) { + gc_visit_stackref(f->localsplus[i]); + } + } + f = f->previous; + } + } + HEAD_UNLOCK(&_PyRuntime); +} + static void merge_queued_objects(_PyThreadStateImpl *tstate, struct collection_state *state) { @@ -617,6 +668,9 @@ deduce_unreachable_heap(PyInterpreterState *interp, gc_visit_heaps(interp, &validate_gc_objects, &state->base); #endif + // Visit the thread stacks to account for any deferred references. + gc_visit_thread_stacks(interp); + // Transitively mark reachable objects by clearing the // _PyGC_BITS_UNREACHABLE flag. if (gc_visit_heaps(interp, &mark_heap_visitor, &state->base) < 0) { @@ -897,6 +951,24 @@ visit_decref_unreachable(PyObject *op, void *data) return 0; } +int +_PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg) +{ + _PyStackRef *ref = _PyFrame_GetLocalsArray(frame); + /* locals and stack */ + for (; ref < frame->stackpointer; ref++) { + // This is a bit tricky! We want to ignore deferred references when + // computing the incoming references, but otherwise treat them like + // regular references. + if (PyStackRef_IsDeferred(*ref) && + (visit == visit_decref || visit == visit_decref_unreachable)) { + continue; + } + Py_VISIT(PyStackRef_AsPyObjectBorrow(*ref)); + } + return 0; +} + // Handle objects that may have resurrected after a call to 'finalize_garbage'. static int handle_resurrected_objects(struct collection_state *state) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 3a41c640fd9599..27faf723745c21 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -104,7 +104,7 @@ _PyRuntimeState _PyRuntime #if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) __attribute__ ((section (".PyRuntime"))) #endif -= _PyRuntimeState_INIT(_PyRuntime); += _PyRuntimeState_INIT(_PyRuntime, _Py_Debug_Cookie); _Py_COMP_DIAG_POP static int runtime_initialized = 0; diff --git a/Python/pystate.c b/Python/pystate.c index bba88b76088e71..4d7bec65ff5c49 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -390,7 +390,7 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS Note that we initialize "initial" relative to _PyRuntime, to ensure pre-initialized pointers point to the active runtime state (and not "initial"). */ -static const _PyRuntimeState initial = _PyRuntimeState_INIT(_PyRuntime); +static const _PyRuntimeState initial = _PyRuntimeState_INIT(_PyRuntime, ""); _Py_COMP_DIAG_POP #define LOCKS_INIT(runtime) \ @@ -455,6 +455,8 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) // Py_Initialize() must be running again. // Reset to _PyRuntimeState_INIT. memcpy(runtime, &initial, sizeof(*runtime)); + // Preserve the cookie from the original runtime. + memcpy(runtime->debug_offsets.cookie, _Py_Debug_Cookie, 8); assert(!runtime->_initialized); } diff --git a/Tools/build/.warningignore_macos b/Tools/build/.warningignore_macos index 1b504dfc54000f..67f50119db7310 100644 --- a/Tools/build/.warningignore_macos +++ b/Tools/build/.warningignore_macos @@ -1,3 +1,5 @@ # Files listed will be ignored by the compiler warning checker # for the macOS/build and test job. # Keep lines sorted lexicographically to help avoid merge conflicts. +# Format example: +# /path/to/file (number of warnings in file) diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu index 8242c8d17c89fb..469c727abfb11c 100644 --- a/Tools/build/.warningignore_ubuntu +++ b/Tools/build/.warningignore_ubuntu @@ -1,3 +1,5 @@ # Files listed will be ignored by the compiler warning checker # for the Ubuntu/build and test job. # Keep lines sorted lexicographically to help avoid merge conflicts. +# Format example: +# /path/to/file (number of warnings in file) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 31258932dbd4ca..1ed83447b6b668 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -9,6 +9,11 @@ import re import sys from pathlib import Path +from typing import NamedTuple + +class FileWarnings(NamedTuple): + name: str + count: int def extract_warnings_from_compiler_output_clang( @@ -19,7 +24,8 @@ def extract_warnings_from_compiler_output_clang( """ # Regex to find warnings in the compiler output clang_warning_regex = re.compile( - r"(?P.*):(?P\d+):(?P\d+): warning: (?P.*)" + r"(?P.*):(?P\d+):(?P\d+): warning: " + r"(?P.*) (?P