diff --git a/Include/Python.h b/Include/Python.h index e1abdd16f031fb..717e27feab62db 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -124,6 +124,7 @@ #include "pylifecycle.h" #include "ceval.h" #include "sysmodule.h" +#include "audit.h" #include "osmodule.h" #include "intrcheck.h" #include "import.h" diff --git a/Include/audit.h b/Include/audit.h new file mode 100644 index 00000000000000..793b7077e1027b --- /dev/null +++ b/Include/audit.h @@ -0,0 +1,30 @@ +#ifndef Py_AUDIT_H +#define Py_AUDIT_H +#ifdef __cplusplus +extern "C" { +#endif + + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(int) PySys_Audit( + const char *event, + const char *argFormat, + ...); + +PyAPI_FUNC(int) PySys_AuditTuple( + const char *event, + PyObject *args); +#endif + + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_AUDIT_H +# include "cpython/audit.h" +# undef Py_CPYTHON_AUDIT_H +#endif + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_AUDIT_H */ diff --git a/Include/cpython/audit.h b/Include/cpython/audit.h new file mode 100644 index 00000000000000..3c5c7a8c06091d --- /dev/null +++ b/Include/cpython/audit.h @@ -0,0 +1,8 @@ +#ifndef Py_CPYTHON_AUDIT_H +# error "this header file must not be included directly" +#endif + + +typedef int(*Py_AuditHookFunction)(const char *, PyObject *, void *); + +PyAPI_FUNC(int) PySys_AddAuditHook(Py_AuditHookFunction, void*); diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h index 78f7405661662f..ca8109e3248a8d 100644 --- a/Include/cpython/ceval.h +++ b/Include/cpython/ceval.h @@ -23,3 +23,21 @@ _PyEval_RequestCodeExtraIndex(freefunc f) { PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *); PyAPI_FUNC(int) _PyEval_SliceIndexNotNone(PyObject *, Py_ssize_t *); + + +// Trampoline API + +typedef struct { + FILE* perf_map; + PyThread_type_lock map_lock; +} PerfMapState; + +PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void); +PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry( + const void *code_addr, + unsigned int code_size, + const char *entry_name); +PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void); +PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename); +PyAPI_FUNC(int) PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *); +PyAPI_FUNC(int) PyUnstable_PerfTrampoline_SetPersistAfterFork(int enable); diff --git a/Include/cpython/sysmodule.h b/Include/cpython/sysmodule.h deleted file mode 100644 index a3ac07f538a94f..00000000000000 --- a/Include/cpython/sysmodule.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef Py_CPYTHON_SYSMODULE_H -# error "this header file must not be included directly" -#endif - -typedef int(*Py_AuditHookFunction)(const char *, PyObject *, void *); - -PyAPI_FUNC(int) PySys_AddAuditHook(Py_AuditHookFunction, void*); - -typedef struct { - FILE* perf_map; - PyThread_type_lock map_lock; -} PerfMapState; - -PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void); -PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry( - const void *code_addr, - unsigned int code_size, - const char *entry_name); -PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void); -PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename); -PyAPI_FUNC(int) PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *); -PyAPI_FUNC(int) PyUnstable_PerfTrampoline_SetPersistAfterFork(int enable); diff --git a/Include/internal/pycore_audit.h b/Include/internal/pycore_audit.h new file mode 100644 index 00000000000000..2811aaa6236123 --- /dev/null +++ b/Include/internal/pycore_audit.h @@ -0,0 +1,35 @@ +#ifndef Py_INTERNAL_AUDIT_H +#define Py_INTERNAL_AUDIT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +/* Runtime audit hook state */ + +typedef struct _Py_AuditHookEntry { + struct _Py_AuditHookEntry *next; + Py_AuditHookFunction hookCFunction; + void *userData; +} _Py_AuditHookEntry; + + +extern int _PySys_Audit( + PyThreadState *tstate, + const char *event, + const char *argFormat, + ...); + +// _PySys_ClearAuditHooks() must not be exported: use extern rather than +// PyAPI_FUNC(). We want minimal exposure of this function. +extern void _PySys_ClearAuditHooks(PyThreadState *tstate); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_AUDIT_H */ diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h new file mode 100644 index 00000000000000..184f4b9360b6d3 --- /dev/null +++ b/Include/internal/pycore_debug_offsets.h @@ -0,0 +1,269 @@ +#ifndef Py_INTERNAL_DEBUG_OFFSETS_H +#define Py_INTERNAL_DEBUG_OFFSETS_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +#define _Py_Debug_Cookie "xdebugpy" + +#ifdef Py_GIL_DISABLED +# define _Py_Debug_gilruntimestate_enabled offsetof(struct _gil_runtime_state, enabled) +# define _Py_Debug_Free_Threaded 1 +#else +# define _Py_Debug_gilruntimestate_enabled 0 +# define _Py_Debug_Free_Threaded 0 +#endif + + +typedef struct _Py_DebugOffsets { + char cookie[8]; + uint64_t version; + uint64_t free_threaded; + // Runtime state offset; + struct _runtime_state { + uint64_t size; + uint64_t finalizing; + uint64_t interpreters_head; + } runtime_state; + + // Interpreter state offset; + struct _interpreter_state { + uint64_t size; + uint64_t id; + uint64_t next; + uint64_t threads_head; + uint64_t gc; + uint64_t imports_modules; + uint64_t sysdict; + uint64_t builtins; + uint64_t ceval_gil; + uint64_t gil_runtime_state; + uint64_t gil_runtime_state_enabled; + uint64_t gil_runtime_state_locked; + uint64_t gil_runtime_state_holder; + } interpreter_state; + + // Thread state offset; + struct _thread_state{ + uint64_t size; + uint64_t prev; + uint64_t next; + uint64_t interp; + uint64_t current_frame; + uint64_t thread_id; + uint64_t native_thread_id; + uint64_t datastack_chunk; + uint64_t status; + } thread_state; + + // InterpreterFrame offset; + struct _interpreter_frame { + uint64_t size; + uint64_t previous; + uint64_t executable; + uint64_t instr_ptr; + uint64_t localsplus; + uint64_t owner; + } interpreter_frame; + + // Code object offset; + struct _code_object { + uint64_t size; + uint64_t filename; + uint64_t name; + uint64_t qualname; + uint64_t linetable; + uint64_t firstlineno; + uint64_t argcount; + uint64_t localsplusnames; + uint64_t localspluskinds; + uint64_t co_code_adaptive; + } code_object; + + // PyObject offset; + struct _pyobject { + uint64_t size; + uint64_t ob_type; + } pyobject; + + // PyTypeObject object offset; + struct _type_object { + uint64_t size; + uint64_t tp_name; + uint64_t tp_repr; + uint64_t tp_flags; + } type_object; + + // PyTuple object offset; + struct _tuple_object { + uint64_t size; + uint64_t ob_item; + uint64_t ob_size; + } tuple_object; + + // PyList object offset; + struct _list_object { + uint64_t size; + uint64_t ob_item; + uint64_t ob_size; + } list_object; + + // PyDict object offset; + struct _dict_object { + uint64_t size; + uint64_t ma_keys; + uint64_t ma_values; + } dict_object; + + // PyFloat object offset; + struct _float_object { + uint64_t size; + uint64_t ob_fval; + } float_object; + + // PyLong object offset; + struct _long_object { + uint64_t size; + uint64_t lv_tag; + uint64_t ob_digit; + } long_object; + + // PyBytes object offset; + struct _bytes_object { + uint64_t size; + uint64_t ob_size; + uint64_t ob_sval; + } bytes_object; + + // Unicode object offset; + struct _unicode_object { + uint64_t size; + uint64_t state; + uint64_t length; + uint64_t asciiobject_size; + } unicode_object; + + // GC runtime state offset; + struct _gc { + uint64_t size; + uint64_t collecting; + } gc; +} _Py_DebugOffsets; + + +#define _Py_DebugOffsets_INIT(debug_cookie) { \ + .cookie = debug_cookie, \ + .version = PY_VERSION_HEX, \ + .free_threaded = _Py_Debug_Free_Threaded, \ + .runtime_state = { \ + .size = sizeof(_PyRuntimeState), \ + .finalizing = offsetof(_PyRuntimeState, _finalizing), \ + .interpreters_head = offsetof(_PyRuntimeState, interpreters.head), \ + }, \ + .interpreter_state = { \ + .size = sizeof(PyInterpreterState), \ + .id = offsetof(PyInterpreterState, id), \ + .next = offsetof(PyInterpreterState, next), \ + .threads_head = offsetof(PyInterpreterState, threads.head), \ + .gc = offsetof(PyInterpreterState, gc), \ + .imports_modules = offsetof(PyInterpreterState, imports.modules), \ + .sysdict = offsetof(PyInterpreterState, sysdict), \ + .builtins = offsetof(PyInterpreterState, builtins), \ + .ceval_gil = offsetof(PyInterpreterState, ceval.gil), \ + .gil_runtime_state = offsetof(PyInterpreterState, _gil), \ + .gil_runtime_state_enabled = _Py_Debug_gilruntimestate_enabled, \ + .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \ + .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \ + }, \ + .thread_state = { \ + .size = sizeof(PyThreadState), \ + .prev = offsetof(PyThreadState, prev), \ + .next = offsetof(PyThreadState, next), \ + .interp = offsetof(PyThreadState, interp), \ + .current_frame = offsetof(PyThreadState, current_frame), \ + .thread_id = offsetof(PyThreadState, thread_id), \ + .native_thread_id = offsetof(PyThreadState, native_thread_id), \ + .datastack_chunk = offsetof(PyThreadState, datastack_chunk), \ + .status = offsetof(PyThreadState, _status), \ + }, \ + .interpreter_frame = { \ + .size = sizeof(_PyInterpreterFrame), \ + .previous = offsetof(_PyInterpreterFrame, previous), \ + .executable = offsetof(_PyInterpreterFrame, f_executable), \ + .instr_ptr = offsetof(_PyInterpreterFrame, instr_ptr), \ + .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ + .owner = offsetof(_PyInterpreterFrame, owner), \ + }, \ + .code_object = { \ + .size = sizeof(PyCodeObject), \ + .filename = offsetof(PyCodeObject, co_filename), \ + .name = offsetof(PyCodeObject, co_name), \ + .qualname = offsetof(PyCodeObject, co_qualname), \ + .linetable = offsetof(PyCodeObject, co_linetable), \ + .firstlineno = offsetof(PyCodeObject, co_firstlineno), \ + .argcount = offsetof(PyCodeObject, co_argcount), \ + .localsplusnames = offsetof(PyCodeObject, co_localsplusnames), \ + .localspluskinds = offsetof(PyCodeObject, co_localspluskinds), \ + .co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \ + }, \ + .pyobject = { \ + .size = sizeof(PyObject), \ + .ob_type = offsetof(PyObject, ob_type), \ + }, \ + .type_object = { \ + .size = sizeof(PyTypeObject), \ + .tp_name = offsetof(PyTypeObject, tp_name), \ + .tp_repr = offsetof(PyTypeObject, tp_repr), \ + .tp_flags = offsetof(PyTypeObject, tp_flags), \ + }, \ + .tuple_object = { \ + .size = sizeof(PyTupleObject), \ + .ob_item = offsetof(PyTupleObject, ob_item), \ + .ob_size = offsetof(PyTupleObject, ob_base.ob_size), \ + }, \ + .list_object = { \ + .size = sizeof(PyListObject), \ + .ob_item = offsetof(PyListObject, ob_item), \ + .ob_size = offsetof(PyListObject, ob_base.ob_size), \ + }, \ + .dict_object = { \ + .size = sizeof(PyDictObject), \ + .ma_keys = offsetof(PyDictObject, ma_keys), \ + .ma_values = offsetof(PyDictObject, ma_values), \ + }, \ + .float_object = { \ + .size = sizeof(PyFloatObject), \ + .ob_fval = offsetof(PyFloatObject, ob_fval), \ + }, \ + .long_object = { \ + .size = sizeof(PyLongObject), \ + .lv_tag = offsetof(PyLongObject, long_value.lv_tag), \ + .ob_digit = offsetof(PyLongObject, long_value.ob_digit), \ + }, \ + .bytes_object = { \ + .size = sizeof(PyBytesObject), \ + .ob_size = offsetof(PyBytesObject, ob_base.ob_size), \ + .ob_sval = offsetof(PyBytesObject, ob_sval), \ + }, \ + .unicode_object = { \ + .size = sizeof(PyUnicodeObject), \ + .state = offsetof(PyUnicodeObject, _base._base.state), \ + .length = offsetof(PyUnicodeObject, _base._base.length), \ + .asciiobject_size = sizeof(PyASCIIObject), \ + }, \ + .gc = { \ + .size = sizeof(struct _gc_runtime_state), \ + .collecting = offsetof(struct _gc_runtime_state, collecting), \ + }, \ +} + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_DEBUG_OFFSETS_H */ diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index ad1a7d7e120519..96f6d61e1c620b 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -208,6 +208,11 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) #ifdef Py_TRACE_REFS _Py_ForgetReference(op); #endif + struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; + if (tracer->tracer_func != NULL) { + void* data = tracer->tracer_data; + tracer->tracer_func(op, PyRefTracer_DESTROY, data); + } destruct(op); } } diff --git a/Include/internal/pycore_object_state.h b/Include/internal/pycore_object_state.h index e7fa7c1f10d6d1..8a47a6d9e6eb0d 100644 --- a/Include/internal/pycore_object_state.h +++ b/Include/internal/pycore_object_state.h @@ -11,6 +11,14 @@ extern "C" { #include "pycore_freelist_state.h" // _Py_freelists #include "pycore_hashtable.h" // _Py_hashtable_t + +/* Reference tracer state */ +struct _reftracer_runtime_state { + PyRefTracer tracer_func; + void* tracer_data; +}; + + struct _py_object_runtime_state { #ifdef Py_REF_DEBUG Py_ssize_t interpreter_leaks; diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index d4291b87261ae0..7f592aa6cf9f05 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -9,8 +9,10 @@ extern "C" { #endif #include "pycore_atexit.h" // struct _atexit_runtime_state +#include "pycore_audit.h" // _Py_AuditHookEntry #include "pycore_ceval_state.h" // struct _ceval_runtime_state -#include "pycore_crossinterp.h" // struct _xidregistry +#include "pycore_crossinterp.h" // struct _xidregistry +#include "pycore_debug_offsets.h" // _Py_DebugOffsets #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_import.h" // struct _import_runtime_state @@ -25,185 +27,12 @@ extern "C" { #include "pycore_typeobject.h" // struct _types_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_state -struct _getargs_runtime_state { - struct _PyArg_Parser *static_parsers; -}; - -/* GIL state */ - -struct _gilstate_runtime_state { - /* bpo-26558: Flag to disable PyGILState_Check(). - If set to non-zero, PyGILState_Check() always return 1. */ - int check_enabled; - /* The single PyInterpreterState used by this process' - GILState implementation - */ - /* TODO: Given interp_main, it may be possible to kill this ref */ - PyInterpreterState *autoInterpreterState; -}; - -/* Runtime audit hook state */ - -#define _Py_Debug_Cookie "xdebugpy" - -#ifdef Py_GIL_DISABLED -# define _Py_Debug_gilruntimestate_enabled offsetof(struct _gil_runtime_state, enabled) -# define _Py_Debug_Free_Threaded 1 -#else -# define _Py_Debug_gilruntimestate_enabled 0 -# define _Py_Debug_Free_Threaded 0 -#endif -typedef struct _Py_AuditHookEntry { - struct _Py_AuditHookEntry *next; - Py_AuditHookFunction hookCFunction; - void *userData; -} _Py_AuditHookEntry; - -typedef struct _Py_DebugOffsets { - char cookie[8]; - uint64_t version; - uint64_t free_threaded; - // Runtime state offset; - struct _runtime_state { - uint64_t size; - uint64_t finalizing; - uint64_t interpreters_head; - } runtime_state; - - // Interpreter state offset; - struct _interpreter_state { - uint64_t size; - uint64_t id; - uint64_t next; - uint64_t threads_head; - uint64_t gc; - uint64_t imports_modules; - uint64_t sysdict; - uint64_t builtins; - uint64_t ceval_gil; - uint64_t gil_runtime_state; - uint64_t gil_runtime_state_enabled; - uint64_t gil_runtime_state_locked; - uint64_t gil_runtime_state_holder; - } interpreter_state; - - // Thread state offset; - struct _thread_state{ - uint64_t size; - uint64_t prev; - uint64_t next; - uint64_t interp; - uint64_t current_frame; - uint64_t thread_id; - uint64_t native_thread_id; - uint64_t datastack_chunk; - uint64_t status; - } thread_state; - - // InterpreterFrame offset; - struct _interpreter_frame { - uint64_t size; - uint64_t previous; - uint64_t executable; - uint64_t instr_ptr; - uint64_t localsplus; - uint64_t owner; - } interpreter_frame; - - // Code object offset; - struct _code_object { - uint64_t size; - uint64_t filename; - uint64_t name; - uint64_t qualname; - uint64_t linetable; - uint64_t firstlineno; - uint64_t argcount; - uint64_t localsplusnames; - uint64_t localspluskinds; - uint64_t co_code_adaptive; - } code_object; - - // PyObject offset; - struct _pyobject { - uint64_t size; - uint64_t ob_type; - } pyobject; - - // PyTypeObject object offset; - struct _type_object { - uint64_t size; - uint64_t tp_name; - uint64_t tp_repr; - uint64_t tp_flags; - } type_object; - - // PyTuple object offset; - struct _tuple_object { - uint64_t size; - uint64_t ob_item; - uint64_t ob_size; - } tuple_object; - - // PyList object offset; - struct _list_object { - uint64_t size; - uint64_t ob_item; - uint64_t ob_size; - } list_object; - - // PyDict object offset; - struct _dict_object { - uint64_t size; - uint64_t ma_keys; - uint64_t ma_values; - } dict_object; - - // PyFloat object offset; - struct _float_object { - uint64_t size; - uint64_t ob_fval; - } float_object; - - // PyLong object offset; - struct _long_object { - uint64_t size; - uint64_t lv_tag; - uint64_t ob_digit; - } long_object; - - // PyBytes object offset; - struct _bytes_object { - uint64_t size; - uint64_t ob_size; - uint64_t ob_sval; - } bytes_object; - - // Unicode object offset; - struct _unicode_object { - uint64_t size; - uint64_t state; - uint64_t length; - uint64_t asciiobject_size; - } unicode_object; - - // GC runtime state offset; - struct _gc { - uint64_t size; - uint64_t collecting; - } gc; -} _Py_DebugOffsets; - -/* Reference tracer state */ -struct _reftracer_runtime_state { - PyRefTracer tracer_func; - void* tracer_data; -}; /* Full Python runtime state */ /* _PyRuntimeState holds the global state for the CPython runtime. - That data is exposed in the internal API as a static variable (_PyRuntime). + That data is exported by the internal API as a global variable + (_PyRuntime, defined near the top of pylifecycle.c). */ typedef struct pyruntimestate { /* This field must be first to facilitate locating it by out of process @@ -299,8 +128,19 @@ typedef struct pyruntimestate { struct _import_runtime_state imports; struct _ceval_runtime_state ceval; - struct _gilstate_runtime_state gilstate; - struct _getargs_runtime_state getargs; + struct _gilstate_runtime_state { + /* bpo-26558: Flag to disable PyGILState_Check(). + If set to non-zero, PyGILState_Check() always return 1. */ + int check_enabled; + /* The single PyInterpreterState used by this process' + GILState implementation + */ + /* TODO: Given interp_main, it may be possible to kill this ref */ + PyInterpreterState *autoInterpreterState; + } gilstate; + struct _getargs_runtime_state { + struct _PyArg_Parser *static_parsers; + } getargs; struct _fileutils_state fileutils; struct _faulthandler_runtime_state faulthandler; struct _tracemalloc_runtime_state tracemalloc; @@ -404,6 +244,7 @@ _PyRuntimeState_SetFinalizing(_PyRuntimeState *runtime, PyThreadState *tstate) { } } + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index a17ba46966daa1..e99febab2f3d57 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_ceval_state.h" // _PyEval_RUNTIME_PERF_INIT +#include "pycore_debug_offsets.h" // _Py_DebugOffsets_INIT() #include "pycore_faulthandler.h" // _faulthandler_runtime_state_INIT #include "pycore_floatobject.h" // _py_float_format_unknown #include "pycore_function.h" @@ -32,111 +33,7 @@ extern PyTypeObject _PyExc_MemoryError; #define _PyRuntimeState_INIT(runtime, debug_cookie) \ { \ - .debug_offsets = { \ - .cookie = debug_cookie, \ - .version = PY_VERSION_HEX, \ - .free_threaded = _Py_Debug_Free_Threaded, \ - .runtime_state = { \ - .size = sizeof(_PyRuntimeState), \ - .finalizing = offsetof(_PyRuntimeState, _finalizing), \ - .interpreters_head = offsetof(_PyRuntimeState, interpreters.head), \ - }, \ - .interpreter_state = { \ - .size = sizeof(PyInterpreterState), \ - .id = offsetof(PyInterpreterState, id), \ - .next = offsetof(PyInterpreterState, next), \ - .threads_head = offsetof(PyInterpreterState, threads.head), \ - .gc = offsetof(PyInterpreterState, gc), \ - .imports_modules = offsetof(PyInterpreterState, imports.modules), \ - .sysdict = offsetof(PyInterpreterState, sysdict), \ - .builtins = offsetof(PyInterpreterState, builtins), \ - .ceval_gil = offsetof(PyInterpreterState, ceval.gil), \ - .gil_runtime_state = offsetof(PyInterpreterState, _gil), \ - .gil_runtime_state_enabled = _Py_Debug_gilruntimestate_enabled, \ - .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \ - .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \ - }, \ - .thread_state = { \ - .size = sizeof(PyThreadState), \ - .prev = offsetof(PyThreadState, prev), \ - .next = offsetof(PyThreadState, next), \ - .interp = offsetof(PyThreadState, interp), \ - .current_frame = offsetof(PyThreadState, current_frame), \ - .thread_id = offsetof(PyThreadState, thread_id), \ - .native_thread_id = offsetof(PyThreadState, native_thread_id), \ - .datastack_chunk = offsetof(PyThreadState, datastack_chunk), \ - .status = offsetof(PyThreadState, _status), \ - }, \ - .interpreter_frame = { \ - .size = sizeof(_PyInterpreterFrame), \ - .previous = offsetof(_PyInterpreterFrame, previous), \ - .executable = offsetof(_PyInterpreterFrame, f_executable), \ - .instr_ptr = offsetof(_PyInterpreterFrame, instr_ptr), \ - .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ - .owner = offsetof(_PyInterpreterFrame, owner), \ - }, \ - .code_object = { \ - .size = sizeof(PyCodeObject), \ - .filename = offsetof(PyCodeObject, co_filename), \ - .name = offsetof(PyCodeObject, co_name), \ - .qualname = offsetof(PyCodeObject, co_qualname), \ - .linetable = offsetof(PyCodeObject, co_linetable), \ - .firstlineno = offsetof(PyCodeObject, co_firstlineno), \ - .argcount = offsetof(PyCodeObject, co_argcount), \ - .localsplusnames = offsetof(PyCodeObject, co_localsplusnames), \ - .localspluskinds = offsetof(PyCodeObject, co_localspluskinds), \ - .co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \ - }, \ - .pyobject = { \ - .size = sizeof(PyObject), \ - .ob_type = offsetof(PyObject, ob_type), \ - }, \ - .type_object = { \ - .size = sizeof(PyTypeObject), \ - .tp_name = offsetof(PyTypeObject, tp_name), \ - .tp_repr = offsetof(PyTypeObject, tp_repr), \ - .tp_flags = offsetof(PyTypeObject, tp_flags), \ - }, \ - .tuple_object = { \ - .size = sizeof(PyTupleObject), \ - .ob_item = offsetof(PyTupleObject, ob_item), \ - .ob_size = offsetof(PyTupleObject, ob_base.ob_size), \ - }, \ - .list_object = { \ - .size = sizeof(PyListObject), \ - .ob_item = offsetof(PyListObject, ob_item), \ - .ob_size = offsetof(PyListObject, ob_base.ob_size), \ - }, \ - .dict_object = { \ - .size = sizeof(PyDictObject), \ - .ma_keys = offsetof(PyDictObject, ma_keys), \ - .ma_values = offsetof(PyDictObject, ma_values), \ - }, \ - .float_object = { \ - .size = sizeof(PyFloatObject), \ - .ob_fval = offsetof(PyFloatObject, ob_fval), \ - }, \ - .long_object = { \ - .size = sizeof(PyLongObject), \ - .lv_tag = offsetof(PyLongObject, long_value.lv_tag), \ - .ob_digit = offsetof(PyLongObject, long_value.ob_digit), \ - }, \ - .bytes_object = { \ - .size = sizeof(PyBytesObject), \ - .ob_size = offsetof(PyBytesObject, ob_base.ob_size), \ - .ob_sval = offsetof(PyBytesObject, ob_sval), \ - }, \ - .unicode_object = { \ - .size = sizeof(PyUnicodeObject), \ - .state = offsetof(PyUnicodeObject, _base._base.state), \ - .length = offsetof(PyUnicodeObject, _base._base.length), \ - .asciiobject_size = sizeof(PyASCIIObject), \ - }, \ - .gc = { \ - .size = sizeof(struct _gc_runtime_state), \ - .collecting = offsetof(struct _gc_runtime_state, collecting), \ - }, \ - }, \ + .debug_offsets = _Py_DebugOffsets_INIT(debug_cookie), \ .allocators = { \ .standard = _pymem_allocators_standard_INIT(runtime), \ .debug = _pymem_allocators_debug_INIT, \ diff --git a/Include/internal/pycore_sysmodule.h b/Include/internal/pycore_sysmodule.h index a1d795e284f6ac..99968df54a45f6 100644 --- a/Include/internal/pycore_sysmodule.h +++ b/Include/internal/pycore_sysmodule.h @@ -14,16 +14,6 @@ PyAPI_FUNC(PyObject*) _PySys_GetAttr(PyThreadState *tstate, PyObject *name); // Export for '_pickle' shared extension PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *); -extern int _PySys_Audit( - PyThreadState *tstate, - const char *event, - const char *argFormat, - ...); - -// _PySys_ClearAuditHooks() must not be exported: use extern rather than -// PyAPI_FUNC(). We want minimal exposure of this function. -extern void _PySys_ClearAuditHooks(PyThreadState *tstate); - extern int _PySys_SetAttr(PyObject *, PyObject *); extern int _PySys_ClearAttrString(PyInterpreterState *interp, diff --git a/Include/sysmodule.h b/Include/sysmodule.h index 5a0af2e1578eb7..c1d5f610fe08a5 100644 --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -21,23 +21,6 @@ Py_DEPRECATED(3.13) PyAPI_FUNC(void) PySys_ResetWarnOptions(void); PyAPI_FUNC(PyObject *) PySys_GetXOptions(void); -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 -PyAPI_FUNC(int) PySys_Audit( - const char *event, - const char *argFormat, - ...); - -PyAPI_FUNC(int) PySys_AuditTuple( - const char *event, - PyObject *args); -#endif - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_SYSMODULE_H -# include "cpython/sysmodule.h" -# undef Py_CPYTHON_SYSMODULE_H -#endif - #ifdef __cplusplus } #endif diff --git a/Lib/ctypes/wintypes.py b/Lib/ctypes/wintypes.py index 9c4e721438aad5..4beba0d19513e2 100644 --- a/Lib/ctypes/wintypes.py +++ b/Lib/ctypes/wintypes.py @@ -63,10 +63,16 @@ def __repr__(self): HBITMAP = HANDLE HBRUSH = HANDLE HCOLORSPACE = HANDLE +HCONV = HANDLE +HCONVLIST = HANDLE +HCURSOR = HANDLE HDC = HANDLE +HDDEDATA = HANDLE HDESK = HANDLE +HDROP = HANDLE HDWP = HANDLE HENHMETAFILE = HANDLE +HFILE = INT HFONT = HANDLE HGDIOBJ = HANDLE HGLOBAL = HANDLE @@ -82,9 +88,11 @@ def __repr__(self): HMONITOR = HANDLE HPALETTE = HANDLE HPEN = HANDLE +HRESULT = LONG HRGN = HANDLE HRSRC = HANDLE HSTR = HANDLE +HSZ = HANDLE HTASK = HANDLE HWINSTA = HANDLE HWND = HANDLE diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py index d69a45d6793069..ff4bfcdcc407b9 100644 --- a/Lib/json/decoder.py +++ b/Lib/json/decoder.py @@ -50,17 +50,18 @@ def __reduce__(self): } +HEXDIGITS = re.compile(r'[0-9A-Fa-f]{4}', FLAGS) STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) BACKSLASH = { '"': '"', '\\': '\\', '/': '/', 'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t', } -def _decode_uXXXX(s, pos): - esc = s[pos + 1:pos + 5] - if len(esc) == 4 and esc[1] not in 'xX': +def _decode_uXXXX(s, pos, _m=HEXDIGITS.match): + esc = _m(s, pos + 1) + if esc is not None: try: - return int(esc, 16) + return int(esc.group(), 16) except ValueError: pass msg = "Invalid \\uXXXX escape" diff --git a/Lib/json/scanner.py b/Lib/json/scanner.py index 7a61cfc2d24dce..090897515fe2f3 100644 --- a/Lib/json/scanner.py +++ b/Lib/json/scanner.py @@ -9,7 +9,7 @@ __all__ = ['make_scanner'] NUMBER_RE = re.compile( - r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?', + r'(-?(?:0|[1-9][0-9]*))(\.[0-9]+)?([eE][-+]?[0-9]+)?', (re.VERBOSE | re.MULTILINE | re.DOTALL)) def py_make_scanner(context): diff --git a/Lib/pdb.py b/Lib/pdb.py index 3e5e6088fdcc7e..cd7a7042fa6987 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -402,6 +402,8 @@ def setup(self, f, tb): self.curframe = self.stack[self.curindex][0] self.set_convenience_variable(self.curframe, '_frame', self.curframe) + self._save_initial_file_mtime(self.curframe) + if self._chained_exceptions: self.set_convenience_variable( self.curframe, @@ -494,9 +496,21 @@ def _cmdloop(self): except KeyboardInterrupt: self.message('--KeyboardInterrupt--') + def _save_initial_file_mtime(self, frame): + """save the mtime of the all the files in the frame stack in the file mtime table + if they haven't been saved yet.""" + while frame: + filename = frame.f_code.co_filename + if filename not in self._file_mtime_table: + try: + self._file_mtime_table[filename] = os.path.getmtime(filename) + except Exception: + pass + frame = frame.f_back + def _validate_file_mtime(self): - """Check if the source file of the current frame has been modified since - the last time we saw it. If so, give a warning.""" + """Check if the source file of the current frame has been modified. + If so, give a warning and reset the modify time to current.""" try: filename = self.curframe.f_code.co_filename mtime = os.path.getmtime(filename) @@ -506,7 +520,7 @@ def _validate_file_mtime(self): mtime != self._file_mtime_table[filename]): self.message(f"*** WARNING: file '{filename}' was edited, " "running stale code until the program is rerun") - self._file_mtime_table[filename] = mtime + self._file_mtime_table[filename] = mtime # Called before loop, handles display expressions # Set up convenience variable containers @@ -836,7 +850,6 @@ def onecmd(self, line): a breakpoint command list definition. """ if not self.commands_defining: - self._validate_file_mtime() if line.startswith('_pdbcmd'): command, arg, line = self.parseline(line) if hasattr(self, command): @@ -980,6 +993,7 @@ def completedefault(self, text, line, begidx, endidx): def _pdbcmd_print_frame_status(self, arg): self.print_stack_trace(0) + self._validate_file_mtime() self._show_display() def _pdbcmd_silence_frame_status(self, arg): @@ -1861,6 +1875,7 @@ def do_list(self, arg): self.message('[EOF]') except KeyboardInterrupt: pass + self._validate_file_mtime() do_l = do_list def do_longlist(self, arg): @@ -1879,6 +1894,7 @@ def do_longlist(self, arg): self.error(err) return self._print_lines(lines, lineno, breaklist, self.curframe) + self._validate_file_mtime() do_ll = do_longlist def do_source(self, arg): diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index a059a6b8340448..065fc27b770438 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -5761,6 +5761,8 @@ def test_resource_tracker_sigterm(self): # Catchable signal (ignored by semaphore tracker) self.check_resource_tracker_death(signal.SIGTERM, False) + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-125620: Skip on NetBSD due to long wait for SIGKILL process termination.") def test_resource_tracker_sigkill(self): # Uncatchable signal. self.check_resource_tracker_death(signal.SIGKILL, True) diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index 79fb239b35d3f2..2250af964c022b 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -16,6 +16,12 @@ def test_float(self): self.assertIsInstance(rval, float) self.assertEqual(rval, 1.0) + def test_nonascii_digits_rejected(self): + # JSON specifies only ascii digits, see gh-125687 + for num in ["1\uff10", "0.\uff10", "0e\uff10"]: + with self.assertRaises(self.JSONDecodeError): + self.loads(num) + def test_bytes(self): self.assertEqual(self.loads(b"1"), 1) diff --git a/Lib/test/test_json/test_scanstring.py b/Lib/test/test_json/test_scanstring.py index 2d3ee8a8bf0f92..cca556a3b95bab 100644 --- a/Lib/test/test_json/test_scanstring.py +++ b/Lib/test/test_json/test_scanstring.py @@ -116,6 +116,11 @@ def test_bad_escapes(self): '"\\u012z"', '"\\u0x12"', '"\\u0X12"', + '"\\u{0}"'.format("\uff10" * 4), + '"\\u 123"', + '"\\u-123"', + '"\\u+123"', + '"\\u1_23"', '"\\ud834\\"', '"\\ud834\\u"', '"\\ud834\\ud"', @@ -127,6 +132,11 @@ def test_bad_escapes(self): '"\\ud834\\udd2z"', '"\\ud834\\u0x20"', '"\\ud834\\u0X20"', + '"\\ud834\\u{0}"'.format("\uff10" * 4), + '"\\ud834\\u 123"', + '"\\ud834\\u-123"', + '"\\ud834\\u+123"', + '"\\ud834\\u1_23"', ] for s in bad_escapes: with self.assertRaises(self.JSONDecodeError, msg=s): diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 8136c591a33001..7e6f276d355a14 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -3711,6 +3711,25 @@ def test_file_modified_after_execution(self): self.assertIn("WARNING:", stdout) self.assertIn("was edited", stdout) + def test_file_modified_and_immediately_restarted(self): + script = """ + print("hello") + """ + + # the time.sleep is needed for low-resolution filesystems like HFS+ + commands = """ + filename = $_frame.f_code.co_filename + f = open(filename, "w") + f.write("print('goodbye')") + import time; time.sleep(1) + f.close() + restart + """ + + stdout, stderr = self.run_pdb_script(script, commands) + self.assertNotIn("WARNING:", stdout) + self.assertNotIn("was edited", stdout) + def test_file_modified_after_execution_with_multiple_instances(self): # the time.sleep is needed for low-resolution filesystems like HFS+ script = """ diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index f8b99a9b6a63f5..d368f08b610870 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -18,7 +18,7 @@ except ImportError: _testinternalcapi = None -from test.support import skip_if_buggy_ucrt_strfptime +from test.support import skip_if_buggy_ucrt_strfptime, SuppressCrashReport # Max year is only limited by the size of C int. SIZEOF_INT = sysconfig.get_config_var('SIZEOF_INT') or 4 @@ -182,6 +182,17 @@ def test_strftime(self): self.assertRaises(TypeError, time.strftime, b'%S', tt) + def test_strftime_invalid_format(self): + tt = time.gmtime(self.t) + with SuppressCrashReport(): + for i in range(1, 128): + format = ' %' + chr(i) + with self.subTest(format=format): + try: + time.strftime(format, tt) + except ValueError as exc: + self.assertEqual(str(exc), 'Invalid format string') + def test_strftime_special(self): tt = time.gmtime(self.t) s1 = time.strftime('%c', tt) diff --git a/Makefile.pre.in b/Makefile.pre.in index 3caaf40363c12c..d7abe133e34163 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1033,6 +1033,7 @@ python.worker.js: $(srcdir)/Tools/wasm/python.worker.js PYTHON_HEADERS= \ $(srcdir)/Include/Python.h \ $(srcdir)/Include/abstract.h \ + $(srcdir)/Include/audit.h \ $(srcdir)/Include/bltinmodule.h \ $(srcdir)/Include/boolobject.h \ $(srcdir)/Include/bytearrayobject.h \ @@ -1111,6 +1112,7 @@ PYTHON_HEADERS= \ $(PARSER_HEADERS) \ \ $(srcdir)/Include/cpython/abstract.h \ + $(srcdir)/Include/cpython/audit.h \ $(srcdir)/Include/cpython/bytearrayobject.h \ $(srcdir)/Include/cpython/bytesobject.h \ $(srcdir)/Include/cpython/cellobject.h \ @@ -1160,7 +1162,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/pythonrun.h \ $(srcdir)/Include/cpython/pythread.h \ $(srcdir)/Include/cpython/setobject.h \ - $(srcdir)/Include/cpython/sysmodule.h \ $(srcdir)/Include/cpython/traceback.h \ $(srcdir)/Include/cpython/tracemalloc.h \ $(srcdir)/Include/cpython/tupleobject.h \ @@ -1175,6 +1176,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_ast.h \ $(srcdir)/Include/internal/pycore_ast_state.h \ $(srcdir)/Include/internal/pycore_atexit.h \ + $(srcdir)/Include/internal/pycore_audit.h \ $(srcdir)/Include/internal/pycore_backoff.h \ $(srcdir)/Include/internal/pycore_bitutils.h \ $(srcdir)/Include/internal/pycore_blocks_output_buffer.h \ @@ -1194,6 +1196,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_context.h \ $(srcdir)/Include/internal/pycore_critical_section.h \ $(srcdir)/Include/internal/pycore_crossinterp.h \ + $(srcdir)/Include/internal/pycore_debug_offsets.h \ $(srcdir)/Include/internal/pycore_descrobject.h \ $(srcdir)/Include/internal/pycore_dict.h \ $(srcdir)/Include/internal/pycore_dict_state.h \ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-18-16-00-10.gh-issue-125703.QRoqMo.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-18-16-00-10.gh-issue-125703.QRoqMo.rst new file mode 100644 index 00000000000000..7cbfa725e78cef --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-18-16-00-10.gh-issue-125703.QRoqMo.rst @@ -0,0 +1,2 @@ +Correctly honour :mod:`tracemalloc` hooks in specialized ``Py_DECREF`` +paths. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Library/2024-10-16-15-55-50.gh-issue-125600.yMsJx0.rst b/Misc/NEWS.d/next/Library/2024-10-16-15-55-50.gh-issue-125600.yMsJx0.rst new file mode 100644 index 00000000000000..19bf4fbefb601b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-16-15-55-50.gh-issue-125600.yMsJx0.rst @@ -0,0 +1 @@ +Only show stale code warning in :mod:`pdb` when we display source code. diff --git a/Misc/NEWS.d/next/Library/2024-10-18-08-58-10.gh-issue-125660.sDdDqO.rst b/Misc/NEWS.d/next/Library/2024-10-18-08-58-10.gh-issue-125660.sDdDqO.rst new file mode 100644 index 00000000000000..74d76c7bddae7d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-18-08-58-10.gh-issue-125660.sDdDqO.rst @@ -0,0 +1 @@ +Reject invalid unicode escapes for Python implementation of :func:`json.loads`. diff --git a/Misc/NEWS.d/next/Library/2024-10-18-09-51-29.gh-issue-125682.vsj4cU.rst b/Misc/NEWS.d/next/Library/2024-10-18-09-51-29.gh-issue-125682.vsj4cU.rst new file mode 100644 index 00000000000000..3eb2905ad8d810 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-18-09-51-29.gh-issue-125682.vsj4cU.rst @@ -0,0 +1,2 @@ +Reject non-ASCII digits in the Python implementation of :func:`json.loads` +conforming to the JSON specification. diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index 5dec240f02c4db..55c43199ee4d79 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -1518,7 +1518,7 @@ static PyObject * queuesmod_destroy(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:destroy", kwlist, qidarg_converter, &qidarg)) { return NULL; @@ -1579,7 +1579,7 @@ static PyObject * queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", "obj", "fmt", "unboundop", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; PyObject *obj; int fmt; int unboundop; @@ -1615,7 +1615,7 @@ static PyObject * queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:get", kwlist, qidarg_converter, &qidarg)) { return NULL; @@ -1651,7 +1651,7 @@ static PyObject * queuesmod_bind(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:bind", kwlist, qidarg_converter, &qidarg)) { return NULL; @@ -1681,7 +1681,7 @@ queuesmod_release(PyObject *self, PyObject *args, PyObject *kwds) { // Note that only the current interpreter is affected. static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:release", kwlist, qidarg_converter, &qidarg)) { @@ -1710,7 +1710,7 @@ static PyObject * queuesmod_get_maxsize(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:get_maxsize", kwlist, qidarg_converter, &qidarg)) { @@ -1735,7 +1735,7 @@ static PyObject * queuesmod_get_queue_defaults(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:get_queue_defaults", kwlist, qidarg_converter, &qidarg)) { @@ -1765,7 +1765,7 @@ static PyObject * queuesmod_is_full(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:is_full", kwlist, qidarg_converter, &qidarg)) { @@ -1793,7 +1793,7 @@ static PyObject * queuesmod_get_count(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:get_count", kwlist, qidarg_converter, &qidarg)) { diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 2476346777c319..0807d1e47b6736 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -51,7 +51,9 @@ # define Py_BUILD_CORE_MODULE 1 #endif #include "Python.h" -#include +#include // _Py_DebugOffsets +#include // FRAME_OWNED_BY_CSTACK +#include // Py_TAG_BITS #ifndef HAVE_PROCESS_VM_READV # define HAVE_PROCESS_VM_READV 0 diff --git a/Objects/funcobject.c b/Objects/funcobject.c index f86ef32f1827bf..3cb247691386bf 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -1264,6 +1264,7 @@ descriptor_set_wrapped_attribute(PyObject *oobj, PyObject *name, PyObject *value PyErr_Format(PyExc_AttributeError, "'%.200s' object has no attribute '%U'", type_name, name); + Py_DECREF(dict); return -1; } else { diff --git a/Objects/object.c b/Objects/object.c index 4a4c5bf7d7f08a..1a15b70d3dc63f 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -17,6 +17,7 @@ #include "pycore_memoryobject.h" // _PyManagedBuffer_Type #include "pycore_namespace.h" // _PyNamespace_Type #include "pycore_object.h" // PyAPI_DATA() _Py_SwappedOp definition +#include "pycore_object_state.h" // struct _reftracer_runtime_state #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_optimizer.h" // _PyUOpExecutor_Type, _PyUOpOptimizer_Type, ... #include "pycore_pyerrors.h" // _PyErr_Occurred() diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 9c85e5efa4af4a..c65341179376ea 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -217,11 +217,6 @@ public override bool Execute() { - - - $(VCInstallDir)\redist\ - - <_RedistFiles Include="$(VCInstallDir)\Redist\MSVC\*\*.*" /> diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 76837759172825..f840e7fd61f985 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -128,6 +128,7 @@ + @@ -137,6 +138,7 @@ + @@ -185,7 +187,6 @@ - @@ -208,6 +209,7 @@ + @@ -227,6 +229,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index d432df1af6a083..24e573ac1cc89b 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -48,6 +48,9 @@ Include + + Include + Include @@ -354,6 +357,9 @@ Include\cpython + + Include\cpython + Include\cpython @@ -513,9 +519,6 @@ Include\cpython - - Include\cpython - Include\cpython @@ -552,6 +555,9 @@ Include\internal + + Include\internal + Include\internal @@ -603,6 +609,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4c6f4302b9efca..6d2299e4489af5 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -8,6 +8,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_audit.h" // _PySys_Audit() #include "pycore_backoff.h" #include "pycore_cell.h" // PyCell_GetRef() #include "pycore_ceval.h" @@ -27,7 +28,6 @@ #include "pycore_range.h" // _PyRangeIterObject #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs -#include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_typeobject.h" // _PySuper_Lookup() diff --git a/Python/ceval.c b/Python/ceval.c index f62539af4cdfba..89f4f0344f5789 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_audit.h" // _PySys_Audit() #include "pycore_backoff.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_cell.h" // PyCell_GetRef() @@ -26,7 +27,6 @@ #include "pycore_range.h" // _PyRangeIterObject #include "pycore_setobject.h" // _PySet_Update() #include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs -#include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_typeobject.h" // _PySuper_Lookup() #include "pycore_uop_ids.h" // Uops diff --git a/Python/errors.c b/Python/errors.c index 9e2a3ce062a6fe..7f3b4aabc432d7 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -2,12 +2,13 @@ /* Error handling */ #include "Python.h" +#include "pycore_audit.h" // _PySys_Audit() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_initconfig.h" // _PyStatus_ERR() #include "pycore_pyerrors.h" // _PyErr_Format() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() -#include "pycore_sysmodule.h" // _PySys_Audit() +#include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_traceback.h" // _PyTraceBack_FromFrame() #ifdef MS_WINDOWS diff --git a/Python/import.c b/Python/import.c index acf849f14562b9..d8ad37b2422795 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1,6 +1,7 @@ /* Module definition and import implementation */ #include "Python.h" +#include "pycore_audit.h" // _PySys_Audit() #include "pycore_ceval.h" #include "pycore_hashtable.h" // _Py_hashtable_new_full() #include "pycore_import.h" // _PyImport_BootstrapImp() @@ -14,7 +15,7 @@ #include "pycore_pylifecycle.h" #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_sysmodule.h" // _PySys_Audit() +#include "pycore_sysmodule.h" // _PySys_ClearAttrString() #include "pycore_time.h" // _PyTime_AsMicroseconds() #include "pycore_weakref.h" // _PyWeakref_GET_REF() diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 1436921a19b768..45af275f1f6dce 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -3,9 +3,9 @@ */ #include "Python.h" +#include "pycore_audit.h" // _PySys_Audit() #include "pycore_ceval.h" // export _PyEval_SetProfile() #include "pycore_object.h" -#include "pycore_sysmodule.h" // _PySys_Audit() #include "opcode.h" #include diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 5fb9c4f7c719fe..b8f424854ecb86 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2,6 +2,7 @@ #include "Python.h" +#include "pycore_audit.h" // _PySys_ClearAuditHooks() #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_ceval.h" // _PyEval_FiniGIL() #include "pycore_codecs.h" // _PyCodec_Lookup() @@ -26,7 +27,7 @@ #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_sliceobject.h" // _PySlice_Fini() -#include "pycore_sysmodule.h" // _PySys_ClearAuditHooks() +#include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_traceback.h" // _Py_DumpTracebackThreads() #include "pycore_uniqueid.h" // _PyObject_FinalizeUniqueIdPool() #include "pycore_typeobject.h" // _PyTypes_InitTypes() @@ -78,6 +79,7 @@ static void wait_for_thread_shutdown(PyThreadState *tstate); static void finalize_subinterpreters(void); static void call_ll_exitfuncs(_PyRuntimeState *runtime); + /* The following places the `_PyRuntime` structure in a location that can be * found without any external information. This is meant to ease access to the * interpreter state for various runtime debugging tools, but is *not* an @@ -107,6 +109,7 @@ __attribute__ ((section (".PyRuntime"))) = _PyRuntimeState_INIT(_PyRuntime, _Py_Debug_Cookie); _Py_COMP_DIAG_POP + static int runtime_initialized = 0; PyStatus diff --git a/Python/pystate.c b/Python/pystate.c index f990a5403aabe6..f6f0ad0ee5f809 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_audit.h" // _Py_AuditHookEntry #include "pycore_ceval.h" #include "pycore_code.h" // stats #include "pycore_critical_section.h" // _PyCriticalSection_Resume() @@ -18,7 +19,6 @@ #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" #include "pycore_runtime_init.h" // _PyRuntimeState_INIT -#include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap() #include "pycore_uniqueid.h" // _PyObject_FinalizePerThreadRefcounts() diff --git a/Python/pythonrun.c b/Python/pythonrun.c index b67597113ead45..fc0f11bc4e8af4 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -13,6 +13,7 @@ #include "Python.h" #include "pycore_ast.h" // PyAST_mod2obj() +#include "pycore_audit.h" // _PySys_Audit() #include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_compile.h" // _PyAST_Compile() #include "pycore_interp.h" // PyInterpreterState.importlib @@ -22,7 +23,7 @@ #include "pycore_pylifecycle.h" // _Py_FdIsInteractive() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_pythonrun.h" // export _PyRun_InteractiveLoopObject() -#include "pycore_sysmodule.h" // _PySys_Audit() +#include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_traceback.h" // _PyTraceBack_Print() #include "errcode.h" // E_EOF diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 9fd194019544dd..079e9d3158cf50 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -15,6 +15,7 @@ Data members: */ #include "Python.h" +#include "pycore_audit.h" // _Py_AuditHookEntry #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_SetAsyncGenFinalizer() #include "pycore_dict.h" // _PyDict_GetItemWithError() diff --git a/Tools/msi/README.txt b/Tools/msi/README.txt index 98e5ba039d2bcd..8ae156450d5240 100644 --- a/Tools/msi/README.txt +++ b/Tools/msi/README.txt @@ -61,18 +61,20 @@ the initial download size by separating them into their own MSIs. Building the Installer ====================== -Before building the installer, download extra build dependencies using -Tools\msi\get_externals.bat. (Note that this is in addition to the +Before building the installer, download the extra build dependencies +using Tools\msi\get_externals.bat. (Note that this is in addition to the similarly named file in PCbuild.) -One of the dependencies used in builds is WiX, a toolset that lets developers -create installers for Windows Installer, the Windows installation engine. +One of the dependencies used in the build process is WiX, a toolset that +allows developers to create installers for Windows Installer, the +Windows installation engine. If you're not using the copy of WiX +installed by Tools\msi\get_externals.bat, you'll need to set the +"WixInstallPath" environment variable before building. -Additionally, make sure "MSVC v14x - VS 20xx C++ ARM64 build tools" are -selected under "Desktop Development with C++" in "Visual Studio installer", -even if you are not building on ARM64. This is required because we have -upgraded to WiX-3.14, which requires these tools for Python 3.11 and later -versions. +Additionally, ensure that "MSVC v14x - VS 20xx C++ ARM64/ARM64EC build tools" +is selected under "Desktop Development with C++" in the "Visual Studio Installer", +even if you're not building on ARM64. This is required because we've upgraded +to WiX 3.14, which requires these tools for Python 3.10 and later versions. For testing, the installer should be built with the Tools/msi/build.bat script: diff --git a/Tools/msi/wix.props b/Tools/msi/wix.props index d8ced317d0ce81..707c8aeacb648d 100644 --- a/Tools/msi/wix.props +++ b/Tools/msi/wix.props @@ -5,8 +5,6 @@ $(MSBuildThisFileDirectory)\Wix\ $(ExternalsDir)\windows-installer\wix-314\ - $(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows Installer XML\3.10@InstallRoot) - $(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\Windows Installer XML\3.10@InstallRoot) $(WixInstallPath)\Wix.targets \ No newline at end of file