Skip to content

Commit

Permalink
Add Include/cpython/pystats.h
Browse files Browse the repository at this point in the history
* Rename '_py_stats' variable to '_Py_stats'.
* Exclude Include/cpython/pystats.h from the Py_LIMITED_API.
* No longer export _py_stats_struct variable.
* Move pystats.h include from object.h to Python.h.
* Add comments to Include/pystats.h and Include/cpython/pystats.h.
* Doc/using/configure.rst lists sys functions addedd by Py_STATS.
  • Loading branch information
vstinner committed Sep 1, 2023
1 parent 3edcf74 commit 354c14e
Show file tree
Hide file tree
Showing 14 changed files with 180 additions and 141 deletions.
15 changes: 13 additions & 2 deletions Doc/using/configure.rst
Original file line number Diff line number Diff line change
Expand Up @@ -194,9 +194,20 @@ General Options

Turn on internal statistics gathering.

Define the ``Py_STATS`` macro.

Add functions to the :mod:`sys` module:

* :func:`sys._stats_on`: Turns on stats gathering (stats gathering is on by
default).
* :func:`sys._stats_off`: Turns off stats gathering (stats gathering is on
by default).
* :func:`sys._stats_clear`: Clears the stats.
* :func:`sys._stats_dump`: Dump stats to file, and clears the stats.

The statistics will be dumped to a arbitrary (probably unique) file in
``/tmp/py_stats/``, or ``C:\temp\py_stats\`` on Windows. If that directory
does not exist, results will be printed on stdout.
``/tmp/py_stats/`` (Unix) or ``C:\temp\py_stats\`` (Windows). If that
directory does not exist, results will be printed on stdout.

Use ``Tools/scripts/summarize_stats.py`` to read the stats.

Expand Down
1 change: 1 addition & 0 deletions Include/Python.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
#include "pymem.h"
#include "pytypedefs.h"
#include "pybuffer.h"
#include "pystats.h"
#include "object.h"
#include "objimpl.h"
#include "typeslots.h"
Expand Down
122 changes: 122 additions & 0 deletions Include/cpython/pystats.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
// Statistics on Python performance.
//
// Main API:
//
// - _Py_INCREF_STAT_INC() and _Py_DECREF_STAT_INC() used by Py_INCREF()
// and Py_DECREF().
// - _Py_stats
// - _Py_StatsClear()
// - _Py_PrintSpecializationStats()
//
// See sys._stats_on() and sys._stats_off() functions.
//
// Python must be built with ./configure --enable-pystats to define the
// Py_STATS macro.
//
// Define _PY_INTERPRETER macro to increment interpreter_increfs and
// interpreter_decrefs. Otherwise, increment increfs and decrefs.

#ifndef Py_CPYTHON_PYSTATS_H
# error "this header file must not be included directly"
#endif

#define SPECIALIZATION_FAILURE_KINDS 36

/* Stats for determining who is calling PyEval_EvalFrame */
#define EVAL_CALL_TOTAL 0
#define EVAL_CALL_VECTOR 1
#define EVAL_CALL_GENERATOR 2
#define EVAL_CALL_LEGACY 3
#define EVAL_CALL_FUNCTION_VECTORCALL 4
#define EVAL_CALL_BUILD_CLASS 5
#define EVAL_CALL_SLOT 6
#define EVAL_CALL_FUNCTION_EX 7
#define EVAL_CALL_API 8
#define EVAL_CALL_METHOD 9

#define EVAL_CALL_KINDS 10

typedef struct _specialization_stats {
uint64_t success;
uint64_t failure;
uint64_t hit;
uint64_t deferred;
uint64_t miss;
uint64_t deopt;
uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS];
} SpecializationStats;

typedef struct _opcode_stats {
SpecializationStats specialization;
uint64_t execution_count;
uint64_t pair_count[256];
} OpcodeStats;

typedef struct _call_stats {
uint64_t inlined_py_calls;
uint64_t pyeval_calls;
uint64_t frames_pushed;
uint64_t frame_objects_created;
uint64_t eval_calls[EVAL_CALL_KINDS];
} CallStats;

typedef struct _object_stats {
uint64_t increfs;
uint64_t decrefs;
uint64_t interpreter_increfs;
uint64_t interpreter_decrefs;
uint64_t allocations;
uint64_t allocations512;
uint64_t allocations4k;
uint64_t allocations_big;
uint64_t frees;
uint64_t to_freelist;
uint64_t from_freelist;
uint64_t new_values;
uint64_t dict_materialized_on_request;
uint64_t dict_materialized_new_key;
uint64_t dict_materialized_too_big;
uint64_t dict_materialized_str_subclass;
uint64_t dict_dematerialized;
uint64_t type_cache_hits;
uint64_t type_cache_misses;
uint64_t type_cache_dunder_hits;
uint64_t type_cache_dunder_misses;
uint64_t type_cache_collisions;
uint64_t optimization_attempts;
uint64_t optimization_traces_created;
uint64_t optimization_traces_executed;
uint64_t optimization_uops_executed;
/* Temporary value used during GC */
uint64_t object_visits;
} ObjectStats;

typedef struct _gc_stats {
uint64_t collections;
uint64_t object_visits;
uint64_t objects_collected;
} GCStats;

typedef struct _stats {
OpcodeStats opcode_stats[256];
CallStats call_stats;
ObjectStats object_stats;
GCStats *gc_stats;
} PyStats;


extern PyStats _Py_stats_struct;

// Export for shared extensions like 'math'
PyAPI_DATA(PyStats*) _Py_stats;

extern void _Py_StatsClear(void);
extern void _Py_PrintSpecializationStats(int to_file);

#ifdef _PY_INTERPRETER
# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_increfs++; } while (0)
# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_decrefs++; } while (0)
#else
# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.increfs++; } while (0)
# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.decrefs++; } while (0)
#endif
18 changes: 9 additions & 9 deletions Include/internal/pycore_code.h
Original file line number Diff line number Diff line change
Expand Up @@ -268,17 +268,17 @@ extern int _PyStaticCode_Init(PyCodeObject *co);

#ifdef Py_STATS

#define STAT_INC(opname, name) do { if (_py_stats) _py_stats->opcode_stats[opname].specialization.name++; } while (0)
#define STAT_DEC(opname, name) do { if (_py_stats) _py_stats->opcode_stats[opname].specialization.name--; } while (0)
#define OPCODE_EXE_INC(opname) do { if (_py_stats) _py_stats->opcode_stats[opname].execution_count++; } while (0)
#define CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.name++; } while (0)
#define OBJECT_STAT_INC(name) do { if (_py_stats) _py_stats->object_stats.name++; } while (0)
#define STAT_INC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name++; } while (0)
#define STAT_DEC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name--; } while (0)
#define OPCODE_EXE_INC(opname) do { if (_Py_stats) _Py_stats->opcode_stats[opname].execution_count++; } while (0)
#define CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.name++; } while (0)
#define OBJECT_STAT_INC(name) do { if (_Py_stats) _Py_stats->object_stats.name++; } while (0)
#define OBJECT_STAT_INC_COND(name, cond) \
do { if (_py_stats && cond) _py_stats->object_stats.name++; } while (0)
#define EVAL_CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.eval_calls[name]++; } while (0)
do { if (_Py_stats && cond) _Py_stats->object_stats.name++; } while (0)
#define EVAL_CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.eval_calls[name]++; } while (0)
#define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \
do { if (_py_stats && PyFunction_Check(callable)) _py_stats->call_stats.eval_calls[name]++; } while (0)
#define GC_STAT_ADD(gen, name, n) do { if (_py_stats) _py_stats->gc_stats[(gen)].name += (n); } while (0)
do { if (_Py_stats && PyFunction_Check(callable)) _Py_stats->call_stats.eval_calls[name]++; } while (0)
#define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0)

// Export for '_opcode' shared extension
PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);
Expand Down
2 changes: 0 additions & 2 deletions Include/object.h
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,6 @@ A standard interface exists for objects that contain an array of items
whose size is determined when the object is allocated.
*/

#include "pystats.h"

/* Py_DEBUG implies Py_REF_DEBUG. */
#if defined(Py_DEBUG) && !defined(Py_REF_DEBUG)
# define Py_REF_DEBUG
Expand Down
124 changes: 13 additions & 111 deletions Include/pystats.h
Original file line number Diff line number Diff line change
@@ -1,124 +1,26 @@

// Statistics on Python performance (public API).
//
// Define _Py_INCREF_STAT_INC() and _Py_DECREF_STAT_INC() used by Py_INCREF()
// and Py_DECREF().
//
// See also Include/cpython/pystats.h for the full API.

#ifndef Py_PYSTATS_H
#define Py_PYSTATS_H
#ifdef __cplusplus
extern "C" {
#endif

#ifdef Py_STATS

#define SPECIALIZATION_FAILURE_KINDS 36

/* Stats for determining who is calling PyEval_EvalFrame */
#define EVAL_CALL_TOTAL 0
#define EVAL_CALL_VECTOR 1
#define EVAL_CALL_GENERATOR 2
#define EVAL_CALL_LEGACY 3
#define EVAL_CALL_FUNCTION_VECTORCALL 4
#define EVAL_CALL_BUILD_CLASS 5
#define EVAL_CALL_SLOT 6
#define EVAL_CALL_FUNCTION_EX 7
#define EVAL_CALL_API 8
#define EVAL_CALL_METHOD 9

#define EVAL_CALL_KINDS 10

typedef struct _specialization_stats {
uint64_t success;
uint64_t failure;
uint64_t hit;
uint64_t deferred;
uint64_t miss;
uint64_t deopt;
uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS];
} SpecializationStats;

typedef struct _opcode_stats {
SpecializationStats specialization;
uint64_t execution_count;
uint64_t pair_count[256];
} OpcodeStats;

typedef struct _call_stats {
uint64_t inlined_py_calls;
uint64_t pyeval_calls;
uint64_t frames_pushed;
uint64_t frame_objects_created;
uint64_t eval_calls[EVAL_CALL_KINDS];
} CallStats;

typedef struct _object_stats {
uint64_t increfs;
uint64_t decrefs;
uint64_t interpreter_increfs;
uint64_t interpreter_decrefs;
uint64_t allocations;
uint64_t allocations512;
uint64_t allocations4k;
uint64_t allocations_big;
uint64_t frees;
uint64_t to_freelist;
uint64_t from_freelist;
uint64_t new_values;
uint64_t dict_materialized_on_request;
uint64_t dict_materialized_new_key;
uint64_t dict_materialized_too_big;
uint64_t dict_materialized_str_subclass;
uint64_t dict_dematerialized;
uint64_t type_cache_hits;
uint64_t type_cache_misses;
uint64_t type_cache_dunder_hits;
uint64_t type_cache_dunder_misses;
uint64_t type_cache_collisions;
uint64_t optimization_attempts;
uint64_t optimization_traces_created;
uint64_t optimization_traces_executed;
uint64_t optimization_uops_executed;
/* Temporary value used during GC */
uint64_t object_visits;
} ObjectStats;

typedef struct _gc_stats {
uint64_t collections;
uint64_t object_visits;
uint64_t objects_collected;
} GCStats;

typedef struct _stats {
OpcodeStats opcode_stats[256];
CallStats call_stats;
ObjectStats object_stats;
GCStats *gc_stats;
} PyStats;


PyAPI_DATA(PyStats) _py_stats_struct;
PyAPI_DATA(PyStats *) _py_stats;

extern void _Py_StatsClear(void);
extern void _Py_PrintSpecializationStats(int to_file);

#ifdef _PY_INTERPRETER

#define _Py_INCREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.interpreter_increfs++; } while (0)
#define _Py_DECREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.interpreter_decrefs++; } while (0)

#if defined(Py_STATS) && !defined(Py_LIMITED_API)
# define Py_CPYTHON_PYSTATS_H
# include "cpython/pystats.h"
# undef Py_CPYTHON_PYSTATS_H
#else

#define _Py_INCREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.increfs++; } while (0)
#define _Py_DECREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.decrefs++; } while (0)

#endif

#else

#define _Py_INCREF_STAT_INC() ((void)0)
#define _Py_DECREF_STAT_INC() ((void)0)

# define _Py_INCREF_STAT_INC() ((void)0)
# define _Py_DECREF_STAT_INC() ((void)0)
#endif // !Py_STATS

#ifdef __cplusplus
}
#endif
#endif /* !Py_PYSTATs_H */
#endif // !Py_PYSTATS_H
1 change: 1 addition & 0 deletions Makefile.pre.in
Original file line number Diff line number Diff line change
Expand Up @@ -1724,6 +1724,7 @@ PYTHON_HEADERS= \
$(srcdir)/Include/cpython/pylifecycle.h \
$(srcdir)/Include/cpython/pymem.h \
$(srcdir)/Include/cpython/pystate.h \
$(srcdir)/Include/cpython/pystats.h \
$(srcdir)/Include/cpython/pythonrun.h \
$(srcdir)/Include/cpython/pythread.h \
$(srcdir)/Include/cpython/setobject.h \
Expand Down
10 changes: 5 additions & 5 deletions Modules/gcmodule.c
Original file line number Diff line number Diff line change
Expand Up @@ -1200,8 +1200,8 @@ gc_collect_main(PyThreadState *tstate, int generation,
{
GC_STAT_ADD(generation, collections, 1);
#ifdef Py_STATS
if (_py_stats) {
_py_stats->object_stats.object_visits = 0;
if (_Py_stats) {
_Py_stats->object_stats.object_visits = 0;
}
#endif
int i;
Expand Down Expand Up @@ -1362,10 +1362,10 @@ gc_collect_main(PyThreadState *tstate, int generation,

GC_STAT_ADD(generation, objects_collected, m);
#ifdef Py_STATS
if (_py_stats) {
if (_Py_stats) {
GC_STAT_ADD(generation, object_visits,
_py_stats->object_stats.object_visits);
_py_stats->object_stats.object_visits = 0;
_Py_stats->object_stats.object_visits);
_Py_stats->object_stats.object_visits = 0;
}
#endif

Expand Down
1 change: 1 addition & 0 deletions PCbuild/pythoncore.vcxproj
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@
<ClInclude Include="..\Include\cpython\pylifecycle.h" />
<ClInclude Include="..\Include\cpython\pymem.h" />
<ClInclude Include="..\Include\cpython\pystate.h" />
<ClInclude Include="..\Include\cpython\pystats.h" />
<ClInclude Include="..\Include\cpython\pythonrun.h" />
<ClInclude Include="..\Include\cpython\pythread.h" />
<ClInclude Include="..\Include\cpython\setobject.h" />
Expand Down
3 changes: 3 additions & 0 deletions PCbuild/pythoncore.vcxproj.filters
Original file line number Diff line number Diff line change
Expand Up @@ -492,6 +492,9 @@
<ClInclude Include="..\Include\cpython\pystate.h">
<Filter>Include\cpython</Filter>
</ClInclude>
<ClInclude Include="..\Include\cpython\pystats.h">
<Filter>Include\cpython</Filter>
</ClInclude>
<ClInclude Include="..\Include\cpython\initconfig.h">
<Filter>Include\cpython</Filter>
</ClInclude>
Expand Down
2 changes: 1 addition & 1 deletion Python/ceval_macros.h
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@
do { \
frame->prev_instr = next_instr++; \
OPCODE_EXE_INC(op); \
if (_py_stats) _py_stats->opcode_stats[lastopcode].pair_count[op]++; \
if (_Py_stats) _Py_stats->opcode_stats[lastopcode].pair_count[op]++; \
lastopcode = op; \
} while (0)
#else
Expand Down
2 changes: 1 addition & 1 deletion Python/initconfig.c
Original file line number Diff line number Diff line change
Expand Up @@ -2103,7 +2103,7 @@ config_read(PyConfig *config, int compute_path_config)

#ifdef Py_STATS
if (config_get_xoption(config, L"pystats")) {
_py_stats = &_py_stats_struct;
_Py_stats = &_Py_stats_struct;
}
#endif

Expand Down
Loading

0 comments on commit 354c14e

Please sign in to comment.