Skip to content

Commit

Permalink
bpo-41796: Make _ast module state per interpreter (pythonGH-23024)
Browse files Browse the repository at this point in the history
The ast module internal state is now per interpreter.

* Rename "astmodulestate" to "struct ast_state"
* Add pycore_ast.h internal header: the ast_state structure is now
  declared in pycore_ast.h.
* Add PyInterpreterState.ast (struct ast_state)
* Remove get_ast_state()
* Rename get_global_ast_state() to get_ast_state()
* PyAST_obj2mod() now handles get_ast_state() failures
  • Loading branch information
vstinner authored Nov 2, 2020
1 parent 34637a0 commit 5cf4782
Show file tree
Hide file tree
Showing 8 changed files with 549 additions and 202 deletions.
2 changes: 1 addition & 1 deletion Include/Python-ast.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

234 changes: 234 additions & 0 deletions Include/internal/pycore_ast.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,234 @@
// File automatically generated by Parser/asdl_c.py.

#ifndef Py_INTERNAL_AST_H
#define Py_INTERNAL_AST_H
#ifdef __cplusplus
extern "C" {
#endif

#ifndef Py_BUILD_CORE
# error "this header requires Py_BUILD_CORE define"
#endif

struct ast_state {
int initialized;
PyObject *AST_type;
PyObject *Add_singleton;
PyObject *Add_type;
PyObject *And_singleton;
PyObject *And_type;
PyObject *AnnAssign_type;
PyObject *Assert_type;
PyObject *Assign_type;
PyObject *AsyncFor_type;
PyObject *AsyncFunctionDef_type;
PyObject *AsyncWith_type;
PyObject *Attribute_type;
PyObject *AugAssign_type;
PyObject *Await_type;
PyObject *BinOp_type;
PyObject *BitAnd_singleton;
PyObject *BitAnd_type;
PyObject *BitOr_singleton;
PyObject *BitOr_type;
PyObject *BitXor_singleton;
PyObject *BitXor_type;
PyObject *BoolOp_type;
PyObject *Break_type;
PyObject *Call_type;
PyObject *ClassDef_type;
PyObject *Compare_type;
PyObject *Constant_type;
PyObject *Continue_type;
PyObject *Del_singleton;
PyObject *Del_type;
PyObject *Delete_type;
PyObject *DictComp_type;
PyObject *Dict_type;
PyObject *Div_singleton;
PyObject *Div_type;
PyObject *Eq_singleton;
PyObject *Eq_type;
PyObject *ExceptHandler_type;
PyObject *Expr_type;
PyObject *Expression_type;
PyObject *FloorDiv_singleton;
PyObject *FloorDiv_type;
PyObject *For_type;
PyObject *FormattedValue_type;
PyObject *FunctionDef_type;
PyObject *FunctionType_type;
PyObject *GeneratorExp_type;
PyObject *Global_type;
PyObject *GtE_singleton;
PyObject *GtE_type;
PyObject *Gt_singleton;
PyObject *Gt_type;
PyObject *IfExp_type;
PyObject *If_type;
PyObject *ImportFrom_type;
PyObject *Import_type;
PyObject *In_singleton;
PyObject *In_type;
PyObject *Interactive_type;
PyObject *Invert_singleton;
PyObject *Invert_type;
PyObject *IsNot_singleton;
PyObject *IsNot_type;
PyObject *Is_singleton;
PyObject *Is_type;
PyObject *JoinedStr_type;
PyObject *LShift_singleton;
PyObject *LShift_type;
PyObject *Lambda_type;
PyObject *ListComp_type;
PyObject *List_type;
PyObject *Load_singleton;
PyObject *Load_type;
PyObject *LtE_singleton;
PyObject *LtE_type;
PyObject *Lt_singleton;
PyObject *Lt_type;
PyObject *MatMult_singleton;
PyObject *MatMult_type;
PyObject *Mod_singleton;
PyObject *Mod_type;
PyObject *Module_type;
PyObject *Mult_singleton;
PyObject *Mult_type;
PyObject *Name_type;
PyObject *NamedExpr_type;
PyObject *Nonlocal_type;
PyObject *NotEq_singleton;
PyObject *NotEq_type;
PyObject *NotIn_singleton;
PyObject *NotIn_type;
PyObject *Not_singleton;
PyObject *Not_type;
PyObject *Or_singleton;
PyObject *Or_type;
PyObject *Pass_type;
PyObject *Pow_singleton;
PyObject *Pow_type;
PyObject *RShift_singleton;
PyObject *RShift_type;
PyObject *Raise_type;
PyObject *Return_type;
PyObject *SetComp_type;
PyObject *Set_type;
PyObject *Slice_type;
PyObject *Starred_type;
PyObject *Store_singleton;
PyObject *Store_type;
PyObject *Sub_singleton;
PyObject *Sub_type;
PyObject *Subscript_type;
PyObject *Try_type;
PyObject *Tuple_type;
PyObject *TypeIgnore_type;
PyObject *UAdd_singleton;
PyObject *UAdd_type;
PyObject *USub_singleton;
PyObject *USub_type;
PyObject *UnaryOp_type;
PyObject *While_type;
PyObject *With_type;
PyObject *YieldFrom_type;
PyObject *Yield_type;
PyObject *__dict__;
PyObject *__doc__;
PyObject *__module__;
PyObject *_attributes;
PyObject *_fields;
PyObject *alias_type;
PyObject *annotation;
PyObject *arg;
PyObject *arg_type;
PyObject *args;
PyObject *argtypes;
PyObject *arguments_type;
PyObject *asname;
PyObject *ast;
PyObject *attr;
PyObject *bases;
PyObject *body;
PyObject *boolop_type;
PyObject *cause;
PyObject *cmpop_type;
PyObject *col_offset;
PyObject *comparators;
PyObject *comprehension_type;
PyObject *context_expr;
PyObject *conversion;
PyObject *ctx;
PyObject *decorator_list;
PyObject *defaults;
PyObject *elt;
PyObject *elts;
PyObject *end_col_offset;
PyObject *end_lineno;
PyObject *exc;
PyObject *excepthandler_type;
PyObject *expr_context_type;
PyObject *expr_type;
PyObject *finalbody;
PyObject *format_spec;
PyObject *func;
PyObject *generators;
PyObject *handlers;
PyObject *id;
PyObject *ifs;
PyObject *is_async;
PyObject *items;
PyObject *iter;
PyObject *key;
PyObject *keys;
PyObject *keyword_type;
PyObject *keywords;
PyObject *kind;
PyObject *kw_defaults;
PyObject *kwarg;
PyObject *kwonlyargs;
PyObject *left;
PyObject *level;
PyObject *lineno;
PyObject *lower;
PyObject *mod_type;
PyObject *module;
PyObject *msg;
PyObject *name;
PyObject *names;
PyObject *op;
PyObject *operand;
PyObject *operator_type;
PyObject *ops;
PyObject *optional_vars;
PyObject *orelse;
PyObject *posonlyargs;
PyObject *returns;
PyObject *right;
PyObject *simple;
PyObject *slice;
PyObject *step;
PyObject *stmt_type;
PyObject *tag;
PyObject *target;
PyObject *targets;
PyObject *test;
PyObject *type;
PyObject *type_comment;
PyObject *type_ignore_type;
PyObject *type_ignores;
PyObject *unaryop_type;
PyObject *upper;
PyObject *value;
PyObject *values;
PyObject *vararg;
PyObject *withitem_type;
};

#ifdef __cplusplus
}
#endif
#endif /* !Py_INTERNAL_AST_H */

11 changes: 7 additions & 4 deletions Include/internal/pycore_interp.h
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,11 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif

#include "pycore_atomic.h" /* _Py_atomic_address */
#include "pycore_gil.h" /* struct _gil_runtime_state */
#include "pycore_gc.h" /* struct _gc_runtime_state */
#include "pycore_warnings.h" /* struct _warnings_runtime_state */
#include "pycore_atomic.h" // _Py_atomic_address
#include "pycore_ast.h" // struct ast_state
#include "pycore_gil.h" // struct _gil_runtime_state
#include "pycore_gc.h" // struct _gc_runtime_state
#include "pycore_warnings.h" // struct _warnings_runtime_state

struct _pending_calls {
PyThread_type_lock lock;
Expand Down Expand Up @@ -258,6 +259,8 @@ struct _is {
struct _Py_async_gen_state async_gen;
struct _Py_context_state context;
struct _Py_exc_state exc_state;

struct ast_state ast;
};

/* Used by _PyImport_Cleanup() */
Expand Down
2 changes: 2 additions & 0 deletions Makefile.pre.in
Original file line number Diff line number Diff line change
Expand Up @@ -840,9 +840,11 @@ regen-ast:
$(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \
$(srcdir)/Parser/Python.asdl \
-H $(srcdir)/Include/Python-ast.h.new \
-I $(srcdir)/Include/internal/pycore_ast.h.new \
-C $(srcdir)/Python/Python-ast.c.new

$(UPDATE_FILE) $(srcdir)/Include/Python-ast.h $(srcdir)/Include/Python-ast.h.new
$(UPDATE_FILE) $(srcdir)/Include/internal/pycore_ast.h $(srcdir)/Include/internal/pycore_ast.h.new
$(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new

.PHONY: regen-opcode
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
The :mod:`ast` module internal state is now per interpreter. Patch by Victor
Stinner.
12 changes: 9 additions & 3 deletions PCbuild/regen.vcxproj
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,8 @@
</None>
<None Include="..\Include\opcode.h">
</None>
<None Include="..\Include\internal\pycore_ast.h">
</None>
<None Include="..\Include\Python-ast.h">
</None>
<None Include="..\Python\Python-ast.c">
Expand All @@ -161,15 +163,18 @@
<Warning Text="Pegen updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedParse)' != ''" />
</Target>
<Target Name="_RegenAST_H" AfterTargets="_RegenGrammar">
<!-- Regenerate Include/Python-ast.h and Python/Python-ast.c using Parser/asdl_c.py -h -->
<Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot; -H &quot;$(IntDir)Python-ast.h&quot; -C &quot;$(IntDir)Python-ast.c&quot;" />
<!-- Regenerate Include/Python-ast.h, Python/Python-ast.c and Include/internal/pycore_ast.h using Parser/asdl_c.py -h -->
<Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot; -H &quot;$(IntDir)Python-ast.h&quot; -C &quot;$(IntDir)Python-ast.c&quot; -I &quot;$(IntDir)pycore_ast.h&quot;" />
<Copy SourceFiles="$(IntDir)Python-ast.h" DestinationFiles="$(PySourcePath)Include\Python-ast.h">
<Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" />
</Copy>
<Copy SourceFiles="$(IntDir)Python-ast.c" DestinationFiles="$(PySourcePath)Python\Python-ast.c">
<Output TaskParameter="CopiedFiles" ItemName="_UpdatedC" />
</Copy>
<Warning Text="ASDL is updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
<Copy SourceFiles="$(IntDir)pycore_ast.h" DestinationFiles="$(PySourcePath)Include\internal\pycore_ast.h">
<Output TaskParameter="CopiedFiles" ItemName="_UpdatedInternalH" />
</Copy>
<Warning Text="ASDL is updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' != '' and '@(_UpdatedC)' != '' and @(_UpdatedInternalH)'" />
</Target>
<Target Name="_RegenOpcodes" AfterTargets="_RegenAST_C">
<!-- Regenerate Include/opcode.h from Lib/opcode.py using Tools/scripts/generate_opcode_h.py-->
Expand Down Expand Up @@ -204,6 +209,7 @@
<Clean Include="$(IntDir)opcode.h" />
<Clean Include="$(IntDir)Python-ast.c" />
<Clean Include="$(IntDir)Python-ast.h" />
<Clean Include="$(IntDir)pycore_ast.h" />
</ItemGroup>
</Target>
</Project>
Loading

0 comments on commit 5cf4782

Please sign in to comment.