From 56a9fdcb8dbb1aad50677a7b7bace81e3aaac9e3 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Fri, 18 Aug 2023 09:11:25 +0900 Subject: [PATCH] Address code review --- .github/workflows/mypy.yml | 10 +- Tools/cases_generator/analysis.py | 4 + Tools/cases_generator/flags.py | 12 +- Tools/cases_generator/formatting.py | 5 +- Tools/cases_generator/generate_cases.py | 73 +++-- Tools/cases_generator/lexer.py | 315 +++++++++++++-------- Tools/cases_generator/mypy.ini | 2 +- Tools/cases_generator/requirements-dev.txt | 2 - Tools/cases_generator/stacking.py | 14 +- Tools/clinic/requirements-dev.txt | 2 - Tools/requirements-dev.txt | 4 + 11 files changed, 273 insertions(+), 170 deletions(-) delete mode 100644 Tools/cases_generator/requirements-dev.txt delete mode 100644 Tools/clinic/requirements-dev.txt create mode 100644 Tools/requirements-dev.txt diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 54766e11285f863..c75715e728bebe7 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -35,8 +35,8 @@ jobs: with: python-version: "3.x" cache: pip - cache-dependency-path: Tools/clinic/requirements-dev.txt - - run: pip install -r Tools/clinic/requirements-dev.txt + cache-dependency-path: Tools/requirements-dev.txt + - run: pip install -r Tools/requirements-dev.txt - run: mypy --config-file Tools/clinic/mypy.ini mypy-cases-generator: @@ -49,6 +49,8 @@ jobs: with: python-version: "3.x" cache: pip - cache-dependency-path: Tools/cases_generator/requirements-dev.txt - - run: pip install -r Tools/cases_generator/requirements-dev.txt + cache-dependency-path: Tools/requirements-dev.txt + - name: Line with black + run: black Tools/cases_generator + - run: pip install -r Tools/requirements-dev.txt - run: mypy --config-file Tools/cases_generator/mypy.ini diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index e4d2945a1c31aa4..b06b134ed3fc03f 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -171,6 +171,8 @@ def parse_file(self, filename: str, instrs_idx: dict[str, int]) -> None: case parsing.Pseudo(name): self.pseudos[name] = thing self.everything.append(thing) + case _: + typing.assert_never(thing) if not psr.eof(): raise psr.make_syntax_error(f"Extra stuff at the end of {filename}") @@ -402,4 +404,6 @@ def check_macro_components( components.append(self.instrs[name]) case parsing.CacheEffect(): components.append(uop) + case _: + typing.assert_never(uop) return components diff --git a/Tools/cases_generator/flags.py b/Tools/cases_generator/flags.py index 79b88cc6dd8bdbf..c82f94a5271e4c8 100644 --- a/Tools/cases_generator/flags.py +++ b/Tools/cases_generator/flags.py @@ -20,7 +20,7 @@ def __post_init__(self) -> None: self.bitmask = {name: (1 << i) for i, name in enumerate(self.names())} @staticmethod - def fromInstruction(instr: parsing.Node) -> 'InstructionFlags': + def fromInstruction(instr: parsing.Node) -> "InstructionFlags": has_free = ( variable_used(instr, "PyCell_New") @@ -41,7 +41,7 @@ def fromInstruction(instr: parsing.Node) -> 'InstructionFlags': ) @staticmethod - def newEmpty() -> 'InstructionFlags': + def newEmpty() -> "InstructionFlags": return InstructionFlags(False, False, False, False, False, False) def add(self, other: "InstructionFlags") -> None: @@ -49,7 +49,7 @@ def add(self, other: "InstructionFlags") -> None: if value: setattr(self, name, value) - def names(self, value: bool|None = None) -> list[str]: + def names(self, value: bool | None = None) -> list[str]: if value is None: return list(dataclasses.asdict(self).keys()) return [n for n, v in dataclasses.asdict(self).items() if v == value] @@ -90,9 +90,9 @@ def variable_used_unspecialized(node: parsing.Node, name: str) -> bool: text = "".join(token.text.split()) # TODO: Handle nested #if if text == "#if": - if ( - i + 1 < len(node.tokens) - and node.tokens[i + 1].text in ("ENABLE_SPECIALIZATION", "TIER_ONE") + if i + 1 < len(node.tokens) and node.tokens[i + 1].text in ( + "ENABLE_SPECIALIZATION", + "TIER_ONE", ): skipping = True elif text in ("#else", "#endif"): diff --git a/Tools/cases_generator/formatting.py b/Tools/cases_generator/formatting.py index d3a6da92def531a..65ed5d7d042ef9b 100644 --- a/Tools/cases_generator/formatting.py +++ b/Tools/cases_generator/formatting.py @@ -1,3 +1,4 @@ +import collections import contextlib import re import typing @@ -58,13 +59,13 @@ def reset_lineno(self) -> None: self.set_lineno(self.lineno + 1, self.filename) @contextlib.contextmanager - def indent(self) -> typing.Iterator[None]: + def indent(self) -> collections.abc.Iterator[None]: self.prefix += " " yield self.prefix = self.prefix[:-4] @contextlib.contextmanager - def block(self, head: str, tail: str = "") -> typing.Iterator[None]: + def block(self, head: str, tail: str = "") -> collections.abc.Iterator[None]: if head: self.emit(head + " {") else: diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 11f6cb3a5c9b647..5b4c61ceed3181f 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -79,12 +79,10 @@ "STORE_FAST", "STORE_FAST_MAYBE_NULL", "COPY", - # Arithmetic "_BINARY_OP_MULTIPLY_INT", "_BINARY_OP_ADD_INT", "_BINARY_OP_SUBTRACT_INT", - } arg_parser = argparse.ArgumentParser( @@ -143,6 +141,7 @@ default=DEFAULT_ABSTRACT_INTERPRETER_OUTPUT, ) + class Generator(Analyzer): def get_stack_effect_info( self, thing: parsing.InstDef | parsing.Macro | parsing.Pseudo @@ -183,10 +182,14 @@ def effect_str(effects: list[StackEffect]) -> str: target_popped = effect_str(target_instr.input_effects) target_pushed = effect_str(target_instr.output_effects) popped, pushed = target_popped, target_pushed + case _: + typing.assert_never(thing) return instr, popped, pushed @contextlib.contextmanager - def metadata_item(self, signature: str, open: str, close: str) -> typing.Iterator[None]: + def metadata_item( + self, signature: str, open: str, close: str + ) -> typing.Iterator[None]: self.out.emit("") self.out.emit(f"extern {signature};") self.out.emit("#ifdef NEED_OPCODE_METADATA") @@ -211,7 +214,9 @@ def write_function( ) -> None: with self.metadata_item( - f"int _PyOpcode_num_{direction}(int opcode, int oparg, bool jump)", "", "" + f"int _PyOpcode_num_{direction}(int opcode, int oparg, bool jump)", + "", + "", ): with self.out.block("switch(opcode)"): for instr, effect in data: @@ -249,10 +254,11 @@ def assign_opcode_ids(self) -> None: for instr in itertools.chain( [instr for instr in self.instrs.values() if instr.kind != "op"], - self.macro_instrs.values()): + self.macro_instrs.values(), + ): assert isinstance(instr, (Instruction, MacroInstruction, PseudoInstruction)) name = instr.name - if name.startswith('INSTRUMENTED_'): + if name.startswith("INSTRUMENTED_"): instrumented_ops.append(name) else: ops.append((instr.instr_flags.HAS_ARG_FLAG, name)) @@ -261,13 +267,13 @@ def assign_opcode_ids(self) -> None: # rather than bytecodes.c, so we need to add it explicitly # here (at least until we add something to bytecodes.c to # declare external instructions). - instrumented_ops.append('INSTRUMENTED_LINE') + instrumented_ops.append("INSTRUMENTED_LINE") # assert lists are unique assert len(set(ops)) == len(ops) assert len(set(instrumented_ops)) == len(instrumented_ops) - opname: list[str|None] = [None] * 512 + opname: list[str | None] = [None] * 512 opmap: dict[str, int] = {} markers: dict[str, int] = {} @@ -278,16 +284,15 @@ def map_op(op: int, name: str) -> None: opname[op] = name opmap[name] = op - # 0 is reserved for cache entries. This helps debugging. - map_op(0, 'CACHE') + map_op(0, "CACHE") # 17 is reserved as it is the initial value for the specializing counter. # This helps catch cases where we attempt to execute a cache. - map_op(17, 'RESERVED') + map_op(17, "RESERVED") # 166 is RESUME - it is hard coded as such in Tools/build/deepfreeze.py - map_op(166, 'RESUME') + map_op(166, "RESUME") next_opcode = 1 @@ -299,13 +304,13 @@ def map_op(op: int, name: str) -> None: assert next_opcode < 255 map_op(next_opcode, name) - if has_arg and 'HAVE_ARGUMENT' not in markers: - markers['HAVE_ARGUMENT'] = next_opcode + if has_arg and "HAVE_ARGUMENT" not in markers: + markers["HAVE_ARGUMENT"] = next_opcode # Instrumented opcodes are at the end of the valid range min_instrumented = 254 - (len(instrumented_ops) - 1) assert next_opcode <= min_instrumented - markers['MIN_INSTRUMENTED_OPCODE'] = min_instrumented + markers["MIN_INSTRUMENTED_OPCODE"] = min_instrumented for i, op in enumerate(instrumented_ops): map_op(min_instrumented + i, op) @@ -317,7 +322,9 @@ def map_op(op: int, name: str) -> None: self.opmap = opmap self.markers = markers - def write_opcode_ids(self, opcode_ids_h_filename: str, opcode_targets_filename: str) -> None: + def write_opcode_ids( + self, opcode_ids_h_filename: str, opcode_targets_filename: str + ) -> None: """Write header file that defined the opcode IDs""" with open(opcode_ids_h_filename, "w") as f: @@ -330,7 +337,7 @@ def write_opcode_ids(self, opcode_ids_h_filename: str, opcode_targets_filename: self.out.emit("#ifndef Py_OPCODE_IDS_H") self.out.emit("#define Py_OPCODE_IDS_H") self.out.emit("#ifdef __cplusplus") - self.out.emit("extern \"C\" {") + self.out.emit('extern "C" {') self.out.emit("#endif") self.out.emit("") self.out.emit("/* Instruction opcodes for compiled code */") @@ -363,7 +370,6 @@ def define(name: str, opcode: int) -> None: targets[op] = f"TARGET_{name}" f.write(",\n".join([f" &&{s}" for s in targets])) - def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> None: """Write instruction metadata to output file.""" @@ -458,7 +464,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No "const struct opcode_metadata " "_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE]", "=", - ";" + ";", ): # Write metadata for each instruction for thing in self.everything: @@ -471,13 +477,15 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No case parsing.Macro(): self.write_metadata_for_macro(self.macro_instrs[thing.name]) case parsing.Pseudo(): - self.write_metadata_for_pseudo(self.pseudo_instrs[thing.name]) + self.write_metadata_for_pseudo( + self.pseudo_instrs[thing.name] + ) with self.metadata_item( "const struct opcode_macro_expansion " "_PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE]", "=", - ";" + ";", ): # Write macro expansion for each non-pseudo instruction for thing in self.everything: @@ -514,7 +522,9 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No self.write_uop_items(lambda name, counter: f'[{name}] = "{name}",') with self.metadata_item( - f"const char *const _PyOpcode_OpName[{1 + max(self.opmap.values())}]", "=", ";" + f"const char *const _PyOpcode_OpName[{1 + max(self.opmap.values())}]", + "=", + ";", ): for name in self.opmap: self.out.emit(f'[{name}] = "{name}",') @@ -527,11 +537,9 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No for m in family.members: deoptcodes[m] = name # special case: - deoptcodes['BINARY_OP_INPLACE_ADD_UNICODE'] = 'BINARY_OP' + deoptcodes["BINARY_OP_INPLACE_ADD_UNICODE"] = "BINARY_OP" - with self.metadata_item( - f"const uint8_t _PyOpcode_Deopt[256]", "=", ";" - ): + with self.metadata_item(f"const uint8_t _PyOpcode_Deopt[256]", "=", ";"): for opt, deopt in sorted(deoptcodes.items()): self.out.emit(f"[{opt}] = {deopt},") @@ -589,10 +597,9 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No if name not in specialized_ops: self.out.emit(f"'{name}': {op},") - for name in ['MIN_INSTRUMENTED_OPCODE', 'HAVE_ARGUMENT']: + for name in ["MIN_INSTRUMENTED_OPCODE", "HAVE_ARGUMENT"]: self.out.emit(f"{name} = {self.markers[name]}") - def write_pseudo_instrs(self) -> None: """Write the IS_PSEUDO_INSTR macro""" self.out.emit("\n\n#define IS_PSEUDO_INSTR(OP) ( \\") @@ -815,7 +822,10 @@ def write_abstract_interpreter_instructions( pass case parsing.InstDef(): instr = AbstractInstruction(self.instrs[thing.name].inst) - if instr.is_viable_uop() and instr.name not in SPECIALLY_HANDLED_ABSTRACT_INSTR: + if ( + instr.is_viable_uop() + and instr.name not in SPECIALLY_HANDLED_ABSTRACT_INSTR + ): self.out.emit("") with self.out.block(f"case {thing.name}:"): instr.write(self.out, tier=TIER_TWO) @@ -878,8 +888,9 @@ def main() -> None: a.write_opcode_ids(args.opcode_ids_h, args.opcode_targets_h) a.write_metadata(args.metadata, args.pymetadata) a.write_executor_instructions(args.executor_cases, args.emit_line_directives) - a.write_abstract_interpreter_instructions(args.abstract_interpreter_cases, - args.emit_line_directives) + a.write_abstract_interpreter_instructions( + args.abstract_interpreter_cases, args.emit_line_directives + ) if __name__ == "__main__": diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index d0e3be9051caf1c..a60f6c11a4c4607 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -10,134 +10,215 @@ def choice(*opts: str) -> str: return "|".join("(%s)" % opt for opt in opts) + # Regexes # Longer operators must go before shorter ones. -PLUSPLUS = r'\+\+' -MINUSMINUS = r'--' +PLUSPLUS = r"\+\+" +MINUSMINUS = r"--" # -> -ARROW = r'->' -ELLIPSIS = r'\.\.\.' +ARROW = r"->" +ELLIPSIS = r"\.\.\." # Assignment operators -TIMESEQUAL = r'\*=' -DIVEQUAL = r'/=' -MODEQUAL = r'%=' -PLUSEQUAL = r'\+=' -MINUSEQUAL = r'-=' -LSHIFTEQUAL = r'<<=' -RSHIFTEQUAL = r'>>=' -ANDEQUAL = r'&=' -OREQUAL = r'\|=' -XOREQUAL = r'\^=' +TIMESEQUAL = r"\*=" +DIVEQUAL = r"/=" +MODEQUAL = r"%=" +PLUSEQUAL = r"\+=" +MINUSEQUAL = r"-=" +LSHIFTEQUAL = r"<<=" +RSHIFTEQUAL = r">>=" +ANDEQUAL = r"&=" +OREQUAL = r"\|=" +XOREQUAL = r"\^=" # Operators -PLUS = r'\+' -MINUS = r'-' -TIMES = r'\*' -DIVIDE = r'/' -MOD = r'%' -NOT = r'~' -XOR = r'\^' -LOR = r'\|\|' -LAND = r'&&' -LSHIFT = r'<<' -RSHIFT = r'>>' -LE = r'<=' -GE = r'>=' -EQ = r'==' -NE = r'!=' -LT = r'<' -GT = r'>' -LNOT = r'!' -OR = r'\|' -AND = r'&' -EQUALS = r'=' +PLUS = r"\+" +MINUS = r"-" +TIMES = r"\*" +DIVIDE = r"/" +MOD = r"%" +NOT = r"~" +XOR = r"\^" +LOR = r"\|\|" +LAND = r"&&" +LSHIFT = r"<<" +RSHIFT = r">>" +LE = r"<=" +GE = r">=" +EQ = r"==" +NE = r"!=" +LT = r"<" +GT = r">" +LNOT = r"!" +OR = r"\|" +AND = r"&" +EQUALS = r"=" # ? -CONDOP = r'\?' +CONDOP = r"\?" # Delimiters -LPAREN = r'\(' -RPAREN = r'\)' -LBRACKET = r'\[' -RBRACKET = r'\]' -LBRACE = r'\{' -RBRACE = r'\}' -COMMA = r',' -PERIOD = r'\.' -SEMI = r';' -COLON = r':' -BACKSLASH = r'\\' - -operators = { op: pattern for op, pattern in globals().items() if op == op.upper() } +LPAREN = r"\(" +RPAREN = r"\)" +LBRACKET = r"\[" +RBRACKET = r"\]" +LBRACE = r"\{" +RBRACE = r"\}" +COMMA = r"," +PERIOD = r"\." +SEMI = r";" +COLON = r":" +BACKSLASH = r"\\" + +operators = {op: pattern for op, pattern in globals().items() if op == op.upper()} for op in operators: globals()[op] = op -opmap = { pattern.replace("\\", "") or '\\' : op for op, pattern in operators.items() } +opmap = {pattern.replace("\\", "") or "\\": op for op, pattern in operators.items()} # Macros -macro = r'# *(ifdef|ifndef|undef|define|error|endif|if|else|include|#)' -MACRO = 'MACRO' +macro = r"# *(ifdef|ifndef|undef|define|error|endif|if|else|include|#)" +MACRO = "MACRO" -id_re = r'[a-zA-Z_][0-9a-zA-Z_]*' -IDENTIFIER = 'IDENTIFIER' +id_re = r"[a-zA-Z_][0-9a-zA-Z_]*" +IDENTIFIER = "IDENTIFIER" -suffix = r'([uU]?[lL]?[lL]?)' -octal = r'0[0-7]+' + suffix -hex = r'0[xX][0-9a-fA-F]+' -decimal_digits = r'(0|[1-9][0-9]*)' +suffix = r"([uU]?[lL]?[lL]?)" +octal = r"0[0-7]+" + suffix +hex = r"0[xX][0-9a-fA-F]+" +decimal_digits = r"(0|[1-9][0-9]*)" decimal = decimal_digits + suffix exponent = r"""([eE][-+]?[0-9]+)""" fraction = r"""([0-9]*\.[0-9]+)|([0-9]+\.)""" -float = '(((('+fraction+')'+exponent+'?)|([0-9]+'+exponent+'))[FfLl]?)' +float = "((((" + fraction + ")" + exponent + "?)|([0-9]+" + exponent + "))[FfLl]?)" number_re = choice(octal, hex, float, decimal) -NUMBER = 'NUMBER' +NUMBER = "NUMBER" simple_escape = r"""([a-zA-Z._~!=&\^\-\\?'"])""" decimal_escape = r"""(\d+)""" hex_escape = r"""(x[0-9a-fA-F]+)""" -escape_sequence = r"""(\\("""+simple_escape+'|'+decimal_escape+'|'+hex_escape+'))' -string_char = r"""([^"\\\n]|"""+escape_sequence+')' -str_re = '"'+string_char+'*"' -STRING = 'STRING' -char = r'\'.\'' # TODO: escape sequence -CHARACTER = 'CHARACTER' +escape_sequence = ( + r"""(\\(""" + simple_escape + "|" + decimal_escape + "|" + hex_escape + "))" +) +string_char = r"""([^"\\\n]|""" + escape_sequence + ")" +str_re = '"' + string_char + '*"' +STRING = "STRING" +char = r"\'.\'" # TODO: escape sequence +CHARACTER = "CHARACTER" -comment_re = r'//.*|/\*([^*]|\*[^/])*\*/' -COMMENT = 'COMMENT' +comment_re = r"//.*|/\*([^*]|\*[^/])*\*/" +COMMENT = "COMMENT" newline = r"\n" -invalid = r"\S" # A single non-space character that's not caught by any of the other patterns -matcher = re.compile(choice(id_re, number_re, str_re, char, newline, macro, comment_re, *operators.values(), invalid)) -letter = re.compile(r'[a-zA-Z_]') - -kwds = ( - 'AUTO', 'BREAK', 'CASE', 'CHAR', 'CONST', - 'CONTINUE', 'DEFAULT', 'DO', 'DOUBLE', 'ELSE', 'ENUM', 'EXTERN', - 'FLOAT', 'FOR', 'GOTO', 'IF', 'INLINE', 'INT', 'LONG', 'OVERRIDE', - 'REGISTER', 'OFFSETOF', - 'RESTRICT', 'RETURN', 'SHORT', 'SIGNED', 'SIZEOF', 'STATIC', 'STRUCT', - 'SWITCH', 'TYPEDEF', 'UNION', 'UNSIGNED', 'VOID', - 'VOLATILE', 'WHILE' +invalid = ( + r"\S" # A single non-space character that's not caught by any of the other patterns ) +matcher = re.compile( + choice( + id_re, + number_re, + str_re, + char, + newline, + macro, + comment_re, + *operators.values(), + invalid, + ) +) +letter = re.compile(r"[a-zA-Z_]") + + +kwds = [] +AUTO = "AUTO" +kwds.append(AUTO) +BREAK = "BREAK" +kwds.append(BREAK) +CASE = "CASE" +kwds.append(CASE) +CHAR = "CHAR" +kwds.append(CHAR) +CONST = "CONST" +kwds.append(CONST) +CONTINUE = "CONTINUE" +kwds.append(CONTINUE) +DEFAULT = "DEFAULT" +kwds.append(DEFAULT) +DO = "DO" +kwds.append(DO) +DOUBLE = "DOUBLE" +kwds.append(DOUBLE) +ELSE = "ELSE" +kwds.append(ELSE) +ENUM = "ENUM" +kwds.append(ENUM) +EXTERN = "EXTERN" +kwds.append(EXTERN) +FLOAT = "FLOAT" +kwds.append(FLOAT) +FOR = "FOR" +kwds.append(FOR) +GOTO = "GOTO" +kwds.append(GOTO) +IF = "IF" +kwds.append(IF) +INLINE = "INLINE" +kwds.append(INLINE) +INT = "INT" +kwds.append(INT) +LONG = "LONG" +kwds.append(LONG) +OVERRIDE = "OVERRIDE" +kwds.append(OVERRIDE) +REGISTER = "REGISTER" +kwds.append(REGISTER) +OFFSETOF = "OFFSETOF" +kwds.append(OFFSETOF) +RESTRICT = "RESTRICT" +kwds.append(RESTRICT) +RETURN = "RETURN" +kwds.append(RETURN) +SHORT = "SHORT" +kwds.append(SHORT) +SIGNED = "SIGNED" +kwds.append(SIGNED) +SIZEOF = "SIZEOF" +kwds.append(SIZEOF) +STATIC = "STATIC" +kwds.append(STATIC) +STRUCT = "STRUCT" +kwds.append(STRUCT) +SWITCH = "SWITCH" +kwds.append(SWITCH) +TYPEDEF = "TYPEDEF" +kwds.append(TYPEDEF) +UNION = "UNION" +kwds.append(UNION) +UNSIGNED = "UNSIGNED" +kwds.append(UNSIGNED) +VOID = "VOID" +kwds.append(VOID) +VOLATILE = "VOLATILE" +kwds.append(VOLATILE) +WHILE = "WHILE" +kwds.append(WHILE) +keywords = {name.lower(): name for name in kwds} -# For mypy -REGISTER = 'REGISTER' -OVERRIDE = 'OVERRIDE' -IF = 'IF' - -for name in kwds: - globals()[name] = name -keywords = { name.lower() : name for name in kwds } +__all__ = [] +__all__.extend(kwds) def make_syntax_error( - message: str, filename: str | None, line: int, column: int, line_text: str, + message: str, + filename: str | None, + line: int, + column: int, + line_text: str, ) -> SyntaxError: return SyntaxError(message, (filename, line, column, line_text)) @@ -169,7 +250,7 @@ def end_column(self) -> int: def width(self) -> int: return self.end[1] - self.begin[1] - def replaceText(self, txt: str) -> 'Token': + def replaceText(self, txt: str) -> "Token": assert isinstance(txt, str) return Token(self.kind, txt, self.begin, self.end) @@ -191,73 +272,75 @@ def tokenize(src: str, line: int = 1, filename: str | None = None) -> Iterator[T kind = keywords[text] elif letter.match(text): kind = IDENTIFIER - elif text == '...': + elif text == "...": kind = ELLIPSIS - elif text == '.': + elif text == ".": kind = PERIOD - elif text[0] in '0123456789.': + elif text[0] in "0123456789.": kind = NUMBER elif text[0] == '"': kind = STRING elif text in opmap: kind = opmap[text] - elif text == '\n': + elif text == "\n": linestart = start line += 1 - kind = '\n' + kind = "\n" elif text[0] == "'": kind = CHARACTER - elif text[0] == '#': + elif text[0] == "#": kind = MACRO - elif text[0] == '/' and text[1] in '/*': + elif text[0] == "/" and text[1] in "/*": kind = COMMENT else: lineend = src.find("\n", start) if lineend == -1: lineend = len(src) - raise make_syntax_error(f"Bad token: {text}", - filename, line, start-linestart+1, src[linestart:lineend]) + raise make_syntax_error( + f"Bad token: {text}", + filename, + line, + start - linestart + 1, + src[linestart:lineend], + ) if kind == COMMENT: - begin = line, start-linestart - newlines = text.count('\n') + begin = line, start - linestart + newlines = text.count("\n") if newlines: - linestart = start + text.rfind('\n') + linestart = start + text.rfind("\n") line += newlines else: - begin = line, start-linestart + begin = line, start - linestart if kind != "\n": - yield Token(kind, text, begin, (line, start-linestart+len(text))) - - -__all__ = [] -__all__.extend([kind for kind in globals() if kind.upper() == kind]) + yield Token(kind, text, begin, (line, start - linestart + len(text))) def to_text(tkns: list[Token], dedent: int = 0) -> str: res: list[str] = [] - line, col = -1, 1+dedent + line, col = -1, 1 + dedent for tkn in tkns: if line == -1: line, _ = tkn.begin l, c = tkn.begin - #assert(l >= line), (line, txt, start, end) + # assert(l >= line), (line, txt, start, end) while l > line: line += 1 - res.append('\n') - col = 1+dedent - res.append(' '*(c-col)) + res.append("\n") + col = 1 + dedent + res.append(" " * (c - col)) text = tkn.text - if dedent != 0 and tkn.kind == 'COMMENT' and '\n' in text: + if dedent != 0 and tkn.kind == "COMMENT" and "\n" in text: if dedent < 0: - text = text.replace('\n', '\n' + ' '*-dedent) + text = text.replace("\n", "\n" + " " * -dedent) # TODO: dedent > 0 res.append(text) line, col = tkn.end - return ''.join(res) + return "".join(res) if __name__ == "__main__": import sys + filename = sys.argv[1] if filename == "-c": src = sys.argv[2] diff --git a/Tools/cases_generator/mypy.ini b/Tools/cases_generator/mypy.ini index 7f56e66fe90232a..12409b8ad953ea0 100644 --- a/Tools/cases_generator/mypy.ini +++ b/Tools/cases_generator/mypy.ini @@ -8,4 +8,4 @@ python_version = 3.10 strict = True strict_concatenate = True enable_error_code = ignore-without-code,redundant-expr,truthy-bool -warn_unreachable = True +warn_unreachable = False diff --git a/Tools/cases_generator/requirements-dev.txt b/Tools/cases_generator/requirements-dev.txt deleted file mode 100644 index e9529f3527e95ed..000000000000000 --- a/Tools/cases_generator/requirements-dev.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Requirements file for external linters and checks we run on Tools/clinic/ in CI -mypy==1.4.1 diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index f9530f2fb94a4d4..3c87a5dcd783cda 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -413,16 +413,16 @@ def write_components( return next_instr_is_set -def write_single_instr_for_abstract_interp( - instr: Instruction, out: Formatter -) -> None: +def write_single_instr_for_abstract_interp(instr: Instruction, out: Formatter) -> None: try: _write_components_for_abstract_interp( [Component(instr, instr.active_caches)], out, ) except AssertionError as err: - raise AssertionError(f"Error writing abstract instruction {instr.name}") from err + raise AssertionError( + f"Error writing abstract instruction {instr.name}" + ) from err def _write_components_for_abstract_interp( @@ -438,5 +438,7 @@ def _write_components_for_abstract_interp( # NULL out the output stack effects for poke in mgr.pokes: if not poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: - out.emit(f"PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)" - f"PARTITIONNODE_NULLROOT, PEEK(-({poke.offset.as_index()})), true);") + out.emit( + f"PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)" + f"PARTITIONNODE_NULLROOT, PEEK(-({poke.offset.as_index()})), true);" + ) diff --git a/Tools/clinic/requirements-dev.txt b/Tools/clinic/requirements-dev.txt deleted file mode 100644 index e9529f3527e95ed..000000000000000 --- a/Tools/clinic/requirements-dev.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Requirements file for external linters and checks we run on Tools/clinic/ in CI -mypy==1.4.1 diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt new file mode 100644 index 000000000000000..279d8c2dcdab42f --- /dev/null +++ b/Tools/requirements-dev.txt @@ -0,0 +1,4 @@ +# Requirements file for external linters and checks we run on +# Tools/clinic and Tools/cases_generator/ in CI +mypy==1.5.1 +black