From e5289de20e494682f144977b3e4ea78e832b2ab0 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Fri, 18 Aug 2023 01:52:59 +0900 Subject: [PATCH] gh-104504: Run mypy on cases_generator --- .github/workflows/mypy.yml | 17 +++++++- Tools/cases_generator/analysis.py | 6 --- Tools/cases_generator/flags.py | 10 ++--- Tools/cases_generator/formatting.py | 10 ++--- Tools/cases_generator/generate_cases.py | 51 +++++++--------------- Tools/cases_generator/lexer.py | 28 +++++++----- Tools/cases_generator/parsing.py | 4 +- Tools/cases_generator/requirements-dev.txt | 2 + Tools/cases_generator/stacking.py | 4 +- 9 files changed, 65 insertions(+), 67 deletions(-) create mode 100644 Tools/cases_generator/requirements-dev.txt diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 1315bb5a966f018..54766e11285f863 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -8,6 +8,7 @@ on: pull_request: paths: - "Tools/clinic/**" + - "Tools/cases_generator/**" - ".github/workflows/mypy.yml" workflow_dispatch: @@ -24,7 +25,7 @@ concurrency: cancel-in-progress: true jobs: - mypy: + mypy-clinic: name: Run mypy on Tools/clinic/ runs-on: ubuntu-latest timeout-minutes: 10 @@ -37,3 +38,17 @@ jobs: cache-dependency-path: Tools/clinic/requirements-dev.txt - run: pip install -r Tools/clinic/requirements-dev.txt - run: mypy --config-file Tools/clinic/mypy.ini + + mypy-cases-generator: + name: Run mypy on Tools/cases_generator/ + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.x" + cache: pip + cache-dependency-path: Tools/cases_generator/requirements-dev.txt + - run: pip install -r Tools/cases_generator/requirements-dev.txt + - run: mypy --config-file Tools/cases_generator/mypy.ini diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 2db1cd01c19ae5d..e4d2945a1c31aa4 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -171,8 +171,6 @@ def parse_file(self, filename: str, instrs_idx: dict[str, int]) -> None: case parsing.Pseudo(name): self.pseudos[name] = thing self.everything.append(thing) - case _: - typing.assert_never(thing) if not psr.eof(): raise psr.make_syntax_error(f"Extra stuff at the end of {filename}") @@ -365,8 +363,6 @@ def analyze_macro(self, macro: parsing.Macro) -> MacroInstruction: part, offset = self.analyze_instruction(instr, offset) parts.append(part) flags.add(instr.instr_flags) - case _: - typing.assert_never(component) format = "IB" if offset: format += "C" + "0" * (offset - 1) @@ -406,6 +402,4 @@ def check_macro_components( components.append(self.instrs[name]) case parsing.CacheEffect(): components.append(uop) - case _: - typing.assert_never(uop) return components diff --git a/Tools/cases_generator/flags.py b/Tools/cases_generator/flags.py index 962f003b194dbde..79b88cc6dd8bdbf 100644 --- a/Tools/cases_generator/flags.py +++ b/Tools/cases_generator/flags.py @@ -16,11 +16,11 @@ class InstructionFlags: HAS_FREE_FLAG: bool HAS_LOCAL_FLAG: bool - def __post_init__(self): + def __post_init__(self) -> None: self.bitmask = {name: (1 << i) for i, name in enumerate(self.names())} @staticmethod - def fromInstruction(instr: parsing.Node): + def fromInstruction(instr: parsing.Node) -> 'InstructionFlags': has_free = ( variable_used(instr, "PyCell_New") @@ -41,7 +41,7 @@ def fromInstruction(instr: parsing.Node): ) @staticmethod - def newEmpty(): + def newEmpty() -> 'InstructionFlags': return InstructionFlags(False, False, False, False, False, False) def add(self, other: "InstructionFlags") -> None: @@ -49,7 +49,7 @@ def add(self, other: "InstructionFlags") -> None: if value: setattr(self, name, value) - def names(self, value=None) -> list[str]: + def names(self, value: bool|None = None) -> list[str]: if value is None: return list(dataclasses.asdict(self).keys()) return [n for n, v in dataclasses.asdict(self).items() if v == value] @@ -62,7 +62,7 @@ def bitmap(self) -> int: return flags @classmethod - def emit_macros(cls, out: Formatter): + def emit_macros(cls, out: Formatter) -> None: flags = cls.newEmpty() for name, value in flags.bitmask.items(): out.emit(f"#define {name} ({value})") diff --git a/Tools/cases_generator/formatting.py b/Tools/cases_generator/formatting.py index 5894751bd9635cd..d3a6da92def531a 100644 --- a/Tools/cases_generator/formatting.py +++ b/Tools/cases_generator/formatting.py @@ -58,13 +58,13 @@ def reset_lineno(self) -> None: self.set_lineno(self.lineno + 1, self.filename) @contextlib.contextmanager - def indent(self): + def indent(self) -> typing.Iterator[None]: self.prefix += " " yield self.prefix = self.prefix[:-4] @contextlib.contextmanager - def block(self, head: str, tail: str = ""): + def block(self, head: str, tail: str = "") -> typing.Iterator[None]: if head: self.emit(head + " {") else: @@ -77,7 +77,7 @@ def stack_adjust( self, input_effects: list[StackEffect], output_effects: list[StackEffect], - ): + ) -> None: shrink, isym = list_effect_size(input_effects) grow, osym = list_effect_size(output_effects) diff = grow - shrink @@ -90,7 +90,7 @@ def stack_adjust( if osym and osym != isym: self.emit(f"STACK_GROW({osym});") - def declare(self, dst: StackEffect, src: StackEffect | None): + def declare(self, dst: StackEffect, src: StackEffect | None) -> None: if dst.name == UNUSED or dst.cond == "0": return typ = f"{dst.type}" if dst.type else "PyObject *" @@ -107,7 +107,7 @@ def declare(self, dst: StackEffect, src: StackEffect | None): sepa = "" if typ.endswith("*") else " " self.emit(f"{typ}{sepa}{dst.name}{init};") - def assign(self, dst: StackEffect, src: StackEffect): + def assign(self, dst: StackEffect, src: StackEffect) -> None: if src.name == UNUSED or dst.name == UNUSED: return cast = self.cast(dst, src) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index f31b6658d6599e0..11f6cb3a5c9b647 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -23,7 +23,6 @@ MacroInstruction, MacroParts, PseudoInstruction, - StackEffect, OverriddenInstructionPlaceHolder, TIER_ONE, TIER_TWO, @@ -183,17 +182,11 @@ def effect_str(effects: list[StackEffect]) -> str: assert target_instr target_popped = effect_str(target_instr.input_effects) target_pushed = effect_str(target_instr.output_effects) - if popped is None and pushed is None: - popped, pushed = target_popped, target_pushed - else: - assert popped == target_popped - assert pushed == target_pushed - case _: - typing.assert_never(thing) + popped, pushed = target_popped, target_pushed return instr, popped, pushed @contextlib.contextmanager - def metadata_item(self, signature, open, close): + def metadata_item(self, signature: str, open: str, close: str) -> typing.Iterator[None]: self.out.emit("") self.out.emit(f"extern {signature};") self.out.emit("#ifdef NEED_OPCODE_METADATA") @@ -243,21 +236,21 @@ def from_source_files(self) -> str: paths = f"\n{self.out.comment} ".join(filenames) return f"{self.out.comment} from:\n{self.out.comment} {paths}\n" - def write_provenance_header(self): + def write_provenance_header(self) -> None: self.out.write_raw(f"{self.out.comment} This file is generated by {THIS}\n") self.out.write_raw(self.from_source_files()) self.out.write_raw(f"{self.out.comment} Do not edit!\n") - def assign_opcode_ids(self): + def assign_opcode_ids(self) -> None: """Assign IDs to opcodes""" - ops: list[(bool, str)] = [] # (has_arg, name) for each opcode + ops: list[tuple[bool, str]] = [] # (has_arg, name) for each opcode instrumented_ops: list[str] = [] for instr in itertools.chain( [instr for instr in self.instrs.values() if instr.kind != "op"], self.macro_instrs.values()): - + assert isinstance(instr, (Instruction, MacroInstruction, PseudoInstruction)) name = instr.name if name.startswith('INSTRUMENTED_'): instrumented_ops.append(name) @@ -274,11 +267,11 @@ def assign_opcode_ids(self): assert len(set(ops)) == len(ops) assert len(set(instrumented_ops)) == len(instrumented_ops) - opname: list[str or None] = [None] * 512 - opmap: dict = {} - markers: dict = {} + opname: list[str|None] = [None] * 512 + opmap: dict[str, int] = {} + markers: dict[str, int] = {} - def map_op(op, name): + def map_op(op: int, name: str) -> None: assert op < len(opname) assert opname[op] is None assert name not in opmap @@ -320,11 +313,11 @@ def map_op(op, name): for i, op in enumerate(sorted(self.pseudos)): map_op(256 + i, op) - assert 255 not in opmap # 255 is reserved + assert 255 not in opmap.values() # 255 is reserved self.opmap = opmap self.markers = markers - def write_opcode_ids(self, opcode_ids_h_filename, opcode_targets_filename): + def write_opcode_ids(self, opcode_ids_h_filename: str, opcode_targets_filename: str) -> None: """Write header file that defined the opcode IDs""" with open(opcode_ids_h_filename, "w") as f: @@ -342,10 +335,10 @@ def write_opcode_ids(self, opcode_ids_h_filename, opcode_targets_filename): self.out.emit("") self.out.emit("/* Instruction opcodes for compiled code */") - def define(name, opcode): + def define(name: str, opcode: int) -> None: self.out.emit(f"#define {name:<38} {opcode:>3}") - all_pairs = [] + all_pairs: list[tuple[int, int, str]] = [] # the second item in the tuple sorts the markers before the ops all_pairs.extend((i, 1, name) for (name, i) in self.markers.items()) all_pairs.extend((i, 2, name) for (name, i) in self.opmap.items()) @@ -392,11 +385,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No assert target_instr if format is None: format = target_instr.instr_fmt - else: - assert format == target_instr.instr_fmt assert format is not None - case _: - typing.assert_never(thing) all_formats.add(format) # Turn it into a sorted list of enum values. @@ -483,8 +472,6 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No self.write_metadata_for_macro(self.macro_instrs[thing.name]) case parsing.Pseudo(): self.write_metadata_for_pseudo(self.pseudo_instrs[thing.name]) - case _: - typing.assert_never(thing) with self.metadata_item( "const struct opcode_macro_expansion " @@ -520,8 +507,6 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No ) case parsing.Pseudo(): pass - case _: - typing.assert_never(thing) with self.metadata_item( "const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE]", "=", ";" @@ -770,8 +755,6 @@ def write_instructions( # self.write_macro(self.macro_instrs[thing.name]) case parsing.Pseudo(): pass - case _: - typing.assert_never(thing) print( f"Wrote {n_instrs} instructions and {n_macros} macros " @@ -814,8 +797,6 @@ def write_executor_instructions( pass case parsing.Pseudo(): pass - case _: - typing.assert_never(thing) print( f"Wrote {n_instrs} instructions and {n_uops} ops to {executor_filename}", file=sys.stderr, @@ -843,8 +824,6 @@ def write_abstract_interpreter_instructions( pass case parsing.Pseudo(): pass - case _: - typing.assert_never(thing) print( f"Wrote some stuff to {abstract_interpreter_filename}", file=sys.stderr, @@ -878,7 +857,7 @@ def write_instr(self, instr: Instruction) -> None: self.out.emit(f"DISPATCH();") -def main(): +def main() -> None: """Parse command line, parse input, analyze, write output.""" args = arg_parser.parse_args() # Prints message and sys.exit(2) on error if len(args.input) == 0: diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index fe9c05ede5aa475..d0e3be9051caf1c 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -4,8 +4,10 @@ import re from dataclasses import dataclass +from collections.abc import Iterator -def choice(*opts): + +def choice(*opts: str) -> str: return "|".join("(%s)" % opt for opt in opts) # Regexes @@ -123,13 +125,19 @@ def choice(*opts): 'SWITCH', 'TYPEDEF', 'UNION', 'UNSIGNED', 'VOID', 'VOLATILE', 'WHILE' ) + +# For mypy +REGISTER = 'REGISTER' +OVERRIDE = 'OVERRIDE' +IF = 'IF' + for name in kwds: globals()[name] = name keywords = { name.lower() : name for name in kwds } def make_syntax_error( - message: str, filename: str, line: int, column: int, line_text: str, + message: str, filename: str | None, line: int, column: int, line_text: str, ) -> SyntaxError: return SyntaxError(message, (filename, line, column, line_text)) @@ -142,30 +150,30 @@ class Token: end: tuple[int, int] @property - def line(self): + def line(self) -> int: return self.begin[0] @property - def column(self): + def column(self) -> int: return self.begin[1] @property - def end_line(self): + def end_line(self) -> int: return self.end[0] @property - def end_column(self): + def end_column(self) -> int: return self.end[1] @property - def width(self): + def width(self) -> int: return self.end[1] - self.begin[1] - def replaceText(self, txt): + def replaceText(self, txt: str) -> 'Token': assert isinstance(txt, str) return Token(self.kind, txt, self.begin, self.end) - def __repr__(self): + def __repr__(self) -> str: b0, b1 = self.begin e0, e1 = self.end if b0 == e0: @@ -174,7 +182,7 @@ def __repr__(self): return f"{self.kind}({self.text!r}, {b0}:{b1}, {e0}:{e1})" -def tokenize(src, line=1, filename=None): +def tokenize(src: str, line: int = 1, filename: str | None = None) -> Iterator[Token]: linestart = -1 for m in matcher.finditer(src): start, end = m.span() diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index cdd20d7a0b3f59c..25de3a5adc2cf98 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -32,7 +32,7 @@ class Context(NamedTuple): end: int owner: PLexer - def __repr__(self): + def __repr__(self) -> str: return f"<{self.owner.filename}: {self.begin}-{self.end}>" @@ -75,7 +75,7 @@ class StackEffect(Node): size: str = "" # Optional `[size]` # Note: size cannot be combined with type or cond - def __repr__(self): + def __repr__(self) -> str: items = [self.name, self.type, self.cond, self.size] while items and items[-1] == "": del items[-1] diff --git a/Tools/cases_generator/requirements-dev.txt b/Tools/cases_generator/requirements-dev.txt new file mode 100644 index 000000000000000..e9529f3527e95ed --- /dev/null +++ b/Tools/cases_generator/requirements-dev.txt @@ -0,0 +1,2 @@ +# Requirements file for external linters and checks we run on Tools/clinic/ in CI +mypy==1.4.1 diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 8361eb99f88a7cc..f9530f2fb94a4d4 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -415,7 +415,7 @@ def write_components( def write_single_instr_for_abstract_interp( instr: Instruction, out: Formatter -): +) -> None: try: _write_components_for_abstract_interp( [Component(instr, instr.active_caches)], @@ -428,7 +428,7 @@ def write_single_instr_for_abstract_interp( def _write_components_for_abstract_interp( parts: list[Component], out: Formatter, -): +) -> None: managers = get_managers(parts) for mgr in managers: if mgr is managers[-1]: