From 52e20a61550e36e527b39d5dfddddc1bb115b291 Mon Sep 17 00:00:00 2001 From: Matej Aleksandrov Date: Fri, 21 Feb 2025 06:03:30 -0800 Subject: [PATCH] Rebase Pyink to Black 25.1.0 Additionally, disable Preview.always_one_newline_after_import formatting feature as it is not compatible with formatting of line ranges. PiperOrigin-RevId: 729501835 --- patches/pyink.patch | 307 ++++++++---------- pyproject.toml | 4 +- src/pyink/__init__.py | 26 +- src/pyink/brackets.py | 3 +- src/pyink/cache.py | 3 +- src/pyink/comments.py | 11 +- src/pyink/concurrency.py | 3 +- src/pyink/debug.py | 3 +- src/pyink/files.py | 13 +- src/pyink/handle_ipynb_magics.py | 27 +- src/pyink/ink_comments.py | 13 +- src/pyink/linegen.py | 184 ++++++----- src/pyink/lines.py | 16 +- src/pyink/mode.py | 18 +- src/pyink/nodes.py | 40 ++- src/pyink/parsing.py | 2 +- src/pyink/ranges.py | 3 +- src/pyink/resources/pyink.schema.json | 10 +- src/pyink/strings.py | 11 +- src/pyink/trans.py | 95 +++--- tests/data/cases/annotations.py | 17 + .../cases/{preview_cantfit.py => cantfit.py} | 1 - tests/data/cases/context_managers_38.py | 1 - tests/data/cases/context_managers_39.py | 1 - .../cases/context_managers_autodetect_39.py | 1 - ...ewline_preview.py => docstring_newline.py} | 1 - tests/data/cases/fmtskip9.py | 1 - ...pe_seq.py => format_unicode_escape_seq.py} | 1 - .../funcdef_return_type_trailing_comma.py | 2 +- tests/data/cases/generics_wrapping.py | 307 ++++++++++++++++++ ...simple_lookup_for_doublestar_expression.py | 1 - ...s.py => long_strings__type_annotations.py} | 1 - tests/data/cases/module_docstring_2.py | 5 +- .../cases/no_blank_line_before_docstring.py | 1 - .../cases/pattern_matching_with_if_stmt.py | 2 +- ...> pep646_typed_star_arg_type_var_tuple.py} | 2 +- tests/data/cases/pep_570.py | 1 - tests/data/cases/pep_572.py | 1 - tests/data/cases/pep_572_py39.py | 1 - tests/data/cases/pep_572_remove_parens.py | 1 - .../cases/prefer_rhs_split_reformatted.py | 19 ++ tests/data/cases/preview_comments7.py | 1 - tests/data/cases/preview_fstring.py | 2 + .../cases/preview_import_line_collapse.py | 180 ++++++++++ tests/data/cases/preview_long_dict_values.py | 200 +++++++++++- tests/data/cases/preview_long_strings.py | 42 ++- .../cases/preview_long_strings__regression.py | 13 +- ..._remove_multiline_lone_list_item_parens.py | 246 ++++++++++++++ tests/data/cases/python37.py | 3 - tests/data/cases/python38.py | 3 - tests/data/cases/python39.py | 2 - .../cases/remove_lone_list_item_parens.py | 157 +++++++++ .../remove_redundant_parens_in_case_guard.py | 2 +- tests/data/cases/remove_with_brackets.py | 27 +- .../skip_magic_trailing_comma_generic_wrap.py | 163 ++++++++++ tests/data/cases/type_param_defaults.py | 22 +- .../data/cases/typed_params_trailing_comma.py | 1 - tests/data/cases/walrus_in_dict.py | 4 +- tests/optional.py | 12 +- tests/test_black.py | 43 +-- tests/test_docs.py | 5 +- tests/test_format.py | 3 +- tests/test_ipynb.py | 8 +- tests/test_ranges.py | 14 +- tests/test_tokenize.py | 7 +- tests/test_trans.py | 4 +- tests/util.py | 29 +- tox.ini | 2 +- 68 files changed, 1815 insertions(+), 540 deletions(-) create mode 100644 tests/data/cases/annotations.py rename tests/data/cases/{preview_cantfit.py => cantfit.py} (99%) rename tests/data/cases/{docstring_newline_preview.py => docstring_newline.py} (83%) rename tests/data/cases/{preview_format_unicode_escape_seq.py => format_unicode_escape_seq.py} (96%) create mode 100644 tests/data/cases/generics_wrapping.py rename tests/data/cases/{preview_long_strings__type_annotations.py => long_strings__type_annotations.py} (98%) rename tests/data/cases/{preview_pep646_typed_star_arg_type_var_tuple.py => pep646_typed_star_arg_type_var_tuple.py} (62%) create mode 100644 tests/data/cases/preview_fstring.py create mode 100644 tests/data/cases/preview_import_line_collapse.py create mode 100644 tests/data/cases/preview_remove_multiline_lone_list_item_parens.py create mode 100644 tests/data/cases/remove_lone_list_item_parens.py create mode 100644 tests/data/cases/skip_magic_trailing_comma_generic_wrap.py diff --git a/patches/pyink.patch b/patches/pyink.patch index 63b1f3fc448..6d374f719b5 100644 --- a/patches/pyink.patch +++ b/patches/pyink.patch @@ -2,16 +2,16 @@ # This file is provided so it's easier to see the actual differences between Black and Pyink. --- a/__init__.py +++ b/__init__.py -@@ -24,6 +24,8 @@ from typing import ( - Union, - ) +@@ -22,6 +22,8 @@ from pathlib import Path + from re import Pattern + from typing import Any, Optional, Union +from blib2to3.pgen2 import token +from blib2to3.pytree import Leaf, Node import click from click.core import ParameterSource from mypy_extensions import mypyc_attr -@@ -64,7 +66,13 @@ from pyink.linegen import LN, LineGenera +@@ -62,7 +64,13 @@ from pyink.linegen import LN, LineGenera from pyink.lines import EmptyLineTracker, LinesBlock from pyink.mode import FUTURE_FLAG_TO_FEATURE, VERSION_TO_FEATURES, Feature from pyink.mode import Mode as Mode # re-exported @@ -26,7 +26,7 @@ from pyink.nodes import STARS, is_number_token, is_simple_decorator_expression, syms from pyink.output import color_diff, diff, dump_to_file, err, ipynb_diff, out from pyink.parsing import ( # noqa F401 -@@ -80,9 +88,8 @@ from pyink.ranges import ( +@@ -78,9 +86,8 @@ from pyink.ranges import ( parse_line_ranges, sanitized_lines, ) @@ -37,7 +37,7 @@ COMPILED = Path(__file__).suffix in (".pyd", ".so") -@@ -338,6 +345,61 @@ def validate_regex( +@@ -336,6 +343,61 @@ def validate_regex( ), ) @click.option( @@ -99,7 +99,7 @@ "--check", is_flag=True, help=( -@@ -530,6 +592,12 @@ def main( # noqa: C901 +@@ -528,6 +590,12 @@ def main( # noqa: C901 preview: bool, unstable: bool, enable_unstable_feature: list[Preview], @@ -112,7 +112,7 @@ quiet: bool, verbose: bool, required_version: Optional[str], -@@ -636,7 +704,16 @@ def main( # noqa: C901 +@@ -634,7 +702,16 @@ def main( # noqa: C901 preview=preview, unstable=unstable, python_cell_magics=set(python_cell_magics), @@ -130,7 +130,7 @@ ) lines: list[tuple[int, int]] = [] -@@ -1132,6 +1209,17 @@ def validate_metadata(nb: MutableMapping +@@ -1136,6 +1213,17 @@ def validate_metadata(nb: MutableMapping if language is not None and language != "python": raise NothingChanged from None @@ -148,7 +148,7 @@ def format_ipynb_string(src_contents: str, *, fast: bool, mode: Mode) -> FileContent: """Format Jupyter notebook. -@@ -1143,7 +1231,6 @@ def format_ipynb_string(src_contents: st +@@ -1147,7 +1235,6 @@ def format_ipynb_string(src_contents: st raise NothingChanged trailing_newline = src_contents[-1] == "\n" @@ -156,7 +156,7 @@ nb = json.loads(src_contents) validate_metadata(nb) for cell in nb["cells"]: -@@ -1155,14 +1242,17 @@ def format_ipynb_string(src_contents: st +@@ -1159,14 +1246,17 @@ def format_ipynb_string(src_contents: st pass else: cell["source"] = dst.splitlines(keepends=True) @@ -182,7 +182,7 @@ def format_str( -@@ -1223,6 +1313,8 @@ def _format_str_once( +@@ -1227,6 +1317,8 @@ def _format_str_once( future_imports = get_future_imports(src_node) versions = detect_target_versions(src_node, future_imports=future_imports) @@ -210,15 +210,15 @@ +) --- a/comments.py +++ b/comments.py -@@ -3,6 +3,7 @@ from dataclasses import dataclass +@@ -4,6 +4,7 @@ from dataclasses import dataclass from functools import lru_cache - from typing import Collection, Final, Iterator, Optional, Union + from typing import Final, Optional, Union +from pyink import ink_comments - from pyink.mode import Mode, Preview + from pyink.mode import Mode from pyink.nodes import ( CLOSING_BRACKETS, -@@ -376,7 +377,7 @@ def children_contains_fmt_on(container: +@@ -373,7 +374,7 @@ def children_contains_fmt_on(container: return False @@ -227,7 +227,7 @@ """ Returns: True iff one of the comments in @comment_list is a pragma used by one -@@ -384,7 +385,7 @@ def contains_pragma_comment(comment_list +@@ -381,7 +382,7 @@ def contains_pragma_comment(comment_list pylint). """ for comment in comment_list: @@ -238,7 +238,7 @@ return False --- a/files.py +++ b/files.py -@@ -228,7 +228,7 @@ def strip_specifier_set(specifier_set: S +@@ -221,7 +221,7 @@ def strip_specifier_set(specifier_set: S def find_user_pyproject_toml() -> Path: r"""Return the path to the top-level user configuration for pyink. @@ -249,45 +249,20 @@ May raise: --- a/handle_ipynb_magics.py +++ b/handle_ipynb_magics.py -@@ -178,12 +178,14 @@ def mask_cell(src: str) -> tuple[str, li - from IPython.core.inputtransformer2 import TransformerManager - - transformer_manager = TransformerManager() -+ # A side effect of the following transformation is that it also removes any -+ # empty lines at the beginning of the cell. - transformed = transformer_manager.transform_cell(src) - transformed, cell_magic_replacements = replace_cell_magics(transformed) - replacements += cell_magic_replacements - transformed = transformer_manager.transform_cell(transformed) - transformed, magic_replacements = replace_magics(transformed) -- if len(transformed.splitlines()) != len(src.splitlines()): -+ if len(transformed.strip().splitlines()) != len(src.strip().splitlines()): - # Multi-line magic, not supported. - raise NothingChanged - replacements += magic_replacements -@@ -269,7 +271,7 @@ def replace_magics(src: str) -> tuple[st - magic_finder = MagicFinder() - magic_finder.visit(ast.parse(src)) - new_srcs = [] -- for i, line in enumerate(src.splitlines(), start=1): -+ for i, line in enumerate(src.split("\n"), start=1): - if i in magic_finder.magics: - offsets_and_magics = magic_finder.magics[i] - if len(offsets_and_magics) != 1: # pragma: nocover -@@ -303,6 +305,8 @@ def unmask_cell(src: str, replacements: +@@ -314,6 +314,8 @@ def unmask_cell(src: str, replacements: """ for replacement in replacements: src = src.replace(replacement.mask, replacement.src) + # Strings in src might have been reformatted with single quotes. -+ src = src.replace(f"'{replacement.mask[1:-1]}'", replacement.src) ++ src = src.replace(f"b'{replacement.mask[2:-1]}'", replacement.src) return src --- a/linegen.py +++ b/linegen.py -@@ -9,6 +9,12 @@ from enum import Enum, auto +@@ -10,6 +10,12 @@ from enum import Enum, auto from functools import partial, wraps - from typing import Collection, Iterator, Optional, Union, cast + from typing import Optional, Union, cast +if sys.version_info < (3, 8): + from typing_extensions import Final, Literal @@ -298,7 +273,7 @@ from pyink.brackets import ( COMMA_PRIORITY, DOT_PRIORITY, -@@ -18,6 +24,7 @@ from pyink.brackets import ( +@@ -19,6 +25,7 @@ from pyink.brackets import ( ) from pyink.comments import FMT_OFF, generate_comments, list_comments from pyink.lines import ( @@ -306,15 +281,15 @@ Line, RHSResult, append_leaves, -@@ -55,7 +62,6 @@ from pyink.nodes import ( - is_stub_body, +@@ -58,7 +65,6 @@ from pyink.nodes import ( is_stub_suite, + is_tuple_containing_star, is_tuple_containing_walrus, - is_type_ignore_comment_string, is_vararg, is_walrus_assignment, is_yield, -@@ -87,6 +93,15 @@ LeafID = int +@@ -90,6 +96,15 @@ LeafID = int LN = Union[Leaf, Node] @@ -330,7 +305,7 @@ class CannotSplit(CannotTransform): """A readable split that fits the allotted line length is impossible.""" -@@ -106,7 +121,9 @@ class LineGenerator(Visitor[Line]): +@@ -109,7 +124,9 @@ class LineGenerator(Visitor[Line]): self.current_line: Line self.__post_init__() @@ -341,7 +316,7 @@ """Generate a line. If the line is empty, only emit if it makes sense. -@@ -115,7 +132,10 @@ class LineGenerator(Visitor[Line]): +@@ -118,7 +135,10 @@ class LineGenerator(Visitor[Line]): If any lines were generated, set up a new current_line. """ if not self.current_line: @@ -353,7 +328,7 @@ return # Line is empty, don't emit. Creating a new one unnecessary. if len(self.current_line.leaves) == 1 and is_async_stmt_or_funcdef( -@@ -128,7 +148,13 @@ class LineGenerator(Visitor[Line]): +@@ -131,7 +151,13 @@ class LineGenerator(Visitor[Line]): return complete_line = self.current_line @@ -368,7 +343,7 @@ yield complete_line def visit_default(self, node: LN) -> Iterator[Line]: -@@ -160,26 +186,27 @@ class LineGenerator(Visitor[Line]): +@@ -163,26 +189,27 @@ class LineGenerator(Visitor[Line]): def visit_test(self, node: Node) -> Iterator[Line]: """Visit an `x if y else z` test""" @@ -409,7 +384,7 @@ yield from self.visit_default(node) def visit_DEDENT(self, node: Leaf) -> Iterator[Line]: -@@ -194,7 +221,7 @@ class LineGenerator(Visitor[Line]): +@@ -197,7 +224,7 @@ class LineGenerator(Visitor[Line]): yield from self.visit_default(node) # Finally, emit the dedent. @@ -418,7 +393,7 @@ def visit_stmt( self, node: Node, keywords: set[str], parens: set[str] -@@ -245,6 +272,7 @@ class LineGenerator(Visitor[Line]): +@@ -248,6 +275,7 @@ class LineGenerator(Visitor[Line]): maybe_make_parens_invisible_in_atom( child, parent=node, @@ -426,7 +401,7 @@ remove_brackets_around_comma=False, ) else: -@@ -265,6 +293,7 @@ class LineGenerator(Visitor[Line]): +@@ -268,6 +296,7 @@ class LineGenerator(Visitor[Line]): if maybe_make_parens_invisible_in_atom( child, parent=node, @@ -434,7 +409,7 @@ remove_brackets_around_comma=False, ): wrap_in_parentheses(node, child, visible=False) -@@ -287,7 +316,9 @@ class LineGenerator(Visitor[Line]): +@@ -290,7 +319,9 @@ class LineGenerator(Visitor[Line]): def visit_suite(self, node: Node) -> Iterator[Line]: """Visit a suite.""" @@ -445,7 +420,7 @@ yield from self.visit(node.children[2]) else: yield from self.visit_default(node) -@@ -301,15 +332,23 @@ class LineGenerator(Visitor[Line]): +@@ -304,15 +335,23 @@ class LineGenerator(Visitor[Line]): prev_type = child.type if node.parent and node.parent.type in STATEMENT: @@ -473,7 +448,7 @@ node.prefix = "" yield from self.visit_default(node) return -@@ -358,7 +397,7 @@ class LineGenerator(Visitor[Line]): +@@ -361,7 +400,7 @@ class LineGenerator(Visitor[Line]): ): wrap_in_parentheses(node, leaf) @@ -482,7 +457,7 @@ yield from self.visit_default(node) -@@ -405,13 +444,18 @@ class LineGenerator(Visitor[Line]): +@@ -408,13 +447,16 @@ class LineGenerator(Visitor[Line]): def foo(a: (int), b: (float) = 7): ... """ assert len(node.children) == 3 @@ -495,15 +470,13 @@ yield from self.visit_default(node) def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: -- if Preview.hex_codes_in_unicode_sequences in self.mode: -+ if ( -+ Preview.hex_codes_in_unicode_sequences in self.mode -+ and not self.mode.is_pyink -+ ): - normalize_unicode_escape_sequences(leaf) +- normalize_unicode_escape_sequences(leaf) ++ if not self.mode.is_pyink: ++ normalize_unicode_escape_sequences(leaf) - if is_docstring(leaf, self.mode) and not re.search(r"\\\s*\n", leaf.value): -@@ -424,7 +468,9 @@ class LineGenerator(Visitor[Line]): + if is_docstring(leaf) and not re.search(r"\\\s*\n", leaf.value): + # We're ignoring docstrings with backslash newline escapes because changing +@@ -426,7 +468,9 @@ class LineGenerator(Visitor[Line]): # see padding logic below), there's a possibility for unstable # formatting. To avoid a situation where this function formats a # docstring differently on the second pass, normalize it early. @@ -514,7 +487,7 @@ else: docstring = leaf.value prefix = get_string_prefix(docstring) -@@ -438,7 +484,7 @@ class LineGenerator(Visitor[Line]): +@@ -440,7 +484,7 @@ class LineGenerator(Visitor[Line]): quote_len = 1 if docstring[1] != quote_char else 3 docstring = docstring[quote_len:-quote_len] docstring_started_empty = not docstring @@ -522,8 +495,8 @@ + indent = " " * self.current_line.indentation_spaces() if is_multiline_string(leaf): - docstring = fix_docstring(docstring, indent) -@@ -473,7 +519,13 @@ class LineGenerator(Visitor[Line]): + docstring = fix_multiline_docstring(docstring, indent) +@@ -475,7 +519,13 @@ class LineGenerator(Visitor[Line]): # If docstring is one line, we don't put the closing quotes on a # separate line because it looks ugly (#3320). lines = docstring.splitlines() @@ -538,7 +511,7 @@ # If adding closing quotes would cause the last line to exceed # the maximum line length, and the closing quote is not -@@ -499,7 +551,9 @@ class LineGenerator(Visitor[Line]): +@@ -498,7 +548,9 @@ class LineGenerator(Visitor[Line]): if self.mode.string_normalization and leaf.type == token.STRING: leaf.value = normalize_string_prefix(leaf.value) @@ -549,7 +522,7 @@ yield from self.visit_default(leaf) def visit_NUMBER(self, leaf: Leaf) -> Iterator[Line]: -@@ -575,7 +629,8 @@ class LineGenerator(Visitor[Line]): +@@ -587,7 +639,8 @@ class LineGenerator(Visitor[Line]): self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS) self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"}) @@ -559,7 +532,7 @@ self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"}) self.visit_async_funcdef = self.visit_async_stmt self.visit_decorated = self.visit_decorators -@@ -621,14 +676,23 @@ def transform_line( +@@ -632,14 +685,23 @@ def transform_line( ll = mode.line_length sn = mode.string_normalization @@ -588,7 +561,7 @@ and not line.should_split_rhs and not line.magic_trailing_comma and ( -@@ -831,7 +895,6 @@ def _first_right_hand_split( +@@ -845,7 +907,6 @@ def _first_right_hand_split( omit: Collection[LeafID] = (), ) -> RHSResult: """Split the line into head, body, tail starting with the last bracket pair. @@ -596,7 +569,7 @@ Note: this function should not have side effects. It's relied upon by _maybe_split_omitting_optional_parens to get an opinion whether to prefer splitting on the right side of an assignment statement. -@@ -1139,7 +1202,7 @@ def bracket_split_build_line( +@@ -1167,7 +1228,7 @@ def bracket_split_build_line( result = Line(mode=original.mode, depth=original.depth) if component is _BracketSplitComponent.body: result.inside_brackets = True @@ -605,7 +578,7 @@ if _ensure_trailing_comma(leaves, original, opening_bracket): for i in range(len(leaves) - 1, -1, -1): if leaves[i].type == STANDALONE_COMMENT: -@@ -1408,15 +1471,17 @@ def normalize_invisible_parens( # noqa: +@@ -1432,15 +1493,17 @@ def normalize_invisible_parens( # noqa: if maybe_make_parens_invisible_in_atom( child, parent=node, @@ -624,7 +597,7 @@ ): wrap_in_parentheses(node, child, visible=False) elif is_one_tuple(child): -@@ -1468,7 +1533,7 @@ def _normalize_import_from(parent: Node, +@@ -1492,7 +1555,7 @@ def _normalize_import_from(parent: Node, parent.append_child(Leaf(token.RPAR, "")) @@ -633,7 +606,7 @@ if node.children[0].type == token.AWAIT and len(node.children) > 1: if ( node.children[1].type == syms.atom -@@ -1477,6 +1542,7 @@ def remove_await_parens(node: Node) -> N +@@ -1501,6 +1564,7 @@ def remove_await_parens(node: Node) -> N if maybe_make_parens_invisible_in_atom( node.children[1], parent=node, @@ -641,7 +614,7 @@ remove_brackets_around_comma=True, ): wrap_in_parentheses(node, node.children[1], visible=False) -@@ -1545,7 +1611,7 @@ def _maybe_wrap_cms_in_parens( +@@ -1569,7 +1633,7 @@ def _maybe_wrap_cms_in_parens( node.insert_child(1, new_child) @@ -650,7 +623,7 @@ """Recursively hide optional parens in `with` statements.""" # Removing all unnecessary parentheses in with statements in one pass is a tad # complex as different variations of bracketed statements result in pretty -@@ -1567,21 +1633,23 @@ def remove_with_parens(node: Node, paren +@@ -1591,21 +1655,23 @@ def remove_with_parens(node: Node, paren if maybe_make_parens_invisible_in_atom( node, parent=parent, @@ -676,15 +649,15 @@ remove_brackets_around_comma=True, ): wrap_in_parentheses(node, node.children[0], visible=False) -@@ -1590,6 +1658,7 @@ def remove_with_parens(node: Node, paren +@@ -1614,6 +1680,7 @@ def remove_with_parens(node: Node, paren def maybe_make_parens_invisible_in_atom( node: LN, parent: LN, -+ mode: Mode, ++ mode: Optional[Mode] = None, remove_brackets_around_comma: bool = False, ) -> bool: """If it's safe, make the parens in the atom `node` invisible, recursively. -@@ -1639,7 +1708,7 @@ def maybe_make_parens_invisible_in_atom( +@@ -1665,13 +1732,14 @@ def maybe_make_parens_invisible_in_atom( if ( # If the prefix of `middle` includes a type comment with # ignore annotation, then we do not remove the parentheses @@ -692,8 +665,7 @@ + not ink_comments.comment_contains_pragma(middle.prefix.strip(), mode) ): first.value = "" - if first.prefix.strip(): -@@ -1649,6 +1718,7 @@ def maybe_make_parens_invisible_in_atom( + last.value = "" maybe_make_parens_invisible_in_atom( middle, parent=parent, @@ -701,7 +673,7 @@ remove_brackets_around_comma=remove_brackets_around_comma, ) -@@ -1707,7 +1777,7 @@ def generate_trailers_to_omit(line: Line +@@ -1737,7 +1805,7 @@ def generate_trailers_to_omit(line: Line if not line.magic_trailing_comma: yield omit @@ -710,7 +682,7 @@ opening_bracket: Optional[Leaf] = None closing_bracket: Optional[Leaf] = None inner_brackets: set[LeafID] = set() -@@ -1792,7 +1862,7 @@ def run_transformer( +@@ -1822,7 +1890,7 @@ def run_transformer( or not line.bracket_tracker.invisible or any(bracket.value for bracket in line.bracket_tracker.invisible) or line.contains_multiline_strings() @@ -725,8 +697,8 @@ +from enum import Enum, auto import itertools import math - from dataclasses import dataclass, field -@@ -17,7 +18,7 @@ from pyink.nodes import ( + from collections.abc import Callable, Iterator, Sequence +@@ -18,7 +19,7 @@ from pyink.nodes import ( is_multiline_string, is_one_sequence_between, is_type_comment, @@ -735,7 +707,7 @@ is_with_or_async_with_stmt, make_simple_prefix, replace_child, -@@ -35,12 +36,24 @@ LeafID = int +@@ -36,12 +37,24 @@ LeafID = int LN = Union[Leaf, Node] @@ -761,7 +733,7 @@ leaves: list[Leaf] = field(default_factory=list) # keys ordered like `leaves` comments: dict[LeafID, list[Leaf]] = field(default_factory=dict) -@@ -49,6 +62,9 @@ class Line: +@@ -50,6 +63,9 @@ class Line: should_split_rhs: bool = False magic_trailing_comma: Optional[Leaf] = None @@ -771,7 +743,7 @@ def append( self, leaf: Leaf, preformatted: bool = False, track_bracket: bool = False ) -> None: -@@ -97,7 +113,7 @@ class Line: +@@ -98,7 +114,7 @@ class Line: or when a standalone comment is not the first leaf on the line. """ if ( @@ -780,7 +752,7 @@ or self.bracket_tracker.any_open_for_or_lambda() ): if self.is_comment: -@@ -262,7 +278,14 @@ class Line: +@@ -261,7 +277,14 @@ class Line: return True return False @@ -796,7 +768,7 @@ ignored_ids = set() try: last_leaf = self.leaves[-1] -@@ -287,11 +310,9 @@ class Line: +@@ -286,11 +309,9 @@ class Line: comment_seen = False for leaf_id, comments in self.comments.items(): for comment in comments: @@ -811,7 +783,7 @@ return True comment_seen = True -@@ -326,7 +347,7 @@ class Line: +@@ -325,7 +346,7 @@ class Line: # line. for node in self.leaves[-2:]: for comment in self.comments.get(id(node), []): @@ -820,7 +792,7 @@ return True return False -@@ -481,7 +502,7 @@ class Line: +@@ -480,7 +501,7 @@ class Line: if not self: return "\n" @@ -829,7 +801,7 @@ leaves = iter(self.leaves) first = next(leaves) res = f"{first.prefix}{indent}{first.value}" -@@ -553,7 +574,7 @@ class EmptyLineTracker: +@@ -552,7 +573,7 @@ class EmptyLineTracker: lines (two on module-level). """ form_feed = ( @@ -838,7 +810,7 @@ and bool(current_line.leaves) and "\f\n" in current_line.leaves[0].prefix ) -@@ -598,7 +619,7 @@ class EmptyLineTracker: +@@ -597,7 +618,7 @@ class EmptyLineTracker: def _maybe_empty_lines(self, current_line: Line) -> tuple[int, int]: # noqa: C901 max_allowed = 1 @@ -847,7 +819,7 @@ max_allowed = 1 if self.mode.is_pyi else 2 if current_line.leaves: -@@ -615,7 +636,7 @@ class EmptyLineTracker: +@@ -614,7 +635,7 @@ class EmptyLineTracker: # Mutate self.previous_defs, remainder of this function should be pure previous_def = None @@ -856,8 +828,18 @@ previous_def = self.previous_defs.pop() if current_line.is_def or current_line.is_class: self.previous_defs.append(current_line) -@@ -671,10 +692,25 @@ class EmptyLineTracker: - ) +@@ -671,18 +692,33 @@ class EmptyLineTracker: + + if ( + self.previous_line.is_import +- and self.previous_line.depth == 0 +- and current_line.depth == 0 ++ and not self.previous_line.depth ++ and not current_line.depth + and not current_line.is_import + and Preview.always_one_newline_after_import in self.mode + ): + return 1, 0 if ( - self.previous_line.is_import @@ -884,7 +866,7 @@ ): return (before or 1), 0 -@@ -691,8 +727,9 @@ class EmptyLineTracker: +@@ -699,8 +735,9 @@ class EmptyLineTracker: return 0, 1 return 0, 0 @@ -896,7 +878,7 @@ ): if self.mode.is_pyi: return 0, 0 -@@ -701,7 +738,7 @@ class EmptyLineTracker: +@@ -709,7 +746,7 @@ class EmptyLineTracker: comment_to_add_newlines: Optional[LinesBlock] = None if ( self.previous_line.is_comment @@ -905,7 +887,7 @@ and before == 0 ): slc = self.semantic_leading_comment -@@ -718,9 +755,9 @@ class EmptyLineTracker: +@@ -726,9 +763,9 @@ class EmptyLineTracker: if self.mode.is_pyi: if current_line.is_class or self.previous_line.is_class: @@ -917,7 +899,7 @@ newlines = 1 elif current_line.is_stub_class and self.previous_line.is_stub_class: # No blank line between classes with an empty body -@@ -749,7 +786,11 @@ class EmptyLineTracker: +@@ -757,7 +794,11 @@ class EmptyLineTracker: newlines = 1 if current_line.depth else 2 # If a user has left no space after a dummy implementation, don't insert # new lines. This is useful for instance for @overload or Protocols. @@ -930,7 +912,7 @@ newlines = 0 if comment_to_add_newlines is not None: previous_block = comment_to_add_newlines.previous_block -@@ -1020,7 +1061,7 @@ def can_omit_invisible_parens( +@@ -1028,7 +1069,7 @@ def can_omit_invisible_parens( def _can_omit_opening_paren(line: Line, *, first: Leaf, line_length: int) -> bool: """See `can_omit_invisible_parens`.""" remainder = False @@ -939,7 +921,7 @@ _index = -1 for _index, leaf, leaf_length in line.enumerate_with_length(): if leaf.type in CLOSING_BRACKETS and leaf.opening_bracket is first: -@@ -1044,7 +1085,7 @@ def _can_omit_opening_paren(line: Line, +@@ -1052,7 +1093,7 @@ def _can_omit_opening_paren(line: Line, def _can_omit_closing_paren(line: Line, *, last: Leaf, line_length: int) -> bool: """See `can_omit_invisible_parens`.""" @@ -959,7 +941,7 @@ from pyink.const import DEFAULT_LINE_LENGTH -@@ -229,7 +229,31 @@ class Deprecated(UserWarning): +@@ -219,7 +219,31 @@ class Deprecated(UserWarning): """Visible deprecation warning.""" @@ -992,7 +974,7 @@ @dataclass -@@ -237,12 +261,21 @@ class Mode: +@@ -227,12 +251,21 @@ class Mode: target_versions: set[TargetVersion] = field(default_factory=set) line_length: int = DEFAULT_LINE_LENGTH string_normalization: bool = True @@ -1014,17 +996,19 @@ unstable: bool = False enabled_features: set[Preview] = field(default_factory=set) -@@ -254,6 +287,9 @@ class Mode: +@@ -244,6 +277,11 @@ class Mode: except those in UNSTABLE_FEATURES are enabled. Any features in `self.enabled_features` are also enabled. """ -+ # no_normalize_fmt_skip_whitespace is temporarily disabled in Pyink. -+ if feature is Preview.no_normalize_fmt_skip_whitespace and self.is_pyink: ++ # The following feature is temporarily disabled in Pyink because it is ++ # not compatible with range formatting. It's because format skipping in ++ # Black is broken. ++ if feature is Preview.always_one_newline_after_import and self.is_pyink: + return False if self.unstable: return True if feature in self.enabled_features: -@@ -285,12 +321,28 @@ class Mode: +@@ -275,12 +313,28 @@ class Mode: version_str, str(self.line_length), str(int(self.string_normalization)), @@ -1055,15 +1039,15 @@ + return Quote.DOUBLE --- a/nodes.py +++ b/nodes.py -@@ -12,6 +12,7 @@ else: +@@ -13,6 +13,7 @@ else: from mypy_extensions import mypyc_attr +from pyink import ink_comments from pyink.cache import CACHE_DIR - from pyink.mode import Mode, Preview + from pyink.mode import Mode from pyink.strings import get_string_prefix, has_triple_quotes -@@ -793,9 +794,13 @@ def is_function_or_class(node: Node) -> +@@ -809,9 +810,13 @@ def is_function_or_class(node: Node) -> return node.type in {syms.funcdef, syms.classdef, syms.async_funcdef} @@ -1079,7 +1063,7 @@ return False # If there is a comment, we want to keep it. -@@ -914,11 +919,13 @@ def is_type_comment(leaf: Leaf) -> bool: +@@ -930,11 +935,13 @@ def is_type_comment(leaf: Leaf) -> bool: return t in {token.COMMENT, STANDALONE_COMMENT} and v.startswith("# type:") @@ -1140,7 +1124,7 @@ -description = "The uncompromising code formatter." +name = "pyink" +description = "Pyink is a python formatter, forked from Black with slightly different behavior." - license = { text = "MIT" } + license = "MIT" requires-python = ">=3.9" -authors = [ - { name = "Łukasz Langa", email = "lukasz@langa.pl" }, @@ -1163,7 +1147,7 @@ "platformdirs>=2", "tomli>=1.1.0; python_version < '3.11'", "typing_extensions>=4.0.1; python_version < '3.11'", -+ "black==24.10.0", ++ "black==25.1.0", ] -dynamic = ["readme", "version"] +dynamic = ["version"] @@ -1286,8 +1270,8 @@ "type": "array", --- a/strings.py +++ b/strings.py -@@ -8,6 +8,7 @@ from functools import lru_cache - from typing import Final, Match, Pattern +@@ -9,6 +9,7 @@ from re import Match, Pattern + from typing import Final from pyink._width_table import WIDTH_TABLE +from pyink.mode import Quote @@ -1307,7 +1291,7 @@ Adds or removes backslashes as appropriate. """ -@@ -234,8 +237,8 @@ def normalize_string_quotes(s: str) -> s +@@ -233,8 +236,8 @@ def normalize_string_quotes(s: str) -> s if new_escape_count > orig_escape_count: return s # Do not introduce more escaping @@ -1338,7 +1322,7 @@ +pyink = false --- a/tests/test_black.py +++ b/tests/test_black.py -@@ -44,7 +44,7 @@ from pyink import Feature, TargetVersion +@@ -33,7 +33,7 @@ from pyink import Feature, TargetVersion from pyink import re_compile_maybe_verbose as compile_pattern from pyink.cache import FileData, get_cache_dir, get_cache_file from pyink.debug import DebugVisitor @@ -1347,9 +1331,9 @@ from pyink.output import color_diff, diff from pyink.parsing import ASTSafetyError from pyink.report import Report -@@ -2365,6 +2365,19 @@ class TestCaching: - {Preview.docstring_check_for_newline}, - {Preview.hex_codes_in_unicode_sequences}, +@@ -2356,6 +2356,19 @@ class TestCaching: + {Preview.multiline_string_handling}, + {Preview.string_processing}, ] + elif field.type is Quote: + values = list(Quote) @@ -1367,7 +1351,7 @@ elif field.type is bool: values = [True, False] elif field.type is int: -@@ -2845,6 +2858,82 @@ class TestFileCollection: +@@ -2836,6 +2849,82 @@ class TestFileCollection: stdin_filename=stdin_filename, ) @@ -1476,30 +1460,7 @@ runner = CliRunner() -@@ -174,6 +182,22 @@ def test_cell_magic_with_magic() -> None - - - @pytest.mark.parametrize( -+ "src, expected", -+ ( -+ ("\n\n\n%time \n\n", "%time"), -+ (" \n\t\n%%timeit -n4 \t \nx=2 \n\r\n", "%%timeit -n4\nx = 2"), -+ ( -+ " \t\n\n%%capture \nx=2 \n%config \n\n%env\n\t \n \n\n", -+ "%%capture\nx = 2\n%config\n\n%env", -+ ), -+ ), -+) -+def test_cell_magic_with_empty_lines(src: str, expected: str) -> None: -+ result = format_cell(src, fast=True, mode=JUPYTER_MODE) -+ assert result == expected -+ -+ -+@pytest.mark.parametrize( - "mode, expected_output, expectation", - [ - pytest.param( -@@ -224,6 +248,13 @@ def test_cell_magic_with_custom_python_m +@@ -240,6 +248,13 @@ def test_cell_magic_with_custom_python_m format_cell(src, fast=True, mode=JUPYTER_MODE) @@ -1513,7 +1474,7 @@ def test_cell_magic_nested() -> None: src = "%%time\n%%time\n2+2" result = format_cell(src, fast=True, mode=JUPYTER_MODE) -@@ -397,6 +428,45 @@ def test_entire_notebook_no_trailing_new +@@ -413,6 +428,45 @@ def test_entire_notebook_no_trailing_new assert result == expected @@ -1559,7 +1520,7 @@ def test_entire_notebook_without_changes() -> None: content = read_jupyter_notebook("jupyter", "notebook_without_changes") with pytest.raises(NothingChanged): -@@ -448,6 +518,30 @@ def test_ipynb_diff_with_no_change() -> +@@ -464,6 +518,30 @@ def test_ipynb_diff_with_no_change() -> assert expected in result.output @@ -1592,7 +1553,7 @@ ) -> None: --- a/tests/util.py +++ b/tests/util.py -@@ -264,6 +264,11 @@ def get_flags_parser() -> argparse.Argum +@@ -265,6 +265,11 @@ def get_flags_parser() -> argparse.Argum ), ) parser.add_argument("--line-ranges", action="append") @@ -1604,7 +1565,7 @@ parser.add_argument( "--no-preview-line-length-1", default=False, -@@ -287,6 +292,9 @@ def parse_mode(flags_line: str) -> TestC +@@ -288,6 +293,9 @@ def parse_mode(flags_line: str) -> TestC is_ipynb=args.ipynb, magic_trailing_comma=not args.skip_magic_trailing_comma, preview=args.preview, @@ -1614,7 +1575,7 @@ unstable=args.unstable, ) if args.line_ranges: -@@ -340,7 +348,8 @@ def read_jupyter_notebook(subdir_name: s +@@ -341,7 +349,8 @@ def read_jupyter_notebook(subdir_name: s def read_jupyter_notebook_from_file(file_name: Path) -> str: with open(file_name, mode="rb") as fd: content_bytes = fd.read() @@ -1642,18 +1603,18 @@ + pyink --check {toxinidir}/src {toxinidir}/tests {toxinidir}/docs {toxinidir}/scripts --- a/trans.py +++ b/trans.py -@@ -24,8 +24,8 @@ from typing import ( +@@ -12,8 +12,8 @@ from typing import Any, ClassVar, Final, from mypy_extensions import trait from pyink.comments import contains_pragma_comment -from pyink.lines import Line, append_leaves --from pyink.mode import Feature, Mode, Preview +-from pyink.mode import Feature, Mode +from pyink.lines import Indentation, Line, append_leaves -+from pyink.mode import Feature, Mode, Preview, Quote ++from pyink.mode import Feature, Mode, Quote from pyink.nodes import ( CLOSING_BRACKETS, OPENING_BRACKETS, -@@ -275,9 +275,18 @@ class StringTransformer(ABC): +@@ -233,9 +233,18 @@ class StringTransformer(ABC): # Ideally this would be a dataclass, but unfortunately mypyc breaks when used with # `abc.ABC`. @@ -1673,7 +1634,7 @@ @abstractmethod def do_match(self, line: Line) -> TMatchResult: -@@ -755,7 +764,9 @@ class StringMerger(StringTransformer, Cu +@@ -713,7 +722,9 @@ class StringMerger(StringTransformer, Cu S_leaf = Leaf(token.STRING, S) if self.normalize_strings: @@ -1684,7 +1645,7 @@ # Fill the 'custom_splits' list with the appropriate CustomSplit objects. temp_string = S_leaf.value[len(prefix) + 1 : -1] -@@ -856,7 +867,7 @@ class StringMerger(StringTransformer, Cu +@@ -831,7 +842,7 @@ class StringMerger(StringTransformer, Cu if id(leaf) in line.comments: num_of_inline_string_comments += 1 @@ -1693,7 +1654,7 @@ return TErr("Cannot merge strings which have pragma comments.") if num_of_strings < 2: -@@ -996,7 +1007,13 @@ class StringParenStripper(StringTransfor +@@ -975,7 +986,13 @@ class StringParenStripper(StringTransfor idx += 1 if string_indices: @@ -1708,7 +1669,7 @@ return TErr("This line has no strings wrapped in parens.") def do_transform( -@@ -1158,7 +1175,7 @@ class BaseStringSplitter(StringTransform +@@ -1137,7 +1154,7 @@ class BaseStringSplitter(StringTransform ) if id(line.leaves[string_idx]) in line.comments and contains_pragma_comment( @@ -1717,7 +1678,7 @@ ): return TErr( "Line appears to end with an inline pragma comment. Splitting the line" -@@ -1200,7 +1217,7 @@ class BaseStringSplitter(StringTransform +@@ -1179,7 +1196,7 @@ class BaseStringSplitter(StringTransform # NN: The leaf that is after N. # WMA4 the whitespace at the beginning of the line. @@ -1726,7 +1687,7 @@ if is_valid_index(string_idx - 1): p_idx = string_idx - 1 -@@ -1554,7 +1571,7 @@ class StringSplitter(BaseStringSplitter, +@@ -1533,7 +1550,7 @@ class StringSplitter(BaseStringSplitter, characters expand to two columns). """ result = self.line_length @@ -1735,7 +1696,7 @@ result -= 1 if ends_with_comma else 0 result -= string_op_leaves_length return result -@@ -1565,11 +1582,11 @@ class StringSplitter(BaseStringSplitter, +@@ -1544,11 +1561,11 @@ class StringSplitter(BaseStringSplitter, # The last index of a string of length N is N-1. max_break_width -= 1 # Leading whitespace is not present in the string value (e.g. Leaf.value). @@ -1749,7 +1710,7 @@ ) return -@@ -1866,7 +1883,9 @@ class StringSplitter(BaseStringSplitter, +@@ -1845,7 +1862,9 @@ class StringSplitter(BaseStringSplitter, def _maybe_normalize_string_quotes(self, leaf: Leaf) -> None: if self.normalize_strings: @@ -1760,7 +1721,7 @@ def _normalize_f_string(self, string: str, prefix: str) -> str: """ -@@ -1989,7 +2008,8 @@ class StringParenWrapper(BaseStringSplit +@@ -1968,7 +1987,8 @@ class StringParenWrapper(BaseStringSplit char == " " or char in SPLIT_SAFE_CHARS for char in string_value ): # And will still violate the line length limit when split... @@ -1770,7 +1731,7 @@ if str_width(string_value) > max_string_width: # And has no associated custom splits... if not self.has_custom_splits(string_value): -@@ -2235,7 +2255,7 @@ class StringParenWrapper(BaseStringSplit +@@ -2214,7 +2234,7 @@ class StringParenWrapper(BaseStringSplit string_value = LL[string_idx].value string_line = Line( mode=line.mode, diff --git a/pyproject.toml b/pyproject.toml index 52f09022232..df74bd19bbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ build-backend = "hatchling.build" [project] name = "pyink" description = "Pyink is a python formatter, forked from Black with slightly different behavior." -license = { text = "MIT" } +license = "MIT" requires-python = ">=3.9" readme = "README.md" authors = [{name = "The Pyink Maintainers", email = "pyink-maintainers@google.com"}] @@ -42,7 +42,7 @@ dependencies = [ "platformdirs>=2", "tomli>=1.1.0; python_version < '3.11'", "typing_extensions>=4.0.1; python_version < '3.11'", - "black==24.10.0", + "black==25.1.0", ] dynamic = ["version"] diff --git a/src/pyink/__init__.py b/src/pyink/__init__.py index 99647535b56..b475cb62f11 100644 --- a/src/pyink/__init__.py +++ b/src/pyink/__init__.py @@ -5,24 +5,22 @@ import sys import tokenize import traceback -from contextlib import contextmanager -from dataclasses import replace -from datetime import datetime, timezone -from enum import Enum -from json.decoder import JSONDecodeError -from pathlib import Path -from typing import ( - Any, +from collections.abc import ( Collection, Generator, Iterator, MutableMapping, - Optional, - Pattern, Sequence, Sized, - Union, ) +from contextlib import contextmanager +from dataclasses import replace +from datetime import datetime, timezone +from enum import Enum +from json.decoder import JSONDecodeError +from pathlib import Path +from re import Pattern +from typing import Any, Optional, Union from blib2to3.pgen2 import token from blib2to3.pytree import Leaf, Node @@ -828,6 +826,12 @@ def get_sources( for s in src: if s == "-" and stdin_filename: path = Path(stdin_filename) + if path_is_excluded(stdin_filename, force_exclude): + report.path_ignored( + path, + "--stdin-filename matches the --force-exclude regular expression", + ) + continue is_stdin = True else: path = Path(s) diff --git a/src/pyink/brackets.py b/src/pyink/brackets.py index 188a318feda..b93add260b9 100644 --- a/src/pyink/brackets.py +++ b/src/pyink/brackets.py @@ -1,7 +1,8 @@ """Builds on top of nodes.py to track brackets.""" +from collections.abc import Iterable, Sequence from dataclasses import dataclass, field -from typing import Final, Iterable, Optional, Sequence, Union +from typing import Final, Optional, Union from pyink.nodes import ( BRACKET, diff --git a/src/pyink/cache.py b/src/pyink/cache.py index f834dacda71..272d5f34618 100644 --- a/src/pyink/cache.py +++ b/src/pyink/cache.py @@ -5,9 +5,10 @@ import pickle import sys import tempfile +from collections.abc import Iterable from dataclasses import dataclass, field from pathlib import Path -from typing import Iterable, NamedTuple +from typing import NamedTuple from platformdirs import user_cache_dir diff --git a/src/pyink/comments.py b/src/pyink/comments.py index 2bbed0dec3d..6613ac2e479 100644 --- a/src/pyink/comments.py +++ b/src/pyink/comments.py @@ -1,10 +1,11 @@ import re +from collections.abc import Collection, Iterator from dataclasses import dataclass from functools import lru_cache -from typing import Collection, Final, Iterator, Optional, Union +from typing import Final, Optional, Union from pyink import ink_comments -from pyink.mode import Mode, Preview +from pyink.mode import Mode from pyink.nodes import ( CLOSING_BRACKETS, STANDALONE_COMMENT, @@ -235,11 +236,7 @@ def convert_one_fmt_off_pair( standalone_comment_prefix += fmt_off_prefix hidden_value = comment.value + "\n" + hidden_value if is_fmt_skip: - hidden_value += ( - comment.leading_whitespace - if Preview.no_normalize_fmt_skip_whitespace in mode - else " " - ) + comment.value + hidden_value += comment.leading_whitespace + comment.value if hidden_value.endswith("\n"): # That happens when one of the `ignored_nodes` ended with a NEWLINE # leaf (possibly followed by a DEDENT). diff --git a/src/pyink/concurrency.py b/src/pyink/concurrency.py index 37fd4e54d9d..1b40747e51f 100644 --- a/src/pyink/concurrency.py +++ b/src/pyink/concurrency.py @@ -10,10 +10,11 @@ import signal import sys import traceback +from collections.abc import Iterable from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor from multiprocessing import Manager from pathlib import Path -from typing import Any, Iterable, Optional +from typing import Any, Optional from mypy_extensions import mypyc_attr diff --git a/src/pyink/debug.py b/src/pyink/debug.py index 0a757cbbdc8..165819fc088 100644 --- a/src/pyink/debug.py +++ b/src/pyink/debug.py @@ -1,5 +1,6 @@ +from collections.abc import Iterator from dataclasses import dataclass, field -from typing import Any, Iterator, TypeVar, Union +from typing import Any, TypeVar, Union from pyink.nodes import Visitor from pyink.output import out diff --git a/src/pyink/files.py b/src/pyink/files.py index 00bf06850c2..8fe1e6009db 100644 --- a/src/pyink/files.py +++ b/src/pyink/files.py @@ -1,18 +1,11 @@ import io import os import sys +from collections.abc import Iterable, Iterator, Sequence from functools import lru_cache from pathlib import Path -from typing import ( - TYPE_CHECKING, - Any, - Iterable, - Iterator, - Optional, - Pattern, - Sequence, - Union, -) +from re import Pattern +from typing import TYPE_CHECKING, Any, Optional, Union from mypy_extensions import mypyc_attr from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet diff --git a/src/pyink/handle_ipynb_magics.py b/src/pyink/handle_ipynb_magics.py index e7cc7e56557..db3347dba6c 100644 --- a/src/pyink/handle_ipynb_magics.py +++ b/src/pyink/handle_ipynb_magics.py @@ -43,7 +43,6 @@ "time", "timeit", )) -TOKEN_HEX = secrets.token_hex @dataclasses.dataclass(frozen=True) @@ -160,7 +159,7 @@ def mask_cell(src: str) -> tuple[str, list[Replacement]]: becomes - "25716f358c32750e" + b"25716f358c32750" 'foo' The replacements are returned, along with the transformed code. @@ -192,6 +191,18 @@ def mask_cell(src: str) -> tuple[str, list[Replacement]]: return transformed, replacements +def create_token(n_chars: int) -> str: + """Create a randomly generated token that is n_chars characters long.""" + assert n_chars > 0 + n_bytes = max(n_chars // 2 - 1, 1) + token = secrets.token_hex(n_bytes) + if len(token) + 3 > n_chars: + token = token[:-1] + # We use a bytestring so that the string does not get interpreted + # as a docstring. + return f'b"{token}"' + + def get_token(src: str, magic: str) -> str: """Return randomly generated token to mask IPython magic with. @@ -201,11 +212,11 @@ def get_token(src: str, magic: str) -> str: not already present anywhere else in the cell. """ assert magic - nbytes = max(len(magic) // 2 - 1, 1) - token = TOKEN_HEX(nbytes) + n_chars = len(magic) + token = create_token(n_chars) counter = 0 while token in src: - token = TOKEN_HEX(nbytes) + token = create_token(n_chars) counter += 1 if counter > 100: raise AssertionError( @@ -213,9 +224,7 @@ def get_token(src: str, magic: str) -> str: "Please report a bug on https://github.com/psf/black/issues. " f"The magic might be helpful: {magic}" ) from None - if len(token) + 2 < len(magic): - token = f"{token}." - return f'"{token}"' + return token def replace_cell_magics(src: str) -> tuple[str, list[Replacement]]: @@ -306,7 +315,7 @@ def unmask_cell(src: str, replacements: list[Replacement]) -> str: for replacement in replacements: src = src.replace(replacement.mask, replacement.src) # Strings in src might have been reformatted with single quotes. - src = src.replace(f"'{replacement.mask[1:-1]}'", replacement.src) + src = src.replace(f"b'{replacement.mask[2:-1]}'", replacement.src) return src diff --git a/src/pyink/ink_comments.py b/src/pyink/ink_comments.py index 0198a24fab7..7088f0d4e72 100644 --- a/src/pyink/ink_comments.py +++ b/src/pyink/ink_comments.py @@ -4,12 +4,13 @@ """ import re +from typing import Optional from pyink.mode import Mode -def comment_contains_pragma(comment: str, mode: Mode) -> bool: - """Check if the given string contains one of the pragma forms. +def comment_contains_pragma(comment: str, mode: Optional[Mode]) -> bool: + """Check if the given string contains one of the pragma forms. A pragma form can appear at the beginning of a comment: # pytype: disable=attribute-error @@ -25,6 +26,8 @@ def comment_contains_pragma(comment: str, mode: Mode) -> bool: Returns: True if the comment contains one of the pragma forms. """ - joined_pragma_expression = "|".join(mode.pyink_annotation_pragmas) - pragma_regex = re.compile(rf"([#|;] ?(?:{joined_pragma_expression}))") - return pragma_regex.search(comment) is not None + if mode is None: + mode = Mode() + joined_pragma_expression = "|".join(mode.pyink_annotation_pragmas) + pragma_regex = re.compile(rf"([#|;] ?(?:{joined_pragma_expression}))") + return pragma_regex.search(comment) is not None diff --git a/src/pyink/linegen.py b/src/pyink/linegen.py index 2ab9587c509..85d824321da 100644 --- a/src/pyink/linegen.py +++ b/src/pyink/linegen.py @@ -4,10 +4,11 @@ import re import sys +from collections.abc import Collection, Iterator from dataclasses import replace from enum import Enum, auto from functools import partial, wraps -from typing import Collection, Iterator, Optional, Union, cast +from typing import Optional, Union, cast if sys.version_info < (3, 8): from typing_extensions import Final, Literal @@ -51,6 +52,7 @@ is_atom_with_invisible_parens, is_docstring, is_empty_tuple, + is_generator, is_lpar_token, is_multiline_string, is_name_token, @@ -61,6 +63,7 @@ is_rpar_token, is_stub_body, is_stub_suite, + is_tuple_containing_star, is_tuple_containing_walrus, is_vararg, is_walrus_assignment, @@ -70,7 +73,7 @@ ) from pyink.numerics import normalize_numeric_literal from pyink.strings import ( - fix_docstring, + fix_multiline_docstring, get_string_prefix, normalize_string_prefix, normalize_string_quotes, @@ -452,13 +455,10 @@ def foo(a: (int), b: (float) = 7): ... yield from self.visit_default(node) def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: - if ( - Preview.hex_codes_in_unicode_sequences in self.mode - and not self.mode.is_pyink - ): + if not self.mode.is_pyink: normalize_unicode_escape_sequences(leaf) - if is_docstring(leaf, self.mode) and not re.search(r"\\\s*\n", leaf.value): + if is_docstring(leaf) and not re.search(r"\\\s*\n", leaf.value): # We're ignoring docstrings with backslash newline escapes because changing # indentation of those changes the AST representation of the code. if self.mode.string_normalization: @@ -487,7 +487,7 @@ def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: indent = " " * self.current_line.indentation_spaces() if is_multiline_string(leaf): - docstring = fix_docstring(docstring, indent) + docstring = fix_multiline_docstring(docstring, indent) else: docstring = docstring.strip() @@ -537,10 +537,7 @@ def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: and len(indent) + quote_len <= self.mode.line_length and not has_trailing_backslash ): - if ( - Preview.docstring_check_for_newline in self.mode - and leaf.value[-1 - quote_len] == "\n" - ): + if leaf.value[-1 - quote_len] == "\n": leaf.value = prefix + quote + docstring + quote else: leaf.value = prefix + quote + docstring + "\n" + indent + quote @@ -560,6 +557,19 @@ def visit_NUMBER(self, leaf: Leaf) -> Iterator[Line]: normalize_numeric_literal(leaf) yield from self.visit_default(leaf) + def visit_atom(self, node: Node) -> Iterator[Line]: + """Visit any atom""" + if len(node.children) == 3: + first = node.children[0] + last = node.children[-1] + if (first.type == token.LSQB and last.type == token.RSQB) or ( + first.type == token.LBRACE and last.type == token.RBRACE + ): + # Lists or sets of one item + maybe_make_parens_invisible_in_atom(node.children[1], parent=node) + + yield from self.visit_default(node) + def visit_fstring(self, node: Node) -> Iterator[Line]: # currently we don't want to format and split f-strings at all. string_leaf = fstring_to_string(node) @@ -638,8 +648,7 @@ def __post_init__(self) -> None: # PEP 634 self.visit_match_stmt = self.visit_match_case self.visit_case_block = self.visit_match_case - if Preview.remove_redundant_guard_parens in self.mode: - self.visit_guard = partial(v, keywords=Ø, parens={"if"}) + self.visit_guard = partial(v, keywords=Ø, parens={"if"}) def _hugging_power_ops_line_to_string( @@ -832,26 +841,29 @@ def left_hand_split( Prefer RHS otherwise. This is why this function is not symmetrical with :func:`right_hand_split` which also handles optional parentheses. """ - tail_leaves: list[Leaf] = [] - body_leaves: list[Leaf] = [] - head_leaves: list[Leaf] = [] - current_leaves = head_leaves - matching_bracket: Optional[Leaf] = None - for leaf in line.leaves: - if ( - current_leaves is body_leaves - and leaf.type in CLOSING_BRACKETS - and leaf.opening_bracket is matching_bracket - and isinstance(matching_bracket, Leaf) - ): - ensure_visible(leaf) - ensure_visible(matching_bracket) - current_leaves = tail_leaves if body_leaves else head_leaves - current_leaves.append(leaf) - if current_leaves is head_leaves: - if leaf.type in OPENING_BRACKETS: - matching_bracket = leaf - current_leaves = body_leaves + for leaf_type in [token.LPAR, token.LSQB]: + tail_leaves: list[Leaf] = [] + body_leaves: list[Leaf] = [] + head_leaves: list[Leaf] = [] + current_leaves = head_leaves + matching_bracket: Optional[Leaf] = None + for leaf in line.leaves: + if ( + current_leaves is body_leaves + and leaf.type in CLOSING_BRACKETS + and leaf.opening_bracket is matching_bracket + and isinstance(matching_bracket, Leaf) + ): + ensure_visible(leaf) + ensure_visible(matching_bracket) + current_leaves = tail_leaves if body_leaves else head_leaves + current_leaves.append(leaf) + if current_leaves is head_leaves: + if leaf.type == leaf_type: + matching_bracket = leaf + current_leaves = body_leaves + if matching_bracket and tail_leaves: + break if not matching_bracket or not tail_leaves: raise CannotSplit("No brackets found") @@ -1017,29 +1029,7 @@ def _maybe_split_omitting_optional_parens( try: # The RHSResult Omitting Optional Parens. rhs_oop = _first_right_hand_split(line, omit=omit) - is_split_right_after_equal = ( - len(rhs.head.leaves) >= 2 and rhs.head.leaves[-2].type == token.EQUAL - ) - rhs_head_contains_brackets = any( - leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1] - ) - # the -1 is for the ending optional paren - rhs_head_short_enough = is_line_short_enough( - rhs.head, mode=replace(mode, line_length=mode.line_length - 1) - ) - rhs_head_explode_blocked_by_magic_trailing_comma = ( - rhs.head.magic_trailing_comma is None - ) - if ( - not ( - is_split_right_after_equal - and rhs_head_contains_brackets - and rhs_head_short_enough - and rhs_head_explode_blocked_by_magic_trailing_comma - ) - # the omit optional parens split is preferred by some other reason - or _prefer_split_rhs_oop_over_rhs(rhs_oop, rhs, mode) - ): + if _prefer_split_rhs_oop_over_rhs(rhs_oop, rhs, mode): yield from _maybe_split_omitting_optional_parens( rhs_oop, line, mode, features=features, omit=omit ) @@ -1050,8 +1040,15 @@ def _maybe_split_omitting_optional_parens( if line.is_chained_assignment: pass - elif not can_be_split(rhs.body) and not is_line_short_enough( - rhs.body, mode=mode + elif ( + not can_be_split(rhs.body) + and not is_line_short_enough(rhs.body, mode=mode) + and not ( + Preview.wrap_long_dict_values_in_parens + and rhs.opening_bracket.parent + and rhs.opening_bracket.parent.parent + and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker + ) ): raise CannotSplit( "Splitting failed, body is still too long and can't be split." @@ -1082,6 +1079,44 @@ def _prefer_split_rhs_oop_over_rhs( Returns whether we should prefer the result from a split omitting optional parens (rhs_oop) over the original (rhs). """ + # contains unsplittable type ignore + if ( + rhs_oop.head.contains_unsplittable_type_ignore() + or rhs_oop.body.contains_unsplittable_type_ignore() + or rhs_oop.tail.contains_unsplittable_type_ignore() + ): + return True + + # Retain optional parens around dictionary values + if ( + Preview.wrap_long_dict_values_in_parens + and rhs.opening_bracket.parent + and rhs.opening_bracket.parent.parent + and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker + and rhs.body.bracket_tracker.delimiters + ): + # Unless the split is inside the key + return any(leaf.type == token.COLON for leaf in rhs_oop.tail.leaves) + + # the split is right after `=` + if not (len(rhs.head.leaves) >= 2 and rhs.head.leaves[-2].type == token.EQUAL): + return True + + # the left side of assignment contains brackets + if not any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1]): + return True + + # the left side of assignment is short enough (the -1 is for the ending optional + # paren) + if not is_line_short_enough( + rhs.head, mode=replace(mode, line_length=mode.line_length - 1) + ): + return True + + # the left side of assignment won't explode further because of magic trailing comma + if rhs.head.magic_trailing_comma is not None: + return True + # If we have multiple targets, we prefer more `=`s on the head vs pushing them to # the body rhs_head_equal_count = [leaf.type for leaf in rhs.head.leaves].count(token.EQUAL) @@ -1109,10 +1144,6 @@ def _prefer_split_rhs_oop_over_rhs( # the first line is short enough and is_line_short_enough(rhs_oop.head, mode=mode) ) - # contains unsplittable type ignore - or rhs_oop.head.contains_unsplittable_type_ignore() - or rhs_oop.body.contains_unsplittable_type_ignore() - or rhs_oop.tail.contains_unsplittable_type_ignore() ) @@ -1157,12 +1188,7 @@ def _ensure_trailing_comma( return False # Don't add commas if we already have any commas if any( - leaf.type == token.COMMA - and ( - Preview.typed_params_trailing_comma not in original.mode - or not is_part_of_annotation(leaf) - ) - for leaf in leaves + leaf.type == token.COMMA and not is_part_of_annotation(leaf) for leaf in leaves ): return False @@ -1443,11 +1469,7 @@ def normalize_invisible_parens( # noqa: C901 ) # Add parentheses around if guards in case blocks - if ( - isinstance(child, Node) - and child.type == syms.guard - and Preview.parens_for_long_if_clauses_in_case_block in mode - ): + if isinstance(child, Node) and child.type == syms.guard: normalize_invisible_parens( child, parens_after={"if"}, mode=mode, features=features ) @@ -1658,7 +1680,7 @@ def remove_with_parens(node: Node, parent: Node, mode: Mode) -> None: def maybe_make_parens_invisible_in_atom( node: LN, parent: LN, - mode: Mode, + mode: Optional[Mode] = None, remove_brackets_around_comma: bool = False, ) -> bool: """If it's safe, make the parens in the atom `node` invisible, recursively. @@ -1680,6 +1702,8 @@ def maybe_make_parens_invisible_in_atom( and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY ) or is_tuple_containing_walrus(node) + or is_tuple_containing_star(node) + or is_generator(node) ): return False @@ -1711,9 +1735,6 @@ def maybe_make_parens_invisible_in_atom( not ink_comments.comment_contains_pragma(middle.prefix.strip(), mode) ): first.value = "" - if first.prefix.strip(): - # Preserve comments before first paren - middle.prefix = first.prefix + middle.prefix last.value = "" maybe_make_parens_invisible_in_atom( middle, @@ -1726,6 +1747,13 @@ def maybe_make_parens_invisible_in_atom( # Strip the invisible parens from `middle` by replacing # it with the child in-between the invisible parens middle.replace(middle.children[1]) + + if middle.children[0].prefix.strip(): + # Preserve comments before first paren + middle.children[1].prefix = ( + middle.children[0].prefix + middle.children[1].prefix + ) + if middle.children[-1].prefix.strip(): # Preserve comments before last paren last.prefix = middle.children[-1].prefix + last.prefix diff --git a/src/pyink/lines.py b/src/pyink/lines.py index f3c1a88af4f..7f241d5a0b0 100644 --- a/src/pyink/lines.py +++ b/src/pyink/lines.py @@ -1,8 +1,9 @@ from enum import Enum, auto import itertools import math +from collections.abc import Callable, Iterator, Sequence from dataclasses import dataclass, field -from typing import Callable, Iterator, Optional, Sequence, TypeVar, Union, cast +from typing import Optional, TypeVar, Union, cast from pyink.brackets import COMMA_PRIORITY, DOT_PRIORITY, BracketTracker from pyink.mode import Mode, Preview @@ -219,9 +220,7 @@ def _is_triple_quoted_string(self) -> bool: @property def is_docstring(self) -> bool: """Is the line a docstring?""" - if Preview.unify_docstring_detection not in self.mode: - return self._is_triple_quoted_string - return bool(self) and is_docstring(self.leaves[0], self.mode) + return bool(self) and is_docstring(self.leaves[0]) @property def is_chained_assignment(self) -> bool: @@ -691,6 +690,15 @@ def _maybe_empty_lines(self, current_line: Line) -> tuple[int, int]: # noqa: C9 current_line, before, user_had_newline ) + if ( + self.previous_line.is_import + and not self.previous_line.depth + and not current_line.depth + and not current_line.is_import + and Preview.always_one_newline_after_import in self.mode + ): + return 1, 0 + if ( ( self.previous_line.is_import diff --git a/src/pyink/mode.py b/src/pyink/mode.py index af74e5a7638..987e9a99640 100644 --- a/src/pyink/mode.py +++ b/src/pyink/mode.py @@ -196,28 +196,18 @@ def supports_feature(target_versions: set[TargetVersion], feature: Feature) -> b class Preview(Enum): """Individual preview style features.""" - hex_codes_in_unicode_sequences = auto() # NOTE: string_processing requires wrap_long_dict_values_in_parens # for https://github.com/psf/black/issues/3117 to be fixed. string_processing = auto() hug_parens_with_braces_and_square_brackets = auto() - unify_docstring_detection = auto() - no_normalize_fmt_skip_whitespace = auto() wrap_long_dict_values_in_parens = auto() multiline_string_handling = auto() - typed_params_trailing_comma = auto() - is_simple_lookup_for_doublestar_expression = auto() - docstring_check_for_newline = auto() - remove_redundant_guard_parens = auto() - parens_for_long_if_clauses_in_case_block = auto() - pep646_typed_star_arg_type_var_tuple = auto() + always_one_newline_after_import = auto() UNSTABLE_FEATURES: set[Preview] = { # Many issues, see summary in https://github.com/psf/black/issues/4042 Preview.string_processing, - # See issues #3452 and #4158 - Preview.wrap_long_dict_values_in_parens, # See issue #4159 Preview.multiline_string_handling, # See issue #4036 (crash), #4098, #4099 (proposed tweaks) @@ -287,8 +277,10 @@ def __contains__(self, feature: Preview) -> bool: except those in UNSTABLE_FEATURES are enabled. Any features in `self.enabled_features` are also enabled. """ - # no_normalize_fmt_skip_whitespace is temporarily disabled in Pyink. - if feature is Preview.no_normalize_fmt_skip_whitespace and self.is_pyink: + # The following feature is temporarily disabled in Pyink because it is + # not compatible with range formatting. It's because format skipping in + # Black is broken. + if feature is Preview.always_one_newline_after_import and self.is_pyink: return False if self.unstable: return True diff --git a/src/pyink/nodes.py b/src/pyink/nodes.py index 1703f402954..49d07e4b6e0 100644 --- a/src/pyink/nodes.py +++ b/src/pyink/nodes.py @@ -3,7 +3,8 @@ """ import sys -from typing import Final, Generic, Iterator, Literal, Optional, TypeVar, Union +from collections.abc import Iterator +from typing import Final, Generic, Literal, Optional, TypeVar, Union if sys.version_info >= (3, 10): from typing import TypeGuard @@ -14,7 +15,7 @@ from pyink import ink_comments from pyink.cache import CACHE_DIR -from pyink.mode import Mode, Preview +from pyink.mode import Mode from pyink.strings import get_string_prefix, has_triple_quotes from blib2to3 import pygram from blib2to3.pgen2 import token @@ -244,13 +245,7 @@ def whitespace(leaf: Leaf, *, complex_subscript: bool, mode: Mode) -> str: # no elif ( prevp.type == token.STAR and parent_type(prevp) == syms.star_expr - and ( - parent_type(prevp.parent) == syms.subscriptlist - or ( - Preview.pep646_typed_star_arg_type_var_tuple in mode - and parent_type(prevp.parent) == syms.tname_star - ) - ) + and parent_type(prevp.parent) in (syms.subscriptlist, syms.tname_star) ): # No space between typevar tuples or unpacking them. return NO @@ -551,7 +546,7 @@ def is_arith_like(node: LN) -> bool: } -def is_docstring(node: NL, mode: Mode) -> bool: +def is_docstring(node: NL) -> bool: if isinstance(node, Leaf): if node.type != token.STRING: return False @@ -561,8 +556,7 @@ def is_docstring(node: NL, mode: Mode) -> bool: return False if ( - Preview.unify_docstring_detection in mode - and node.parent + node.parent and node.parent.type == syms.simple_stmt and not node.parent.prev_sibling and node.parent.parent @@ -621,6 +615,28 @@ def is_tuple_containing_walrus(node: LN) -> bool: return any(child.type == syms.namedexpr_test for child in gexp.children) +def is_tuple_containing_star(node: LN) -> bool: + """Return True if `node` holds a tuple that contains a star operator.""" + if node.type != syms.atom: + return False + gexp = unwrap_singleton_parenthesis(node) + if gexp is None or gexp.type != syms.testlist_gexp: + return False + + return any(child.type == syms.star_expr for child in gexp.children) + + +def is_generator(node: LN) -> bool: + """Return True if `node` holds a generator.""" + if node.type != syms.atom: + return False + gexp = unwrap_singleton_parenthesis(node) + if gexp is None or gexp.type != syms.testlist_gexp: + return False + + return any(child.type == syms.old_comp_for for child in gexp.children) + + def is_one_sequence_between( opening: Leaf, closing: Leaf, diff --git a/src/pyink/parsing.py b/src/pyink/parsing.py index e5f839e6e2b..036130d8eee 100644 --- a/src/pyink/parsing.py +++ b/src/pyink/parsing.py @@ -5,7 +5,7 @@ import ast import sys import warnings -from typing import Collection, Iterator +from collections.abc import Collection, Iterator from pyink.mode import VERSION_TO_FEATURES, Feature, TargetVersion, supports_feature from pyink.nodes import syms diff --git a/src/pyink/ranges.py b/src/pyink/ranges.py index 12968e99f06..2a4c484635b 100644 --- a/src/pyink/ranges.py +++ b/src/pyink/ranges.py @@ -1,8 +1,9 @@ """Functions related to Black's formatting by line ranges feature.""" import difflib +from collections.abc import Collection, Iterator, Sequence from dataclasses import dataclass -from typing import Collection, Iterator, Sequence, Union +from typing import Union from pyink.nodes import ( LN, diff --git a/src/pyink/resources/pyink.schema.json b/src/pyink/resources/pyink.schema.json index 0b604baaa62..a97a8d6254d 100644 --- a/src/pyink/resources/pyink.schema.json +++ b/src/pyink/resources/pyink.schema.json @@ -79,19 +79,11 @@ "type": "array", "items": { "enum": [ - "hex_codes_in_unicode_sequences", "string_processing", "hug_parens_with_braces_and_square_brackets", - "unify_docstring_detection", - "no_normalize_fmt_skip_whitespace", "wrap_long_dict_values_in_parens", "multiline_string_handling", - "typed_params_trailing_comma", - "is_simple_lookup_for_doublestar_expression", - "docstring_check_for_newline", - "remove_redundant_guard_parens", - "parens_for_long_if_clauses_in_case_block", - "pep646_typed_star_arg_type_var_tuple" + "always_one_newline_after_import" ] }, "description": "Enable specific features included in the `--unstable` style. Requires `--preview`. No compatibility guarantees are provided on the behavior or existence of any unstable features." diff --git a/src/pyink/strings.py b/src/pyink/strings.py index 1a2f49d49ea..89166cd2b2a 100644 --- a/src/pyink/strings.py +++ b/src/pyink/strings.py @@ -5,7 +5,8 @@ import re import sys from functools import lru_cache -from typing import Final, Match, Pattern +from re import Match, Pattern +from typing import Final from pyink._width_table import WIDTH_TABLE from pyink.mode import Quote @@ -63,10 +64,9 @@ def lines_with_leading_tabs_expanded(s: str) -> list[str]: return lines -def fix_docstring(docstring: str, prefix: str) -> str: +def fix_multiline_docstring(docstring: str, prefix: str) -> str: # https://www.python.org/dev/peps/pep-0257/#handling-docstring-indentation - if not docstring: - return "" + assert docstring, "INTERNAL ERROR: Multiline docstrings cannot be empty" lines = lines_with_leading_tabs_expanded(docstring) # Determine minimum indentation (first line doesn't count): indent = sys.maxsize @@ -188,8 +188,7 @@ def normalize_string_quotes(s: str, *, preferred_quote: Quote) -> str: orig_quote = "'" new_quote = '"' first_quote_pos = s.find(orig_quote) - if first_quote_pos == -1: - return s # There's an internal error + assert first_quote_pos != -1, f"INTERNAL ERROR: Malformed string {s!r}" prefix = s[:first_quote_pos] unescaped_new_quote = _cached_compile(rf"(([^\\]|^)(\\\\)*){new_quote}") diff --git a/src/pyink/trans.py b/src/pyink/trans.py index e6209fb24ef..94501078d0d 100644 --- a/src/pyink/trans.py +++ b/src/pyink/trans.py @@ -5,27 +5,15 @@ import re from abc import ABC, abstractmethod from collections import defaultdict +from collections.abc import Callable, Collection, Iterable, Iterator, Sequence from dataclasses import dataclass -from typing import ( - Any, - Callable, - ClassVar, - Collection, - Final, - Iterable, - Iterator, - Literal, - Optional, - Sequence, - TypeVar, - Union, -) +from typing import Any, ClassVar, Final, Literal, Optional, TypeVar, Union from mypy_extensions import trait from pyink.comments import contains_pragma_comment from pyink.lines import Indentation, Line, append_leaves -from pyink.mode import Feature, Mode, Preview, Quote +from pyink.mode import Feature, Mode, Quote from pyink.nodes import ( CLOSING_BRACKETS, OPENING_BRACKETS, @@ -94,18 +82,12 @@ def is_simple_lookup(index: int, kind: Literal[1, -1]) -> bool: # Brackets and parentheses indicate calls, subscripts, etc. ... # basically stuff that doesn't count as "simple". Only a NAME lookup # or dotted lookup (eg. NAME.NAME) is OK. - if Preview.is_simple_lookup_for_doublestar_expression not in mode: - return original_is_simple_lookup_func(line, index, kind) - + if kind == -1: + return handle_is_simple_look_up_prev(line, index, {token.RPAR, token.RSQB}) else: - if kind == -1: - return handle_is_simple_look_up_prev( - line, index, {token.RPAR, token.RSQB} - ) - else: - return handle_is_simple_lookup_forward( - line, index, {token.LPAR, token.LSQB} - ) + return handle_is_simple_lookup_forward( + line, index, {token.LPAR, token.LSQB} + ) def is_simple_operand(index: int, kind: Literal[1, -1]) -> bool: # An operand is considered "simple" if's a NAME, a numeric CONSTANT, a simple @@ -151,30 +133,6 @@ def is_simple_operand(index: int, kind: Literal[1, -1]) -> bool: yield new_line -def original_is_simple_lookup_func( - line: Line, index: int, step: Literal[1, -1] -) -> bool: - if step == -1: - disallowed = {token.RPAR, token.RSQB} - else: - disallowed = {token.LPAR, token.LSQB} - - while 0 <= index < len(line.leaves): - current = line.leaves[index] - if current.type in disallowed: - return False - if current.type not in {token.NAME, token.DOT} or current.value == "for": - # If the current token isn't disallowed, we'll assume this is - # simple as only the disallowed tokens are semantically - # attached to this lookup expression we're checking. Also, - # stop early if we hit the 'for' bit of a comprehension. - return True - - index += step - - return True - - def handle_is_simple_look_up_prev(line: Line, index: int, disallowed: set[int]) -> bool: """ Handling the determination of is_simple_lookup for the lines prior to the doublestar @@ -681,10 +639,10 @@ def make_naked(string: str, string_prefix: str) -> str: """ assert_is_leaf_string(string) if "f" in string_prefix: - f_expressions = ( + f_expressions = [ string[span[0] + 1 : span[1] - 1] # +-1 to get rid of curly braces for span in iter_fexpr_spans(string) - ) + ] debug_expressions_contain_visible_quotes = any( re.search(r".*[\'\"].*(? TResult[None]: - The set of all string prefixes in the string group is of length greater than one and is not equal to {"", "f"}. - The string group consists of raw strings. + - The string group would merge f-strings with different quote types + and internal quotes. - The string group is stringified type annotations. We don't want to process stringified type annotations since pyright doesn't support them spanning multiple string values. (NOTE: mypy, pytype, pyre do @@ -843,6 +803,8 @@ def _validate_msg(line: Line, string_idx: int) -> TResult[None]: i += inc + QUOTE = line.leaves[string_idx].value[-1] + num_of_inline_string_comments = 0 set_of_prefixes = set() num_of_strings = 0 @@ -865,6 +827,19 @@ def _validate_msg(line: Line, string_idx: int) -> TResult[None]: set_of_prefixes.add(prefix) + if ( + "f" in prefix + and leaf.value[-1] != QUOTE + and ( + "'" in leaf.value[len(prefix) + 1 : -1] + or '"' in leaf.value[len(prefix) + 1 : -1] + ) + ): + return TErr( + "StringMerger does NOT merge f-strings with different quote types" + " and internal quotes." + ) + if id(leaf) in line.comments: num_of_inline_string_comments += 1 if contains_pragma_comment(line.comments[id(leaf)], line.mode): @@ -893,6 +868,7 @@ class StringParenStripper(StringTransformer): The line contains a string which is surrounded by parentheses and: - The target string is NOT the only argument to a function call. - The target string is NOT a "pointless" string. + - The target string is NOT a dictionary value. - If the target string contains a PERCENT, the brackets are not preceded or followed by an operator with higher precedence than PERCENT. @@ -940,11 +916,14 @@ def do_match(self, line: Line) -> TMatchResult: ): continue - # That LPAR should NOT be preceded by a function name or a closing - # bracket (which could be a function which returns a function or a - # list/dictionary that contains a function)... + # That LPAR should NOT be preceded by a colon (which could be a + # dictionary value), function name, or a closing bracket (which + # could be a function returning a function or a list/dictionary + # containing a function)... if is_valid_index(idx - 2) and ( - LL[idx - 2].type == token.NAME or LL[idx - 2].type in CLOSING_BRACKETS + LL[idx - 2].type == token.COLON + or LL[idx - 2].type == token.NAME + or LL[idx - 2].type in CLOSING_BRACKETS ): continue @@ -2279,12 +2258,12 @@ def do_transform( elif right_leaves and right_leaves[-1].type == token.RPAR: # Special case for lambda expressions as dict's value, e.g.: # my_dict = { - # "key": lambda x: f"formatted: {x}, + # "key": lambda x: f"formatted: {x}", # } # After wrapping the dict's value with parentheses, the string is # followed by a RPAR but its opening bracket is lambda's, not # the string's: - # "key": (lambda x: f"formatted: {x}), + # "key": (lambda x: f"formatted: {x}"), opening_bracket = right_leaves[-1].opening_bracket if opening_bracket is not None and opening_bracket in left_leaves: index = left_leaves.index(opening_bracket) diff --git a/tests/data/cases/annotations.py b/tests/data/cases/annotations.py new file mode 100644 index 00000000000..4d7af3d077f --- /dev/null +++ b/tests/data/cases/annotations.py @@ -0,0 +1,17 @@ +# regression test for #1765 +class Foo: + def foo(self): + if True: + content_ids: Mapping[ + str, Optional[ContentId] + ] = self.publisher_content_store.store_config_contents(files) + +# output + +# regression test for #1765 +class Foo: + def foo(self): + if True: + content_ids: Mapping[str, Optional[ContentId]] = ( + self.publisher_content_store.store_config_contents(files) + ) \ No newline at end of file diff --git a/tests/data/cases/preview_cantfit.py b/tests/data/cases/cantfit.py similarity index 99% rename from tests/data/cases/preview_cantfit.py rename to tests/data/cases/cantfit.py index 29789c7e653..f002326947d 100644 --- a/tests/data/cases/preview_cantfit.py +++ b/tests/data/cases/cantfit.py @@ -1,4 +1,3 @@ -# flags: --preview # long variable name this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it = 0 this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it = 1 # with a comment diff --git a/tests/data/cases/context_managers_38.py b/tests/data/cases/context_managers_38.py index 54fb97c708b..f125cdffb8a 100644 --- a/tests/data/cases/context_managers_38.py +++ b/tests/data/cases/context_managers_38.py @@ -1,4 +1,3 @@ -# flags: --minimum-version=3.8 with \ make_context_manager1() as cm1, \ make_context_manager2() as cm2, \ diff --git a/tests/data/cases/context_managers_39.py b/tests/data/cases/context_managers_39.py index 60fd1a56409..c9fcf9c8ba2 100644 --- a/tests/data/cases/context_managers_39.py +++ b/tests/data/cases/context_managers_39.py @@ -1,4 +1,3 @@ -# flags: --minimum-version=3.9 with \ make_context_manager1() as cm1, \ make_context_manager2() as cm2, \ diff --git a/tests/data/cases/context_managers_autodetect_39.py b/tests/data/cases/context_managers_autodetect_39.py index 98e674b2f9d..0d28f993108 100644 --- a/tests/data/cases/context_managers_autodetect_39.py +++ b/tests/data/cases/context_managers_autodetect_39.py @@ -1,4 +1,3 @@ -# flags: --minimum-version=3.9 # This file uses parenthesized context managers introduced in Python 3.9. diff --git a/tests/data/cases/docstring_newline_preview.py b/tests/data/cases/docstring_newline.py similarity index 83% rename from tests/data/cases/docstring_newline_preview.py rename to tests/data/cases/docstring_newline.py index 5c129ca5f80..75b8db48175 100644 --- a/tests/data/cases/docstring_newline_preview.py +++ b/tests/data/cases/docstring_newline.py @@ -1,4 +1,3 @@ -# flags: --preview """ 87 characters ............................................................................ """ diff --git a/tests/data/cases/fmtskip9.py b/tests/data/cases/fmtskip9.py index 30085bdd973..d070a7f17eb 100644 --- a/tests/data/cases/fmtskip9.py +++ b/tests/data/cases/fmtskip9.py @@ -1,4 +1,3 @@ -# flags: --preview print () # fmt: skip print () # fmt:skip diff --git a/tests/data/cases/preview_format_unicode_escape_seq.py b/tests/data/cases/format_unicode_escape_seq.py similarity index 96% rename from tests/data/cases/preview_format_unicode_escape_seq.py rename to tests/data/cases/format_unicode_escape_seq.py index 65c3d8d166e..3440696c303 100644 --- a/tests/data/cases/preview_format_unicode_escape_seq.py +++ b/tests/data/cases/format_unicode_escape_seq.py @@ -1,4 +1,3 @@ -# flags: --preview x = "\x1F" x = "\\x1B" x = "\\\x1B" diff --git a/tests/data/cases/funcdef_return_type_trailing_comma.py b/tests/data/cases/funcdef_return_type_trailing_comma.py index 14fd763d9d1..6335cf73396 100644 --- a/tests/data/cases/funcdef_return_type_trailing_comma.py +++ b/tests/data/cases/funcdef_return_type_trailing_comma.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.10 +# flags: --minimum-version=3.10 # normal, short, function definition def foo(a, b) -> tuple[int, float]: ... diff --git a/tests/data/cases/generics_wrapping.py b/tests/data/cases/generics_wrapping.py new file mode 100644 index 00000000000..734e2a3c752 --- /dev/null +++ b/tests/data/cases/generics_wrapping.py @@ -0,0 +1,307 @@ +# flags: --minimum-version=3.12 +def plain[T, B](a: T, b: T) -> T: + return a + +def arg_magic[T, B](a: T, b: T,) -> T: + return a + +def type_param_magic[T, B,](a: T, b: T) -> T: + return a + +def both_magic[T, B,](a: T, b: T,) -> T: + return a + + +def plain_multiline[ + T, + B +]( + a: T, + b: T +) -> T: + return a + +def arg_magic_multiline[ + T, + B +]( + a: T, + b: T, +) -> T: + return a + +def type_param_magic_multiline[ + T, + B, +]( + a: T, + b: T +) -> T: + return a + +def both_magic_multiline[ + T, + B, +]( + a: T, + b: T, +) -> T: + return a + + +def plain_mixed1[ + T, + B +](a: T, b: T) -> T: + return a + +def plain_mixed2[T, B]( + a: T, + b: T +) -> T: + return a + +def arg_magic_mixed1[ + T, + B +](a: T, b: T,) -> T: + return a + +def arg_magic_mixed2[T, B]( + a: T, + b: T, +) -> T: + return a + +def type_param_magic_mixed1[ + T, + B, +](a: T, b: T) -> T: + return a + +def type_param_magic_mixed2[T, B,]( + a: T, + b: T +) -> T: + return a + +def both_magic_mixed1[ + T, + B, +](a: T, b: T,) -> T: + return a + +def both_magic_mixed2[T, B,]( + a: T, + b: T, +) -> T: + return a + +def something_something_function[ + T: Model +](param: list[int], other_param: type[T], *, some_other_param: bool = True) -> QuerySet[ + T +]: + pass + + +def func[A_LOT_OF_GENERIC_TYPES: AreBeingDefinedHere, LIKE_THIS, AND_THIS, ANOTHER_ONE, AND_YET_ANOTHER_ONE: ThisOneHasTyping](a: T, b: T, c: T, d: T, e: T, f: T, g: T, h: T, i: T, j: T, k: T, l: T, m: T, n: T, o: T, p: T) -> T: + return a + + +def with_random_comments[ + Z + # bye +](): + return a + + +def func[ + T, # comment + U # comment + , + Z: # comment + int +](): pass + + +def func[ + T, # comment but it's long so it doesn't just move to the end of the line + U # comment comment comm comm ent ent + , + Z: # comment ent ent comm comm comment + int +](): pass + + +# output +def plain[T, B](a: T, b: T) -> T: + return a + + +def arg_magic[T, B]( + a: T, + b: T, +) -> T: + return a + + +def type_param_magic[ + T, + B, +]( + a: T, b: T +) -> T: + return a + + +def both_magic[ + T, + B, +]( + a: T, + b: T, +) -> T: + return a + + +def plain_multiline[T, B](a: T, b: T) -> T: + return a + + +def arg_magic_multiline[T, B]( + a: T, + b: T, +) -> T: + return a + + +def type_param_magic_multiline[ + T, + B, +]( + a: T, b: T +) -> T: + return a + + +def both_magic_multiline[ + T, + B, +]( + a: T, + b: T, +) -> T: + return a + + +def plain_mixed1[T, B](a: T, b: T) -> T: + return a + + +def plain_mixed2[T, B](a: T, b: T) -> T: + return a + + +def arg_magic_mixed1[T, B]( + a: T, + b: T, +) -> T: + return a + + +def arg_magic_mixed2[T, B]( + a: T, + b: T, +) -> T: + return a + + +def type_param_magic_mixed1[ + T, + B, +]( + a: T, b: T +) -> T: + return a + + +def type_param_magic_mixed2[ + T, + B, +]( + a: T, b: T +) -> T: + return a + + +def both_magic_mixed1[ + T, + B, +]( + a: T, + b: T, +) -> T: + return a + + +def both_magic_mixed2[ + T, + B, +]( + a: T, + b: T, +) -> T: + return a + + +def something_something_function[T: Model]( + param: list[int], other_param: type[T], *, some_other_param: bool = True +) -> QuerySet[T]: + pass + + +def func[ + A_LOT_OF_GENERIC_TYPES: AreBeingDefinedHere, + LIKE_THIS, + AND_THIS, + ANOTHER_ONE, + AND_YET_ANOTHER_ONE: ThisOneHasTyping, +]( + a: T, + b: T, + c: T, + d: T, + e: T, + f: T, + g: T, + h: T, + i: T, + j: T, + k: T, + l: T, + m: T, + n: T, + o: T, + p: T, +) -> T: + return a + + +def with_random_comments[ + Z + # bye +](): + return a + + +def func[T, U, Z: int](): # comment # comment # comment + pass + + +def func[ + T, # comment but it's long so it doesn't just move to the end of the line + U, # comment comment comm comm ent ent + Z: int, # comment ent ent comm comm comment +](): + pass diff --git a/tests/data/cases/is_simple_lookup_for_doublestar_expression.py b/tests/data/cases/is_simple_lookup_for_doublestar_expression.py index a0d2e2ba842..ae3643ba4e8 100644 --- a/tests/data/cases/is_simple_lookup_for_doublestar_expression.py +++ b/tests/data/cases/is_simple_lookup_for_doublestar_expression.py @@ -1,4 +1,3 @@ -# flags: --preview m2 = None if not isinstance(dist, Normal) else m** 2 + s * 2 m3 = None if not isinstance(dist, Normal) else m ** 2 + s * 2 m4 = None if not isinstance(dist, Normal) else m**2 + s * 2 diff --git a/tests/data/cases/preview_long_strings__type_annotations.py b/tests/data/cases/long_strings__type_annotations.py similarity index 98% rename from tests/data/cases/preview_long_strings__type_annotations.py rename to tests/data/cases/long_strings__type_annotations.py index 8beb877bdd1..45de882d02c 100644 --- a/tests/data/cases/preview_long_strings__type_annotations.py +++ b/tests/data/cases/long_strings__type_annotations.py @@ -1,4 +1,3 @@ -# flags: --preview def func( arg1, arg2, diff --git a/tests/data/cases/module_docstring_2.py b/tests/data/cases/module_docstring_2.py index ac486096c02..b1d0aaf4ab9 100644 --- a/tests/data/cases/module_docstring_2.py +++ b/tests/data/cases/module_docstring_2.py @@ -1,7 +1,6 @@ -# flags: --preview """I am a very helpful module docstring. -With trailing spaces (only removed with unify_docstring_detection on): +With trailing spaces: Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, @@ -39,7 +38,7 @@ # output """I am a very helpful module docstring. -With trailing spaces (only removed with unify_docstring_detection on): +With trailing spaces: Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, diff --git a/tests/data/cases/no_blank_line_before_docstring.py b/tests/data/cases/no_blank_line_before_docstring.py index ced125fef78..74d43cd7eaf 100644 --- a/tests/data/cases/no_blank_line_before_docstring.py +++ b/tests/data/cases/no_blank_line_before_docstring.py @@ -62,5 +62,4 @@ class MultilineDocstringsAsWell: class SingleQuotedDocstring: - "I'm a docstring but I don't even get triple quotes." diff --git a/tests/data/cases/pattern_matching_with_if_stmt.py b/tests/data/cases/pattern_matching_with_if_stmt.py index ff54af91771..1c5d58f16f2 100644 --- a/tests/data/cases/pattern_matching_with_if_stmt.py +++ b/tests/data/cases/pattern_matching_with_if_stmt.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.10 +# flags: --minimum-version=3.10 match match: case "test" if case != "not very loooooooooooooog condition": # comment pass diff --git a/tests/data/cases/preview_pep646_typed_star_arg_type_var_tuple.py b/tests/data/cases/pep646_typed_star_arg_type_var_tuple.py similarity index 62% rename from tests/data/cases/preview_pep646_typed_star_arg_type_var_tuple.py rename to tests/data/cases/pep646_typed_star_arg_type_var_tuple.py index fb79e9983b1..6dfb5445efe 100644 --- a/tests/data/cases/preview_pep646_typed_star_arg_type_var_tuple.py +++ b/tests/data/cases/pep646_typed_star_arg_type_var_tuple.py @@ -1,4 +1,4 @@ -# flags: --minimum-version=3.11 --preview +# flags: --minimum-version=3.11 def fn(*args: *tuple[*A, B]) -> None: diff --git a/tests/data/cases/pep_570.py b/tests/data/cases/pep_570.py index 2641c2b970e..ca8f7ab1d95 100644 --- a/tests/data/cases/pep_570.py +++ b/tests/data/cases/pep_570.py @@ -1,4 +1,3 @@ -# flags: --minimum-version=3.8 def positional_only_arg(a, /): pass diff --git a/tests/data/cases/pep_572.py b/tests/data/cases/pep_572.py index 742b6d5b7e4..d41805f1cb1 100644 --- a/tests/data/cases/pep_572.py +++ b/tests/data/cases/pep_572.py @@ -1,4 +1,3 @@ -# flags: --minimum-version=3.8 (a := 1) (a := a) if (match := pattern.search(data)) is None: diff --git a/tests/data/cases/pep_572_py39.py b/tests/data/cases/pep_572_py39.py index d1614624d99..b8b081b8c45 100644 --- a/tests/data/cases/pep_572_py39.py +++ b/tests/data/cases/pep_572_py39.py @@ -1,4 +1,3 @@ -# flags: --minimum-version=3.9 # Unparenthesized walruses are now allowed in set literals & set comprehensions # since Python 3.9 {x := 1, 2, 3} diff --git a/tests/data/cases/pep_572_remove_parens.py b/tests/data/cases/pep_572_remove_parens.py index f0026ceb032..75113771ae0 100644 --- a/tests/data/cases/pep_572_remove_parens.py +++ b/tests/data/cases/pep_572_remove_parens.py @@ -1,4 +1,3 @@ -# flags: --minimum-version=3.8 if (foo := 0): pass diff --git a/tests/data/cases/prefer_rhs_split_reformatted.py b/tests/data/cases/prefer_rhs_split_reformatted.py index e15e5ddc86d..2ec0728af82 100644 --- a/tests/data/cases/prefer_rhs_split_reformatted.py +++ b/tests/data/cases/prefer_rhs_split_reformatted.py @@ -11,6 +11,14 @@ # exactly line length limit + 1, it won't be split like that. xxxxxxxxx_yyy_zzzzzzzz[xx.xxxxxx(x_yyy_zzzzzz.xxxxx[0]), x_yyy_zzzzzz.xxxxxx(xxxx=1)] = 1 +# Regression test for #1187 +print( + dict( + a=1, + b=2 if some_kind_of_data is not None else some_other_kind_of_data, # some explanation of why this is actually necessary + c=3, + ) +) # output @@ -36,3 +44,14 @@ xxxxxxxxx_yyy_zzzzzzzz[ xx.xxxxxx(x_yyy_zzzzzz.xxxxx[0]), x_yyy_zzzzzz.xxxxxx(xxxx=1) ] = 1 + +# Regression test for #1187 +print( + dict( + a=1, + b=( + 2 if some_kind_of_data is not None else some_other_kind_of_data + ), # some explanation of why this is actually necessary + c=3, + ) +) diff --git a/tests/data/cases/preview_comments7.py b/tests/data/cases/preview_comments7.py index e4d547138db..703e3c8fbde 100644 --- a/tests/data/cases/preview_comments7.py +++ b/tests/data/cases/preview_comments7.py @@ -177,7 +177,6 @@ def test_fails_invalid_post_data( MyLovelyCompanyTeamProjectComponent as component, # DRY ) - result = 1 # look ma, no comment migration xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx result = 1 # look ma, no comment migration xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx diff --git a/tests/data/cases/preview_fstring.py b/tests/data/cases/preview_fstring.py new file mode 100644 index 00000000000..e0a3eacfa20 --- /dev/null +++ b/tests/data/cases/preview_fstring.py @@ -0,0 +1,2 @@ +# flags: --unstable +f"{''=}" f'{""=}' \ No newline at end of file diff --git a/tests/data/cases/preview_import_line_collapse.py b/tests/data/cases/preview_import_line_collapse.py new file mode 100644 index 00000000000..74ae349a2ca --- /dev/null +++ b/tests/data/cases/preview_import_line_collapse.py @@ -0,0 +1,180 @@ +# flags: --preview +from middleman.authentication import validate_oauth_token + + +logger = logging.getLogger(__name__) + + +# case 2 comment after import +from middleman.authentication import validate_oauth_token +#comment + +logger = logging.getLogger(__name__) + + +# case 3 comment after import +from middleman.authentication import validate_oauth_token +# comment +logger = logging.getLogger(__name__) + + +from middleman.authentication import validate_oauth_token + + + +logger = logging.getLogger(__name__) + + +# case 4 try catch with import after import +import os +import os + + + +try: + import os +except Exception: + pass + +try: + import os + def func(): + a = 1 +except Exception: + pass + + +# case 5 multiple imports +import os +import os + +import os +import os + + + + + +for i in range(10): + print(i) + + +# case 6 import in function +def func(): + print() + import os + def func(): + pass + print() + + +def func(): + import os + a = 1 + print() + + +def func(): + import os + + + a = 1 + print() + + +def func(): + import os + + + + a = 1 + print() + +# output + + +from middleman.authentication import validate_oauth_token + +logger = logging.getLogger(__name__) + + +# case 2 comment after import +from middleman.authentication import validate_oauth_token + +# comment + +logger = logging.getLogger(__name__) + + +# case 3 comment after import +from middleman.authentication import validate_oauth_token + +# comment +logger = logging.getLogger(__name__) + + +from middleman.authentication import validate_oauth_token + +logger = logging.getLogger(__name__) + + +# case 4 try catch with import after import +import os +import os + +try: + import os +except Exception: + pass + +try: + import os + + def func(): + a = 1 + +except Exception: + pass + + +# case 5 multiple imports +import os +import os + +import os +import os + +for i in range(10): + print(i) + + +# case 6 import in function +def func(): + print() + import os + + def func(): + pass + + print() + + +def func(): + import os + + a = 1 + print() + + +def func(): + import os + + a = 1 + print() + + +def func(): + import os + + a = 1 + print() diff --git a/tests/data/cases/preview_long_dict_values.py b/tests/data/cases/preview_long_dict_values.py index a19210605f6..c1b30f27e22 100644 --- a/tests/data/cases/preview_long_dict_values.py +++ b/tests/data/cases/preview_long_dict_values.py @@ -1,4 +1,25 @@ -# flags: --unstable +# flags: --preview +x = { + "xx_xxxxx_xxxxxxxxxx_xxxxxxxxx_xx": ( + "xx:xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxx{xx}xxx_xxxxx_xxxxxxxxx_xxxxxxxxxxxx_xxxx" + ) +} +x = { + "xx_xxxxx_xxxxxxxxxx_xxxxxxxxx_xx": ( + "xx:xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxx{xx}xxx_xxxxx_xxxxxxxxx_xxxxxxxxxxxx_xxxx" + ), +} +x = { + "foo": bar, + "foo": bar, + "foo": ( + xx_xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxxxxx_xxxxx_xxxxxxxxx_xxxxxxxxxxxx_xxxx + ), +} +x = { + "xx_xxxxx_xxxxxxxxxx_xxxxxxxxx_xx": "xx:xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxx" +} + my_dict = { "something_something": r"Lorem ipsum dolor sit amet, an sed convenire eloquentiam \t" @@ -6,23 +27,90 @@ r"signiferumque, duo ea vocibus consetetur scriptorem. Facer \t", } +# Function calls as keys +tasks = { + get_key_name( + foo, + bar, + baz, + ): src, + loop.run_in_executor(): src, + loop.run_in_executor(xx_xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxxxxx): src, + loop.run_in_executor( + xx_xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxxxxx_xxxxx_xxxxx + ): src, + loop.run_in_executor(): ( + xx_xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxxxxx_xxxxx_xxxxxxxxx_xxxxxxxxxxxx_xxxx + ), +} + +# Dictionary comprehensions +tasks = { + key_name: ( + xx_xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxxxxx_xxxxx_xxxxxxxxx_xxxxxxxxxxxx_xxxx + ) + for src in sources +} +tasks = {key_name: foobar for src in sources} +tasks = { + get_key_name( + src, + ): "foo" + for src in sources +} +tasks = { + get_key_name( + foo, + bar, + baz, + ): src + for src in sources +} +tasks = { + get_key_name(): ( + xx_xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxxxxx_xxxxx_xxxxxxxxx_xxxxxxxxxxxx_xxxx + ) + for src in sources +} +tasks = {get_key_name(): foobar for src in sources} + + +# Delimiters inside the value +def foo(): + def bar(): + x = { + common.models.DateTimeField: datetime(2020, 1, 31, tzinfo=utc) + timedelta( + days=i + ), + } + x = { + common.models.DateTimeField: ( + datetime(2020, 1, 31, tzinfo=utc) + timedelta(days=i) + ), + } + x = { + "foobar": (123 + 456), + } + x = { + "foobar": (123) + 456, + } + + my_dict = { "a key in my dict": a_very_long_variable * and_a_very_long_function_call() / 100000.0 } - my_dict = { "a key in my dict": a_very_long_variable * and_a_very_long_function_call() * and_another_long_func() / 100000.0 } - my_dict = { "a key in my dict": MyClass.some_attribute.first_call().second_call().third_call(some_args="some value") } { - 'xxxxxx': + "xxxxxx": xxxxxxxxxxxxxxxxxxx.xxxxxxxxxxxxxx( xxxxxxxxxxxxxx={ - 'x': + "x": xxxxxxxxxxxxxxxxxxxxxxxxxx.xxxxxxxxxxxxxxxxxxxxxxxxxxxxx( xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=( xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx @@ -30,8 +118,8 @@ xxxxxxxxxxxxx=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx .xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx( xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx={ - 'x': x.xx, - 'x': x.x, + "x": x.xx, + "x": x.x, })))) }), } @@ -58,7 +146,26 @@ def func(): # output - +x = { + "xx_xxxxx_xxxxxxxxxx_xxxxxxxxx_xx": ( + "xx:xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxx{xx}xxx_xxxxx_xxxxxxxxx_xxxxxxxxxxxx_xxxx" + ) +} +x = { + "xx_xxxxx_xxxxxxxxxx_xxxxxxxxx_xx": ( + "xx:xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxx{xx}xxx_xxxxx_xxxxxxxxx_xxxxxxxxxxxx_xxxx" + ), +} +x = { + "foo": bar, + "foo": bar, + "foo": ( + xx_xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxxxxx_xxxxx_xxxxxxxxx_xxxxxxxxxxxx_xxxx + ), +} +x = { + "xx_xxxxx_xxxxxxxxxx_xxxxxxxxx_xx": "xx:xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxx" +} my_dict = { "something_something": ( @@ -68,12 +175,80 @@ def func(): ), } +# Function calls as keys +tasks = { + get_key_name( + foo, + bar, + baz, + ): src, + loop.run_in_executor(): src, + loop.run_in_executor(xx_xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxxxxx): src, + loop.run_in_executor( + xx_xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxxxxx_xxxxx_xxxxx + ): src, + loop.run_in_executor(): ( + xx_xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxxxxx_xxxxx_xxxxxxxxx_xxxxxxxxxxxx_xxxx + ), +} + +# Dictionary comprehensions +tasks = { + key_name: ( + xx_xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxxxxx_xxxxx_xxxxxxxxx_xxxxxxxxxxxx_xxxx + ) + for src in sources +} +tasks = {key_name: foobar for src in sources} +tasks = { + get_key_name( + src, + ): "foo" + for src in sources +} +tasks = { + get_key_name( + foo, + bar, + baz, + ): src + for src in sources +} +tasks = { + get_key_name(): ( + xx_xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxxxxx_xxxxx_xxxxxxxxx_xxxxxxxxxxxx_xxxx + ) + for src in sources +} +tasks = {get_key_name(): foobar for src in sources} + + +# Delimiters inside the value +def foo(): + def bar(): + x = { + common.models.DateTimeField: ( + datetime(2020, 1, 31, tzinfo=utc) + timedelta(days=i) + ), + } + x = { + common.models.DateTimeField: ( + datetime(2020, 1, 31, tzinfo=utc) + timedelta(days=i) + ), + } + x = { + "foobar": 123 + 456, + } + x = { + "foobar": (123) + 456, + } + + my_dict = { "a key in my dict": ( a_very_long_variable * and_a_very_long_function_call() / 100000.0 ) } - my_dict = { "a key in my dict": ( a_very_long_variable @@ -82,7 +257,6 @@ def func(): / 100000.0 ) } - my_dict = { "a key in my dict": ( MyClass.some_attribute.first_call() @@ -113,8 +287,8 @@ def func(): class Random: def func(): - random_service.status.active_states.inactive = ( - make_new_top_level_state_from_dict({ + random_service.status.active_states.inactive = make_new_top_level_state_from_dict( + { "topLevelBase": { "secondaryBase": { "timestamp": 1234, @@ -125,5 +299,5 @@ def func(): ), } }, - }) + } ) diff --git a/tests/data/cases/preview_long_strings.py b/tests/data/cases/preview_long_strings.py index 86fa1b0c7e1..cf1d12b6e3e 100644 --- a/tests/data/cases/preview_long_strings.py +++ b/tests/data/cases/preview_long_strings.py @@ -279,7 +279,7 @@ def foo(): "........................................................................... \\N{LAO KO LA}" ) -msg = lambda x: f"this is a very very very long lambda value {x} that doesn't fit on a single line" +msg = lambda x: f"this is a very very very very long lambda value {x} that doesn't fit on a single line" dict_with_lambda_values = { "join": lambda j: ( @@ -329,6 +329,20 @@ def foo(): log.info(f"""Skipping: {'a' == 'b'} {desc['ms_name']} {money=} {dte=} {pos_share=} {desc['status']} {desc['exposure_max']}""") +x = { + "xx_xxxxx_xxxxxxxxxx_xxxxxxxxx_xx": ( + "xx:xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxx{xx}xxx_xxxxx_xxxxxxxxx_xxxxxxxxxxxx_xxxx" + ) +} +x = { + "xx_xxxxx_xxxxxxxxxx_xxxxxxxxx_xx": "xx:xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxx{xx}xxx_xxxxx_xxxxxxxxx_xxxxxxxxxxxx_xxxx", +} +x = { + "xx_xxxxx_xxxxxxxxxx_xxxxxxxxx_xx": ( + "xx:xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxx" + ) +} + # output @@ -842,11 +856,9 @@ def foo(): " \\N{LAO KO LA}" ) -msg = ( - lambda x: ( - f"this is a very very very long lambda value {x} that doesn't fit on a single" - " line" - ) +msg = lambda x: ( + f"this is a very very very very long lambda value {x} that doesn't fit on a" + " single line" ) dict_with_lambda_values = { @@ -882,7 +894,7 @@ def foo(): log.info( "Skipping:" - f" {desc['db_id']} {foo('bar',x=123)} {'foo' != 'bar'} {(x := 'abc=')} {pos_share=} {desc['status']} {desc['exposure_max']}" + f' {desc["db_id"]} {foo("bar",x=123)} {"foo" != "bar"} {(x := "abc=")} {pos_share=} {desc["status"]} {desc["exposure_max"]}' ) log.info( @@ -902,7 +914,7 @@ def foo(): log.info( "Skipping:" - f" {'a' == 'b' == 'c' == 'd'} {desc['ms_name']} {money=} {dte=} {pos_share=} {desc['status']} {desc['exposure_max']}" + f' {"a" == "b" == "c" == "d"} {desc["ms_name"]} {money=} {dte=} {pos_share=} {desc["status"]} {desc["exposure_max"]}' ) log.info( @@ -926,3 +938,17 @@ def foo(): log.info( f"""Skipping: {'a' == 'b'} {desc['ms_name']} {money=} {dte=} {pos_share=} {desc['status']} {desc['exposure_max']}""" ) + +x = { + "xx_xxxxx_xxxxxxxxxx_xxxxxxxxx_xx": ( + "xx:xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxx{xx}xxx_xxxxx_xxxxxxxxx_xxxxxxxxxxxx_xxxx" + ) +} +x = { + "xx_xxxxx_xxxxxxxxxx_xxxxxxxxx_xx": ( + "xx:xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxxx{xx}xxx_xxxxx_xxxxxxxxx_xxxxxxxxxxxx_xxxx" + ), +} +x = { + "xx_xxxxx_xxxxxxxxxx_xxxxxxxxx_xx": "xx:xxxxxxxxxxxxxxxxx_xxxxx_xxxxxxx_xxxxxxxxxx" +} diff --git a/tests/data/cases/preview_long_strings__regression.py b/tests/data/cases/preview_long_strings__regression.py index afe2b311cf4..123342f575c 100644 --- a/tests/data/cases/preview_long_strings__regression.py +++ b/tests/data/cases/preview_long_strings__regression.py @@ -552,6 +552,7 @@ async def foo(self): } # Regression test for https://github.com/psf/black/issues/3506. +# Regressed again by https://github.com/psf/black/pull/4498 s = ( "With single quote: ' " f" {my_dict['foo']}" @@ -1239,9 +1240,15 @@ async def foo(self): } # Regression test for https://github.com/psf/black/issues/3506. -s = f"With single quote: ' {my_dict['foo']} With double quote: \" {my_dict['bar']}" +# Regressed again by https://github.com/psf/black/pull/4498 +s = ( + "With single quote: ' " + f" {my_dict['foo']}" + ' With double quote: " ' + f' {my_dict["bar"]}' +) s = ( "Lorem Ipsum is simply dummy text of the printing and typesetting" - f" industry:'{my_dict['foo']}'" -) + f' industry:\'{my_dict["foo"]}\'' +) \ No newline at end of file diff --git a/tests/data/cases/preview_remove_multiline_lone_list_item_parens.py b/tests/data/cases/preview_remove_multiline_lone_list_item_parens.py new file mode 100644 index 00000000000..08563026245 --- /dev/null +++ b/tests/data/cases/preview_remove_multiline_lone_list_item_parens.py @@ -0,0 +1,246 @@ +# flags: --unstable +items = [(x for x in [1])] + +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) +] +items = [ + ( + {"key1": "val1", "key2": "val2"} + if some_var == "" + else {"key": "val"} + ) +] +items = [ + ( + "123456890123457890123468901234567890" + if some_var == "long strings" + else "123467890123467890" + ) +] +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + and some_var == "long strings" + and {"key": "val"} + ) +] +items = [ + ( + "123456890123457890123468901234567890" + and some_var == "long strings" + and "123467890123467890" + ) +] +items = [ + ( + long_variable_name + and even_longer_variable_name + and yet_another_very_long_variable_name + ) +] + +# Shouldn't remove trailing commas +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ), +] +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + and some_var == "long strings" + and {"key": "val"} + ), +] +items = [ + ( + "123456890123457890123468901234567890" + and some_var == "long strings" + and "123467890123467890" + ), +] +items = [ + ( + long_variable_name + and even_longer_variable_name + and yet_another_very_long_variable_name + ), +] + +# Shouldn't add parentheses +items = [ + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} +] +items = [{"key1": "val1", "key2": "val2"} if some_var == "" else {"key": "val"}] + +# Shouldn't crash with comments +items = [ + ( # comment + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) +] +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) # comment +] + +items = [ # comment + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) +] +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) +] # comment + +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} # comment + if some_var == "long strings" + else {"key": "val"} + ) +] + +items = [ # comment + ( # comment + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) +] +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) # comment +] # comment + + +# output +items = [(x for x in [1])] + +items = [ + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} +] +items = [{"key1": "val1", "key2": "val2"} if some_var == "" else {"key": "val"}] +items = [ + "123456890123457890123468901234567890" + if some_var == "long strings" + else "123467890123467890" +] +items = [ + {"key1": "val1", "key2": "val2", "key3": "val3"} + and some_var == "long strings" + and {"key": "val"} +] +items = [ + "123456890123457890123468901234567890" + and some_var == "long strings" + and "123467890123467890" +] +items = [ + long_variable_name + and even_longer_variable_name + and yet_another_very_long_variable_name +] + +# Shouldn't remove trailing commas +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ), +] +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + and some_var == "long strings" + and {"key": "val"} + ), +] +items = [ + ( + "123456890123457890123468901234567890" + and some_var == "long strings" + and "123467890123467890" + ), +] +items = [ + ( + long_variable_name + and even_longer_variable_name + and yet_another_very_long_variable_name + ), +] + +# Shouldn't add parentheses +items = [ + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} +] +items = [{"key1": "val1", "key2": "val2"} if some_var == "" else {"key": "val"}] + +# Shouldn't crash with comments +items = [ # comment + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} +] +items = [ + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} +] # comment + +items = [ # comment + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} +] +items = [ + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} +] # comment + +items = [ + {"key1": "val1", "key2": "val2", "key3": "val3"} # comment + if some_var == "long strings" + else {"key": "val"} +] + +items = [ # comment # comment + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} +] +items = [ + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} +] # comment # comment diff --git a/tests/data/cases/python37.py b/tests/data/cases/python37.py index 3f61106c45d..f69f6b4e58c 100644 --- a/tests/data/cases/python37.py +++ b/tests/data/cases/python37.py @@ -1,6 +1,3 @@ -# flags: --minimum-version=3.7 - - def f(): return (i * 2 async for i in arange(42)) diff --git a/tests/data/cases/python38.py b/tests/data/cases/python38.py index 919ea6aeed4..715641b1871 100644 --- a/tests/data/cases/python38.py +++ b/tests/data/cases/python38.py @@ -1,6 +1,3 @@ -# flags: --minimum-version=3.8 - - def starred_return(): my_list = ["value2", "value3"] return "value1", *my_list diff --git a/tests/data/cases/python39.py b/tests/data/cases/python39.py index 85eddc38e00..719c2b55745 100644 --- a/tests/data/cases/python39.py +++ b/tests/data/cases/python39.py @@ -1,5 +1,3 @@ -# flags: --minimum-version=3.9 - @relaxed_decorator[0] def f(): ... diff --git a/tests/data/cases/remove_lone_list_item_parens.py b/tests/data/cases/remove_lone_list_item_parens.py new file mode 100644 index 00000000000..8127e038e1c --- /dev/null +++ b/tests/data/cases/remove_lone_list_item_parens.py @@ -0,0 +1,157 @@ +items = [(123)] +items = [(True)] +items = [(((((True)))))] +items = [(((((True,)))))] +items = [((((()))))] +items = [(x for x in [1])] +items = {(123)} +items = {(True)} +items = {(((((True)))))} + +# Requires `hug_parens_with_braces_and_square_brackets` unstable style to remove parentheses +# around multiline values +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) +] +items = [ + ( + {"key1": "val1", "key2": "val2"} + if some_var == "" + else {"key": "val"} + ) +] + +# Comments should not cause crashes +items = [ + ( # comment + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) +] +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) # comment +] + +items = [ # comment + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) +] +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) +] # comment + +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} # comment + if some_var == "long strings" + else {"key": "val"} + ) +] + +items = [ # comment + ( # comment + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) +] +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) # comment +] # comment + + +# output +items = [123] +items = [True] +items = [True] +items = [(True,)] +items = [()] +items = [(x for x in [1])] +items = {123} +items = {True} +items = {True} + +# Requires `hug_parens_with_braces_and_square_brackets` unstable style to remove parentheses +# around multiline values +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) +] +items = [{"key1": "val1", "key2": "val2"} if some_var == "" else {"key": "val"}] + +# Comments should not cause crashes +items = [ + ( # comment + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) +] +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) # comment +] + +items = [ # comment + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) +] +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) +] # comment + +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} # comment + if some_var == "long strings" + else {"key": "val"} + ) +] + +items = [ # comment + ( # comment + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) +] +items = [ + ( + {"key1": "val1", "key2": "val2", "key3": "val3"} + if some_var == "long strings" + else {"key": "val"} + ) # comment +] # comment diff --git a/tests/data/cases/remove_redundant_parens_in_case_guard.py b/tests/data/cases/remove_redundant_parens_in_case_guard.py index bec4a3c3fcd..4739fc5478e 100644 --- a/tests/data/cases/remove_redundant_parens_in_case_guard.py +++ b/tests/data/cases/remove_redundant_parens_in_case_guard.py @@ -1,4 +1,4 @@ -# flags: --minimum-version=3.10 --preview --line-length=79 +# flags: --minimum-version=3.10 --line-length=79 match 1: case _ if (True): diff --git a/tests/data/cases/remove_with_brackets.py b/tests/data/cases/remove_with_brackets.py index 3ee64902a30..f2319e0da84 100644 --- a/tests/data/cases/remove_with_brackets.py +++ b/tests/data/cases/remove_with_brackets.py @@ -1,4 +1,3 @@ -# flags: --minimum-version=3.9 with (open("bla.txt")): pass @@ -54,6 +53,19 @@ with ((((CtxManager1()))) as example1, (((CtxManager2()))) as example2): ... +# regression tests for #3678 +with (a, *b): + pass + +with (a, (b, *c)): + pass + +with (a for b in c): + pass + +with (a, (b for c in d)): + pass + # output with open("bla.txt"): pass @@ -118,3 +130,16 @@ with CtxManager1() as example1, CtxManager2() as example2: ... + +# regression tests for #3678 +with (a, *b): + pass + +with a, (b, *c): + pass + +with (a for b in c): + pass + +with a, (b for c in d): + pass diff --git a/tests/data/cases/skip_magic_trailing_comma_generic_wrap.py b/tests/data/cases/skip_magic_trailing_comma_generic_wrap.py new file mode 100644 index 00000000000..a833f3df863 --- /dev/null +++ b/tests/data/cases/skip_magic_trailing_comma_generic_wrap.py @@ -0,0 +1,163 @@ +# flags: --minimum-version=3.12 --skip-magic-trailing-comma +def plain[T, B](a: T, b: T) -> T: + return a + +def arg_magic[T, B](a: T, b: T,) -> T: + return a + +def type_param_magic[T, B,](a: T, b: T) -> T: + return a + +def both_magic[T, B,](a: T, b: T,) -> T: + return a + + +def plain_multiline[ + T, + B +]( + a: T, + b: T +) -> T: + return a + +def arg_magic_multiline[ + T, + B +]( + a: T, + b: T, +) -> T: + return a + +def type_param_magic_multiline[ + T, + B, +]( + a: T, + b: T +) -> T: + return a + +def both_magic_multiline[ + T, + B, +]( + a: T, + b: T, +) -> T: + return a + + +def plain_mixed1[ + T, + B +](a: T, b: T) -> T: + return a + +def plain_mixed2[T, B]( + a: T, + b: T +) -> T: + return a + +def arg_magic_mixed1[ + T, + B +](a: T, b: T,) -> T: + return a + +def arg_magic_mixed2[T, B]( + a: T, + b: T, +) -> T: + return a + +def type_param_magic_mixed1[ + T, + B, +](a: T, b: T) -> T: + return a + +def type_param_magic_mixed2[T, B,]( + a: T, + b: T +) -> T: + return a + +def both_magic_mixed1[ + T, + B, +](a: T, b: T,) -> T: + return a + +def both_magic_mixed2[T, B,]( + a: T, + b: T, +) -> T: + return a + + +# output +def plain[T, B](a: T, b: T) -> T: + return a + + +def arg_magic[T, B](a: T, b: T) -> T: + return a + + +def type_param_magic[T, B](a: T, b: T) -> T: + return a + + +def both_magic[T, B](a: T, b: T) -> T: + return a + + +def plain_multiline[T, B](a: T, b: T) -> T: + return a + + +def arg_magic_multiline[T, B](a: T, b: T) -> T: + return a + + +def type_param_magic_multiline[T, B](a: T, b: T) -> T: + return a + + +def both_magic_multiline[T, B](a: T, b: T) -> T: + return a + + +def plain_mixed1[T, B](a: T, b: T) -> T: + return a + + +def plain_mixed2[T, B](a: T, b: T) -> T: + return a + + +def arg_magic_mixed1[T, B](a: T, b: T) -> T: + return a + + +def arg_magic_mixed2[T, B](a: T, b: T) -> T: + return a + + +def type_param_magic_mixed1[T, B](a: T, b: T) -> T: + return a + + +def type_param_magic_mixed2[T, B](a: T, b: T) -> T: + return a + + +def both_magic_mixed1[T, B](a: T, b: T) -> T: + return a + + +def both_magic_mixed2[T, B](a: T, b: T) -> T: + return a diff --git a/tests/data/cases/type_param_defaults.py b/tests/data/cases/type_param_defaults.py index cd844fe0746..feba64a2c72 100644 --- a/tests/data/cases/type_param_defaults.py +++ b/tests/data/cases/type_param_defaults.py @@ -37,25 +37,27 @@ def trailing_comma2[T=int](a: str,): ] = something_that_is_long -def simple[ - T = something_that_is_long -](short1: int, short2: str, short3: bytes) -> float: +def simple[T = something_that_is_long]( + short1: int, short2: str, short3: bytes +) -> float: pass -def longer[ - something_that_is_long = something_that_is_long -](something_that_is_long: something_that_is_long) -> something_that_is_long: +def longer[something_that_is_long = something_that_is_long]( + something_that_is_long: something_that_is_long, +) -> something_that_is_long: pass def trailing_comma1[ T = int, -](a: str): +]( + a: str, +): pass -def trailing_comma2[ - T = int -](a: str,): +def trailing_comma2[T = int]( + a: str, +): pass diff --git a/tests/data/cases/typed_params_trailing_comma.py b/tests/data/cases/typed_params_trailing_comma.py index a53b908b18b..5bcdb941966 100644 --- a/tests/data/cases/typed_params_trailing_comma.py +++ b/tests/data/cases/typed_params_trailing_comma.py @@ -1,4 +1,3 @@ -# flags: --preview def long_function_name_goes_here( x: Callable[List[int]] ) -> Union[List[int], float, str, bytes, Tuple[int]]: diff --git a/tests/data/cases/walrus_in_dict.py b/tests/data/cases/walrus_in_dict.py index c91ad9e8611..68ec5d5df2f 100644 --- a/tests/data/cases/walrus_in_dict.py +++ b/tests/data/cases/walrus_in_dict.py @@ -1,9 +1,9 @@ # flags: --preview -# This is testing an issue that is specific to the preview style +# This is testing an issue that is specific to the preview style (wrap_long_dict_values_in_parens) { "is_update": (up := commit.hash in update_hashes) } # output -# This is testing an issue that is specific to the preview style +# This is testing an issue that is specific to the preview style (wrap_long_dict_values_in_parens) {"is_update": (up := commit.hash in update_hashes)} diff --git a/tests/optional.py b/tests/optional.py index 142da844898..018d602f284 100644 --- a/tests/optional.py +++ b/tests/optional.py @@ -18,7 +18,7 @@ import logging import re from functools import lru_cache -from typing import TYPE_CHECKING, FrozenSet, List, Set +from typing import TYPE_CHECKING import pytest @@ -46,8 +46,8 @@ from _pytest.nodes import Node -ALL_POSSIBLE_OPTIONAL_MARKERS = StashKey[FrozenSet[str]]() -ENABLED_OPTIONAL_MARKERS = StashKey[FrozenSet[str]]() +ALL_POSSIBLE_OPTIONAL_MARKERS = StashKey[frozenset[str]]() +ENABLED_OPTIONAL_MARKERS = StashKey[frozenset[str]]() def pytest_addoption(parser: "Parser") -> None: @@ -69,7 +69,7 @@ def pytest_configure(config: "Config") -> None: """ ot_ini = config.inicfg.get("optional-tests") or [] ot_markers = set() - ot_run: Set[str] = set() + ot_run: set[str] = set() if isinstance(ot_ini, str): ot_ini = ot_ini.strip().split("\n") marker_re = re.compile(r"^\s*(?Pno_)?(?P\w+)(:\s*(?P.*))?") @@ -103,7 +103,7 @@ def pytest_configure(config: "Config") -> None: store[ENABLED_OPTIONAL_MARKERS] = frozenset(ot_run) -def pytest_collection_modifyitems(config: "Config", items: "List[Node]") -> None: +def pytest_collection_modifyitems(config: "Config", items: "list[Node]") -> None: store = config._store all_possible_optional_markers = store[ALL_POSSIBLE_OPTIONAL_MARKERS] enabled_optional_markers = store[ENABLED_OPTIONAL_MARKERS] @@ -120,7 +120,7 @@ def pytest_collection_modifyitems(config: "Config", items: "List[Node]") -> None @lru_cache -def skip_mark(tests: FrozenSet[str]) -> "MarkDecorator": +def skip_mark(tests: frozenset[str]) -> "MarkDecorator": names = ", ".join(sorted(tests)) return pytest.mark.skip(reason=f"Marked with disabled optional tests ({names})") diff --git a/tests/test_black.py b/tests/test_black.py index 4a8452065e9..3b20268af3c 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -10,6 +10,7 @@ import sys import textwrap import types +from collections.abc import Callable, Iterator, Sequence from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager, redirect_stderr from dataclasses import fields, replace @@ -17,19 +18,7 @@ from pathlib import Path, WindowsPath from platform import system from tempfile import TemporaryDirectory -from typing import ( - Any, - Callable, - Dict, - Iterator, - List, - Optional, - Sequence, - Set, - Type, - TypeVar, - Union, -) +from typing import Any, Optional, TypeVar, Union from unittest.mock import MagicMock, patch import click @@ -107,11 +96,11 @@ class FakeContext(click.Context): """A fake click Context for when calling functions that need it.""" def __init__(self) -> None: - self.default_map: Dict[str, Any] = {} - self.params: Dict[str, Any] = {} + self.default_map: dict[str, Any] = {} + self.params: dict[str, Any] = {} self.command: click.Command = pyink.main # Dummy root, since most of the tests don't care about it - self.obj: Dict[str, Any] = {"root": PROJECT_ROOT} + self.obj: dict[str, Any] = {"root": PROJECT_ROOT} class FakeParameter(click.Parameter): @@ -129,7 +118,7 @@ def __init__(self) -> None: def invokeBlack( - args: List[str], exit_code: int = 0, ignore_config: bool = True + args: list[str], exit_code: int = 0, ignore_config: bool = True ) -> None: runner = BlackRunner() if ignore_config: @@ -933,7 +922,7 @@ def test_get_features_used(self) -> None: "with ((a, ((b as c)))): pass", {Feature.PARENTHESIZED_CONTEXT_MANAGERS} ) - def check_features_used(self, source: str, expected: Set[Feature]) -> None: + def check_features_used(self, source: str, expected: set[Feature]) -> None: node = pyink.lib2to3_parse(source) actual = pyink.get_features_used(node) msg = f"Expected {expected} but got {actual} for {source!r}" @@ -1365,9 +1354,11 @@ def test_reformat_one_with_stdin_empty(self) -> None: ] def _new_wrapper( - output: io.StringIO, io_TextIOWrapper: Type[io.TextIOWrapper] - ) -> Callable[[Any, Any], io.TextIOWrapper]: - def get_output(*args: Any, **kwargs: Any) -> io.TextIOWrapper: + output: io.StringIO, io_TextIOWrapper: type[io.TextIOWrapper] + ) -> Callable[[Any, Any], Union[io.StringIO, io.TextIOWrapper]]: + def get_output( + *args: Any, **kwargs: Any + ) -> Union[io.StringIO, io.TextIOWrapper]: if args == (sys.stdout.buffer,): # It's `format_stdin_to_stdout()` calling `io.TextIOWrapper()`, # return our mock object. @@ -2350,7 +2341,7 @@ def test_read_cache_line_lengths(self) -> None: def test_cache_key(self) -> None: # Test that all members of the mode enum affect the cache key. for field in fields(Mode): - values: List[Any] + values: list[Any] if field.name == "target_versions": values = [ {TargetVersion.PY312}, @@ -2362,8 +2353,8 @@ def test_cache_key(self) -> None: # If you are looking to remove one of these features, just # replace it with any other feature. values = [ - {Preview.docstring_check_for_newline}, - {Preview.hex_codes_in_unicode_sequences}, + {Preview.multiline_string_handling}, + {Preview.string_processing}, ] elif field.type is Quote: values = list(Quote) @@ -2476,7 +2467,7 @@ def test_gitignore_exclude(self) -> None: gitignore = PathSpec.from_lines( "gitwildmatch", ["exclude/", ".definitely_exclude"] ) - sources: List[Path] = [] + sources: list[Path] = [] expected = [ Path(path / "b/dont_exclude/a.py"), Path(path / "b/dont_exclude/a.pyi"), @@ -2504,7 +2495,7 @@ def test_nested_gitignore(self) -> None: exclude = re.compile(r"") root_gitignore = pyink.files.get_gitignore(path) report = pyink.Report() - expected: List[Path] = [ + expected: list[Path] = [ Path(path / "x.py"), Path(path / "root/b.py"), Path(path / "root/c.py"), diff --git a/tests/test_docs.py b/tests/test_docs.py index 761f46dbcb4..ba90a7dfcb6 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -5,9 +5,10 @@ """ import re +from collections.abc import Sequence from itertools import islice from pathlib import Path -from typing import Optional, Sequence, Set +from typing import Optional import pytest @@ -17,7 +18,7 @@ def check_feature_list( - lines: Sequence[str], expected_feature_names: Set[str], label: str + lines: Sequence[str], expected_feature_names: set[str], label: str ) -> Optional[str]: start_index = lines.index(f"(labels/{label}-features)=\n") if start_index == -1: diff --git a/tests/test_format.py b/tests/test_format.py index 0fd0190fc98..072574d2338 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -1,5 +1,6 @@ +from collections.abc import Iterator from dataclasses import replace -from typing import Any, Iterator +from typing import Any from unittest.mock import patch import pytest diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 971d3427e87..fdcd37b271b 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -1,9 +1,9 @@ import contextlib import pathlib import re +from contextlib import AbstractContextManager from contextlib import ExitStack as does_not_raise from dataclasses import replace -from typing import ContextManager import pytest from _pytest.monkeypatch import MonkeyPatch @@ -221,7 +221,7 @@ def test_cell_magic_with_empty_lines(src: str, expected: str) -> None: ], ) def test_cell_magic_with_custom_python_magic( - mode: Mode, expected_output: str, expectation: ContextManager[object] + mode: Mode, expected_output: str, expectation: AbstractContextManager[object] ) -> None: with expectation: result = format_cell( @@ -619,8 +619,8 @@ def test_ipynb_and_pyi_flags() -> None: def test_unable_to_replace_magics(monkeypatch: MonkeyPatch) -> None: - src = "%%time\na = 'foo'" - monkeypatch.setattr("pyink.handle_ipynb_magics.TOKEN_HEX", lambda _: "foo") + src = '%%time\na = b"foo"' + monkeypatch.setattr("secrets.token_hex", lambda _: "foo") with pytest.raises( AssertionError, match="Black was not able to replace IPython magic" ): diff --git a/tests/test_ranges.py b/tests/test_ranges.py index 39cd78d247d..d497715e5ca 100644 --- a/tests/test_ranges.py +++ b/tests/test_ranges.py @@ -1,7 +1,5 @@ """Test the pyink.ranges module.""" -from typing import List, Tuple - import pytest from pyink.ranges import adjusted_lines, sanitized_lines @@ -11,7 +9,7 @@ "lines", [[(1, 1)], [(1, 3)], [(1, 1), (3, 4)]], ) -def test_no_diff(lines: List[Tuple[int, int]]) -> None: +def test_no_diff(lines: list[tuple[int, int]]) -> None: source = """\ import re @@ -32,7 +30,7 @@ def func(): [(0, 8), (3, 1)], ], ) -def test_invalid_lines(lines: List[Tuple[int, int]]) -> None: +def test_invalid_lines(lines: list[tuple[int, int]]) -> None: original_source = """\ import re def foo(arg): @@ -83,7 +81,7 @@ def func(arg1, arg2, arg3): ], ) def test_removals( - lines: List[Tuple[int, int]], adjusted: List[Tuple[int, int]] + lines: list[tuple[int, int]], adjusted: list[tuple[int, int]] ) -> None: original_source = """\ 1. first line @@ -118,7 +116,7 @@ def test_removals( ], ) def test_additions( - lines: List[Tuple[int, int]], adjusted: List[Tuple[int, int]] + lines: list[tuple[int, int]], adjusted: list[tuple[int, int]] ) -> None: original_source = """\ 1. first line @@ -154,7 +152,7 @@ def test_additions( ([(9, 10), (1, 1)], [(1, 1), (9, 9)]), ], ) -def test_diffs(lines: List[Tuple[int, int]], adjusted: List[Tuple[int, int]]) -> None: +def test_diffs(lines: list[tuple[int, int]], adjusted: list[tuple[int, int]]) -> None: original_source = """\ 1. import re 2. def foo(arg): @@ -231,7 +229,7 @@ def test_diffs(lines: List[Tuple[int, int]], adjusted: List[Tuple[int, int]]) -> ], ) def test_sanitize( - lines: List[Tuple[int, int]], sanitized: List[Tuple[int, int]] + lines: list[tuple[int, int]], sanitized: list[tuple[int, int]] ) -> None: source = """\ 1. import re diff --git a/tests/test_tokenize.py b/tests/test_tokenize.py index 216669bf9b3..9e534e775c4 100644 --- a/tests/test_tokenize.py +++ b/tests/test_tokenize.py @@ -4,7 +4,6 @@ import sys import textwrap from dataclasses import dataclass -from typing import List import pyink from blib2to3.pgen2 import token, tokenize @@ -18,10 +17,10 @@ class Token: end: tokenize.Coord -def get_tokens(text: str) -> List[Token]: +def get_tokens(text: str) -> list[Token]: """Return the tokens produced by the tokenizer.""" readline = io.StringIO(text).readline - tokens: List[Token] = [] + tokens: list[Token] = [] def tokeneater( type: int, string: str, start: tokenize.Coord, end: tokenize.Coord, line: str @@ -32,7 +31,7 @@ def tokeneater( return tokens -def assert_tokenizes(text: str, tokens: List[Token]) -> None: +def assert_tokenizes(text: str, tokens: list[Token]) -> None: """Assert that the tokenizer produces the expected tokens.""" actual_tokens = get_tokens(text) assert actual_tokens == tokens diff --git a/tests/test_trans.py b/tests/test_trans.py index d5d21b176a5..1c2c63419b6 100644 --- a/tests/test_trans.py +++ b/tests/test_trans.py @@ -1,11 +1,9 @@ -from typing import List, Tuple - from pyink.trans import iter_fexpr_spans def test_fexpr_spans() -> None: def check( - string: str, expected_spans: List[Tuple[int, int]], expected_slices: List[str] + string: str, expected_spans: list[tuple[int, int]], expected_slices: list[str] ) -> None: spans = list(iter_fexpr_spans(string)) diff --git a/tests/util.py b/tests/util.py index 11534f403c5..84db1e16e05 100644 --- a/tests/util.py +++ b/tests/util.py @@ -4,11 +4,12 @@ import shlex import sys import unittest +from collections.abc import Collection, Iterator from contextlib import contextmanager from dataclasses import dataclass, field, replace from functools import partial from pathlib import Path -from typing import Any, Collection, Iterator, List, Optional, Tuple +from typing import Any, Optional import pyink from pyink.const import DEFAULT_LINE_LENGTH @@ -44,8 +45,8 @@ class TestCaseArgs: mode: pyink.Mode = field(default_factory=pyink.Mode) fast: bool = False - minimum_version: Optional[Tuple[int, int]] = None - lines: Collection[Tuple[int, int]] = () + minimum_version: Optional[tuple[int, int]] = None + lines: Collection[tuple[int, int]] = () no_preview_line_length_1: bool = False @@ -95,8 +96,8 @@ def assert_format( mode: pyink.Mode = DEFAULT_MODE, *, fast: bool = False, - minimum_version: Optional[Tuple[int, int]] = None, - lines: Collection[Tuple[int, int]] = (), + minimum_version: Optional[tuple[int, int]] = None, + lines: Collection[tuple[int, int]] = (), no_preview_line_length_1: bool = False, ) -> None: """Convenience function to check that Black formats as expected. @@ -164,8 +165,8 @@ def _assert_format_inner( mode: pyink.Mode = DEFAULT_MODE, *, fast: bool = False, - minimum_version: Optional[Tuple[int, int]] = None, - lines: Collection[Tuple[int, int]] = (), + minimum_version: Optional[tuple[int, int]] = None, + lines: Collection[tuple[int, int]] = (), ) -> None: actual = pyink.format_str(source, mode=mode, lines=lines) if expected is not None: @@ -195,7 +196,7 @@ def get_base_dir(data: bool) -> Path: return DATA_DIR if data else PROJECT_ROOT -def all_data_cases(subdir_name: str, data: bool = True) -> List[str]: +def all_data_cases(subdir_name: str, data: bool = True) -> list[str]: cases_dir = get_base_dir(data) / subdir_name assert cases_dir.is_dir() return [case_path.stem for case_path in cases_dir.iterdir()] @@ -214,18 +215,18 @@ def get_case_path( def read_data_with_mode( subdir_name: str, name: str, data: bool = True -) -> Tuple[TestCaseArgs, str, str]: +) -> tuple[TestCaseArgs, str, str]: """read_data_with_mode('test_name') -> Mode(), 'input', 'output'""" return read_data_from_file(get_case_path(subdir_name, name, data)) -def read_data(subdir_name: str, name: str, data: bool = True) -> Tuple[str, str]: +def read_data(subdir_name: str, name: str, data: bool = True) -> tuple[str, str]: """read_data('test_name') -> 'input', 'output'""" _, input, output = read_data_with_mode(subdir_name, name, data) return input, output -def _parse_minimum_version(version: str) -> Tuple[int, int]: +def _parse_minimum_version(version: str) -> tuple[int, int]: major, minor = version.split(".") return int(major), int(minor) @@ -310,11 +311,11 @@ def parse_mode(flags_line: str) -> TestCaseArgs: ) -def read_data_from_file(file_name: Path) -> Tuple[TestCaseArgs, str, str]: +def read_data_from_file(file_name: Path) -> tuple[TestCaseArgs, str, str]: with open(file_name, encoding="utf8") as test: lines = test.readlines() - _input: List[str] = [] - _output: List[str] = [] + _input: list[str] = [] + _output: list[str] = [] result = _input mode = TestCaseArgs() for line in lines: diff --git a/tox.ini b/tox.ini index ca51e590d7f..36af9c23cab 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] isolated_build = true -envlist = {,ci-}py{38,39,310,311,py3},fuzz,run_self,generate_schema +envlist = {,ci-}py{39,310,311,312,313,py3},fuzz,run_self,generate_schema [testenv] setenv =