From c6f822a5cf16466709b19066f4fb2292d6ca0795 Mon Sep 17 00:00:00 2001 From: Jesse Clemens Date: Tue, 22 Aug 2023 17:18:36 -0400 Subject: [PATCH 01/11] Add a function that calculates the max wrapped line length for functions This should be able to recurisively work its way down a stack of nested function calls to figure out whether wrapping is actually necessary. --- yapf/yapflib/format_decision_state.py | 58 ++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py index 06f3455d9..c98a097e7 100644 --- a/yapf/yapflib/format_decision_state.py +++ b/yapf/yapflib/format_decision_state.py @@ -515,7 +515,14 @@ def SurroundedByParens(token): (opening.matching_bracket.next_token and opening.matching_bracket.next_token.value != ',' and not opening.matching_bracket.next_token.ClosesScope())): - return True + + argLengths = _CalculateArgLengths(opening) + startCol = self.column + len(current.value) + len(opening.value) + for length in argLengths: + length += startCol + if length > self.column_limit: + return True + return False if (previous.OpensScope() and not current.OpensScope() and not current.is_comment and @@ -1218,6 +1225,55 @@ def _ScopeHasNoCommas(token): return True +def _CalculateArgLengths(opening): + """Calculate the length of each function arg, if the args are wrapped""" + argList = list() + token = opening.next_token + shortList = list() + deltaList = list() + delta = 0 + while token.name in ("NAME", "EQUAL", "DOT", "STRING", "NUMBER"): + shortList.append(token) + token = token.next_token + if token.name == "COMMA": + shortList.append(token) + argList.append(shortList) + deltaList.append(delta) + shortList = list() + delta = 0 + token = token.next_token + elif token.name == "LPAR": + shortList.append(token) + if _IsFunctionCallWithArguments(token.previous_token): + maxArg = max(_CalculateArgLengths(token)) + endToken = token.matching_bracket + innerLength = endToken.total_length - token.total_length + delta = innerLength - maxArg + token = token.matching_bracket + if token.next_token.name == "COMMA": + token = token.next_token + shortList.append(token) + argList.append(shortList) + deltaList.append(delta) + shortList = list() + delta = 0 + else: + shortList.append(token) + token = token.next_token + elif token.name == "RPAR": + shortList.append(token) + deltaList.append(delta) + shortList = list() + delta = 0 + break + + argLengths = list() + for l, d in zip(argList, deltaList): + argLen = l[-1].total_length - l[0].total_length + len(l[0].value) - d + argLengths.append(argLen) + return argLengths + + class _ParenState(object): """Maintains the state of the bracket enclosures. From 5a656f09b0a07a29b0ac1df8b359a64527478318 Mon Sep 17 00:00:00 2001 From: Jesse Clemens Date: Tue, 22 Aug 2023 17:40:15 -0400 Subject: [PATCH 02/11] Fix a bug where not all arguments were correctly parsing Need to parse everything that isn't a special character instead of creating a restricted set of things that I will parse --- yapf/yapflib/format_decision_state.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py index c98a097e7..60130e31a 100644 --- a/yapf/yapflib/format_decision_state.py +++ b/yapf/yapflib/format_decision_state.py @@ -1232,8 +1232,12 @@ def _CalculateArgLengths(opening): shortList = list() deltaList = list() delta = 0 - while token.name in ("NAME", "EQUAL", "DOT", "STRING", "NUMBER"): + while True: shortList.append(token) + if token.name == "RPAR": + argList.append(shortList) + deltaList.append(delta) + break token = token.next_token if token.name == "COMMA": shortList.append(token) @@ -1262,9 +1266,8 @@ def _CalculateArgLengths(opening): token = token.next_token elif token.name == "RPAR": shortList.append(token) + argList.append(shortList) deltaList.append(delta) - shortList = list() - delta = 0 break argLengths = list() From 00ae91b35f40df7b95f17d7986767335d0a99ac8 Mon Sep 17 00:00:00 2001 From: Jesse Clemens Date: Thu, 24 Aug 2023 12:11:29 -0400 Subject: [PATCH 03/11] Simplify the loop logic and add a check for token existence --- yapf/yapflib/format_decision_state.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py index 60130e31a..0da5e97c9 100644 --- a/yapf/yapflib/format_decision_state.py +++ b/yapf/yapflib/format_decision_state.py @@ -1232,20 +1232,14 @@ def _CalculateArgLengths(opening): shortList = list() deltaList = list() delta = 0 - while True: + while token: shortList.append(token) - if token.name == "RPAR": - argList.append(shortList) - deltaList.append(delta) - break - token = token.next_token if token.name == "COMMA": shortList.append(token) argList.append(shortList) deltaList.append(delta) shortList = list() delta = 0 - token = token.next_token elif token.name == "LPAR": shortList.append(token) if _IsFunctionCallWithArguments(token.previous_token): @@ -1263,12 +1257,12 @@ def _CalculateArgLengths(opening): delta = 0 else: shortList.append(token) - token = token.next_token elif token.name == "RPAR": shortList.append(token) argList.append(shortList) deltaList.append(delta) break + token = token.next_token argLengths = list() for l, d in zip(argList, deltaList): From a782ca5059739fc8a673ececfd7f84af0b0b9fa1 Mon Sep 17 00:00:00 2001 From: Jesse Clemens Date: Thu, 24 Aug 2023 12:23:25 -0400 Subject: [PATCH 04/11] Strip out even more unnecessary logic --- yapf/yapflib/format_decision_state.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py index 0da5e97c9..2eeca20a2 100644 --- a/yapf/yapflib/format_decision_state.py +++ b/yapf/yapflib/format_decision_state.py @@ -1248,15 +1248,7 @@ def _CalculateArgLengths(opening): innerLength = endToken.total_length - token.total_length delta = innerLength - maxArg token = token.matching_bracket - if token.next_token.name == "COMMA": - token = token.next_token - shortList.append(token) - argList.append(shortList) - deltaList.append(delta) - shortList = list() - delta = 0 - else: - shortList.append(token) + shortList.append(token) elif token.name == "RPAR": shortList.append(token) argList.append(shortList) From 447eb3cff9ed6af101e0dac29f80a635a8592df2 Mon Sep 17 00:00:00 2001 From: Jesse Clemens Date: Thu, 24 Aug 2023 12:37:37 -0400 Subject: [PATCH 05/11] Remove a return that might short-circuit additional logic later --- yapf/yapflib/format_decision_state.py | 1 - 1 file changed, 1 deletion(-) diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py index 2eeca20a2..c0b73f683 100644 --- a/yapf/yapflib/format_decision_state.py +++ b/yapf/yapflib/format_decision_state.py @@ -522,7 +522,6 @@ def SurroundedByParens(token): length += startCol if length > self.column_limit: return True - return False if (previous.OpensScope() and not current.OpensScope() and not current.is_comment and From 3b59c64b0226b323cbea971dabb3d0967aff171d Mon Sep 17 00:00:00 2001 From: Jesse Clemens Date: Thu, 24 Aug 2023 12:43:35 -0400 Subject: [PATCH 06/11] Remove an extra space --- yapf/yapflib/format_decision_state.py | 1 - 1 file changed, 1 deletion(-) diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py index c0b73f683..480fc24af 100644 --- a/yapf/yapflib/format_decision_state.py +++ b/yapf/yapflib/format_decision_state.py @@ -515,7 +515,6 @@ def SurroundedByParens(token): (opening.matching_bracket.next_token and opening.matching_bracket.next_token.value != ',' and not opening.matching_bracket.next_token.ClosesScope())): - argLengths = _CalculateArgLengths(opening) startCol = self.column + len(current.value) + len(opening.value) for length in argLengths: From 299fa563ea00e6666a28e67dd2f06c6c094560dd Mon Sep 17 00:00:00 2001 From: Jesse Clemens Date: Thu, 24 Aug 2023 12:59:39 -0400 Subject: [PATCH 07/11] Remove duplicate appends to shortList. --- yapf/yapflib/format_decision_state.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py index 480fc24af..100a67598 100644 --- a/yapf/yapflib/format_decision_state.py +++ b/yapf/yapflib/format_decision_state.py @@ -1233,13 +1233,11 @@ def _CalculateArgLengths(opening): while token: shortList.append(token) if token.name == "COMMA": - shortList.append(token) argList.append(shortList) deltaList.append(delta) shortList = list() delta = 0 elif token.name == "LPAR": - shortList.append(token) if _IsFunctionCallWithArguments(token.previous_token): maxArg = max(_CalculateArgLengths(token)) endToken = token.matching_bracket @@ -1248,7 +1246,6 @@ def _CalculateArgLengths(opening): token = token.matching_bracket shortList.append(token) elif token.name == "RPAR": - shortList.append(token) argList.append(shortList) deltaList.append(delta) break From be89f426f6886f43d707b59c11cc3c4222948f4a Mon Sep 17 00:00:00 2001 From: Jesse Clemens Date: Mon, 28 Aug 2023 11:05:13 -0400 Subject: [PATCH 08/11] Fix variable naming to match project conventions --- yapf/yapflib/format_decision_state.py | 46 +++++++++++++-------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py index 100a67598..9315144f6 100644 --- a/yapf/yapflib/format_decision_state.py +++ b/yapf/yapflib/format_decision_state.py @@ -515,10 +515,10 @@ def SurroundedByParens(token): (opening.matching_bracket.next_token and opening.matching_bracket.next_token.value != ',' and not opening.matching_bracket.next_token.ClosesScope())): - argLengths = _CalculateArgLengths(opening) - startCol = self.column + len(current.value) + len(opening.value) - for length in argLengths: - length += startCol + arg_lengths = _CalculateArgLengths(opening) + start_col = self.column + len(current.value) + len(opening.value) + for length in arg_lengths: + length += start_col if length > self.column_limit: return True @@ -1225,37 +1225,37 @@ def _ScopeHasNoCommas(token): def _CalculateArgLengths(opening): """Calculate the length of each function arg, if the args are wrapped""" - argList = list() + arg_list = list() token = opening.next_token - shortList = list() - deltaList = list() + short_list = list() + delta_list = list() delta = 0 while token: - shortList.append(token) + short_list.append(token) if token.name == "COMMA": - argList.append(shortList) - deltaList.append(delta) - shortList = list() + arg_list.append(short_list) + delta_list.append(delta) + short_list = list() delta = 0 elif token.name == "LPAR": if _IsFunctionCallWithArguments(token.previous_token): - maxArg = max(_CalculateArgLengths(token)) - endToken = token.matching_bracket - innerLength = endToken.total_length - token.total_length - delta = innerLength - maxArg + max_arg = max(_CalculateArgLengths(token)) + end_token = token.matching_bracket + inner_length = end_token.total_length - token.total_length + delta = inner_length - max_arg token = token.matching_bracket - shortList.append(token) + short_list.append(token) elif token.name == "RPAR": - argList.append(shortList) - deltaList.append(delta) + arg_list.append(short_list) + delta_list.append(delta) break token = token.next_token - argLengths = list() - for l, d in zip(argList, deltaList): - argLen = l[-1].total_length - l[0].total_length + len(l[0].value) - d - argLengths.append(argLen) - return argLengths + arg_lengths = list() + for l, d in zip(arg_list, delta_list): + arg_len = l[-1].total_length - l[0].total_length + len(l[0].value) - d + arg_lengths.append(arg_len) + return arg_lengths class _ParenState(object): From e538abba7c9728fe52cd78ee110fe7439366740f Mon Sep 17 00:00:00 2001 From: Jesse Clemens Date: Wed, 30 Aug 2023 18:00:48 -0400 Subject: [PATCH 09/11] Add a knob to control whether yapf tries not to squish arguments --- yapf/yapflib/format_decision_state.py | 3 ++- yapf/yapflib/style.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py index 9315144f6..58e0f146a 100644 --- a/yapf/yapflib/format_decision_state.py +++ b/yapf/yapflib/format_decision_state.py @@ -551,7 +551,8 @@ def SurroundedByParens(token): return True i += 1 - if (self.column_limit - self.column) / float(self.column_limit) < 0.3: + per = style.Get('SPLIT_ARGUMENTS_SQUISH_PERCENTAGE') / 100 + if (self.column_limit - self.column) / float(self.column_limit) < per: # Try not to squish all of the arguments off to the right. return True else: diff --git a/yapf/yapflib/style.py b/yapf/yapflib/style.py index 7642c01f4..969d481b7 100644 --- a/yapf/yapflib/style.py +++ b/yapf/yapflib/style.py @@ -359,6 +359,12 @@ def method(): Split before arguments, but do not split all subexpressions recursively (unless needed). """), + SPLIT_ARGUMENTS_SQUISH_PERCENTAGE=textwrap.dedent("""\ + The percentage (as an integer) of the line remaining where an argument + wrap will be forced if the argument starts after that location. Set this + to zero to prevent yapf from wrapping arguments that do not absolutely + need to be wrapped. + """), SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED=textwrap.dedent("""\ Split before arguments if the argument list is terminated by a comma. @@ -515,6 +521,7 @@ def CreatePEP8Style(): SPACES_BEFORE_COMMENT=2, SPLIT_ALL_COMMA_SEPARATED_VALUES=False, SPLIT_ALL_TOP_LEVEL_COMMA_SEPARATED_VALUES=False, + SPLIT_ARGUMENTS_SQUISH_PERCENTAGE=30, SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED=False, SPLIT_BEFORE_ARITHMETIC_OPERATOR=False, SPLIT_BEFORE_BITWISE_OPERATOR=True, @@ -704,6 +711,7 @@ def _IntOrIntListConverter(s): SPACES_BEFORE_COMMENT=_IntOrIntListConverter, SPLIT_ALL_COMMA_SEPARATED_VALUES=_BoolConverter, SPLIT_ALL_TOP_LEVEL_COMMA_SEPARATED_VALUES=_BoolConverter, + SPLIT_ARGUMENTS_SQUISH_PERCENTAGE=int, SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED=_BoolConverter, SPLIT_BEFORE_ARITHMETIC_OPERATOR=_BoolConverter, SPLIT_BEFORE_BITWISE_OPERATOR=_BoolConverter, From 54672d8005b88f939ea8492768a7c052aa753e8f Mon Sep 17 00:00:00 2001 From: Jesse Clemens Date: Wed, 18 Oct 2023 14:40:23 -0400 Subject: [PATCH 10/11] Allow containers to wrap gracefully And do not force breaks on commas in the middle of for comprehensions --- yapf/yapflib/format_decision_state.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py index 58e0f146a..38a7178bf 100644 --- a/yapf/yapflib/format_decision_state.py +++ b/yapf/yapflib/format_decision_state.py @@ -213,7 +213,9 @@ def MustSplit(self): # Allow the fallthrough code to handle the closing bracket. if current != opening.matching_bracket: # If the container doesn't fit in the current line, must split - return not self._ContainerFitsOnStartLine(opening) + if (subtypes.COMP_FOR not in current.subtypes and + not self._ContainerFitsOnStartLine(opening)): + return True if (self.stack[-1].split_before_closing_bracket and (current.value in '}]' and style.Get('SPLIT_BEFORE_CLOSING_BRACKET') or @@ -557,9 +559,14 @@ def SurroundedByParens(token): return True else: # Split after the opening of a container if it doesn't fit on the - # current line. + # current line, including checking for wrapping. if not self._FitsOnLine(previous, previous.matching_bracket): - return True + arg_lengths = _CalculateArgLengths(previous) + start_col = self.column + len(current.value) + len(previous.value) + for length in arg_lengths: + length += start_col + if length > self.column_limit: + return True ########################################################################### # Original Formatting Splitting From cbf85502a629ea7b034617bd0c01d98ad4fbfd2a Mon Sep 17 00:00:00 2001 From: Jesse Clemens Date: Wed, 18 Oct 2023 15:20:42 -0400 Subject: [PATCH 11/11] Add a fix for wrapping of container literals in function args. --- yapf/yapflib/format_decision_state.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py index 38a7178bf..d17840605 100644 --- a/yapf/yapflib/format_decision_state.py +++ b/yapf/yapflib/format_decision_state.py @@ -199,6 +199,10 @@ def MustSplit(self): # Avoid breaking in a container that fits in the current line if possible opening = _GetOpeningBracket(current) + # If at the start of a container, then we need to double check if it is + # actually nested. + if opening == current: + opening = _GetOpeningBracket(previous) # Can't find opening bracket, behave the same way as # SPLIT_ALL_COMMA_SEPARATED_VALUES. @@ -384,7 +388,6 @@ def SurroundedByParens(token): ########################################################################### # Argument List Splitting - if style.Get('SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED'): # Split before arguments in a function call or definition if the # arguments are terminated by a comma.