pythongh-105194: Fix format specifier escaped characters in f-strings (…

…python#105231)
JelleZijlstra · Jun 2, 2023 · 41de543 · 41de543
1 parent 4bfa01b
commit 41de543
Show file tree

Hide file tree

Showing 6 changed files with 34 additions and 2 deletions.
diff --git a/Grammar/python.gram b/Grammar/python.gram
@@ -923,7 +923,7 @@ fstring_conversion[ResultTokenWithMetadata*]:
 fstring_full_format_spec[ResultTokenWithMetadata*]:
     | colon=':' spec=fstring_format_spec* { _PyPegen_setup_full_format_spec(p, colon, (asdl_expr_seq *) spec, EXTRA) }
 fstring_format_spec[expr_ty]:
-    | t=FSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) }
+    | t=FSTRING_MIDDLE { _PyPegen_decoded_constant_from_token(p, t) }
     | fstring_replacement_field
 fstring[expr_ty]:
     | a=FSTRING_START b=fstring_middle* c=FSTRING_END { _PyPegen_joined_str(p, a, (asdl_expr_seq*)b, c) }

diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
@@ -764,6 +764,16 @@ def test_format_specifier_expressions(self):
                              """f'{"s"!{"r"}}'""",
                              ])
 
+    def test_custom_format_specifier(self):
+        class CustomFormat:
+            def __format__(self, format_spec):
+                return format_spec
+
+        self.assertEqual(f'{CustomFormat():\n}', '\n')
+        self.assertEqual(f'{CustomFormat():\u2603}', '☃')
+        with self.assertWarns(SyntaxWarning):
+            exec('f"{F():¯\_(ツ)_/¯}"', {'F': CustomFormat})
+
     def test_side_effect_order(self):
         class X:
             def __init__(self):

diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-02-11-37-12.gh-issue-105194.4eu56B.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-02-11-37-12.gh-issue-105194.4eu56B.rst
@@ -0,0 +1,2 @@
+Do not escape with backslashes f-string format specifiers. Patch by Pablo
+Galindo
diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c
@@ -1350,6 +1350,25 @@ _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b
                             p->arena);
 }
 
+expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) {
+    Py_ssize_t bsize;
+    char* bstr;
+    if (PyBytes_AsStringAndSize(tok->bytes, &bstr, &bsize) == -1) {
+        return NULL;
+    }
+    PyObject* str = _PyPegen_decode_string(p, 0, bstr, bsize, tok);
+    if (str == NULL) {
+        return NULL;
+    }
+    if (_PyArena_AddPyObject(p->arena, str) < 0) {
+        Py_DECREF(str);
+        return NULL;
+    }
+    return _PyAST_Constant(str, NULL, tok->lineno, tok->col_offset,
+                           tok->end_lineno, tok->end_col_offset,
+                           p->arena);
+}
+
 expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok) {
     char* bstr = PyBytes_AsString(tok->bytes);
     if (bstr == NULL) {

diff --git a/Parser/parser.c b/Parser/parser.c
diff --git a/Parser/pegen.h b/Parser/pegen.h
@@ -328,6 +328,7 @@ expr_ty _PyPegen_collect_call_seqs(Parser *, asdl_expr_seq *, asdl_seq *,
                      int lineno, int col_offset, int end_lineno,
                      int end_col_offset, PyArena *arena);
 expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok);
+expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok);
 expr_ty _PyPegen_constant_from_string(Parser* p, Token* tok);
 expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *, int, int, int, int, PyArena *);
 expr_ty _PyPegen_FetchRawForm(Parser *p, int, int, int, int);