Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-12-31 04:23:37 +00:00
Code Issues Projects Releases Packages Wiki Activity

[3.14] gh-139516: Fix lambda colon start format spec in f-string in tokenizer (GH-139657) (#139701)

Browse source 
gh-139516: Fix lambda colon start format spec in f-string in tokenizer (GH-139657)
(cherry picked from commit 539461d9ec)

Co-authored-by: Tomasz Pytel <tompytel@gmail.com>
This commit is contained in:
Miss Islington (bot) 2025-10-07 19:35:17 +02:00 • committed by GitHub
parent cec4ddf23e
commit de84d09050
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 28 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Lib/test/test_fstring.py
View file

@ -1859,6 +1859,13 @@ def __format__(self, format):
# Test multiple format specs in same raw f-string
self.assertEqual(rf"{UnchangedFormat():\xFF} {UnchangedFormat():\n}", '\\xFF \\n')
def test_gh139516(self):
with temp_cwd():
script = 'script.py'
with open(script, 'wb') as f:
f.write('''def f(a): pass\nf"{f(a=lambda: 'à'\n)}"'''.encode())
assert_python_ok(script)
if __name__ == '__main__':
unittest.main()

17
Lib/test/test_tokenize.py
View file

@ -1216,6 +1216,23 @@ def test_multiline_non_ascii_fstring_with_expr(self):
FSTRING_END "\'\'\'" (3, 1) (3, 4)
""")
# gh-139516, the '\n' is explicit to ensure no trailing whitespace which would invalidate the test
self.check_tokenize('''f"{f(a=lambda: 'à'\n)}"''', """\
FSTRING_START \'f"\' (1, 0) (1, 2)
OP '{' (1, 2) (1, 3)
NAME 'f' (1, 3) (1, 4)
OP '(' (1, 4) (1, 5)
NAME 'a' (1, 5) (1, 6)
OP '=' (1, 6) (1, 7)
NAME 'lambda' (1, 7) (1, 13)
OP ':' (1, 13) (1, 14)
STRING "\'à\'" (1, 15) (1, 18)
NL '\\n' (1, 18) (1, 19)
OP ')' (2, 0) (2, 1)
OP '}' (2, 1) (2, 2)
FSTRING_END \'"\' (2, 2) (2, 3)
""")
class GenerateTokensTest(TokenizeTest):
def check_tokenize(self, s, expected):
# Format the tokens in s in a table format.

1
Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-13-15-26.gh-issue-139516.d9Pkur.rst Normal file
View file

@ -0,0 +1 @@
Fix lambda colon erroneously start format spec in f-string in tokenizer.

2
Parser/lexer/lexer.c
View file

@ -1376,7 +1376,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c));
}
if( c == '=' && INSIDE_FSTRING_EXPR(current_tok)) {
if( c == '=' && INSIDE_FSTRING_EXPR_AT_TOP(current_tok)) {
current_tok->in_debug = 1;
}

2
Parser/lexer/state.h
View file

@ -9,6 +9,8 @@
#define INSIDE_FSTRING(tok) (tok->tok_mode_stack_index > 0)
#define INSIDE_FSTRING_EXPR(tok) (tok->curly_bracket_expr_start_depth >= 0)
#define INSIDE_FSTRING_EXPR_AT_TOP(tok) \
(tok->curly_bracket_depth - tok->curly_bracket_expr_start_depth == 1)
enum decoding_state {
STATE_INIT,