gh-144759: Fix undefined behavior from NULL pointer arithmetic in lexer (#144788)

Guard against NULL pointer arithmetic in `_PyLexer_remember_fstring_buffers`
and `_PyLexer_restore_fstring_buffers`. When `start` or `multi_line_start`
are NULL (uninitialized in tok_mode_stack[0]), performing `NULL - tok->buf`
is undefined behavior. Add explicit NULL checks to store -1 as sentinel
and restore NULL accordingly.

Add test_lexer_buffer_realloc_with_null_start to test_repl.py that
exercises the code path where the lexer buffer is reallocated while
tok_mode_stack[0] has NULL start/multi_line_start pointers. This
triggers _PyLexer_remember_fstring_buffers and verifies the NULL
checks prevent undefined behavior.
This commit is contained in:
Ramin Farajpour Cami 2026-02-15 18:09:57 +03:30 committed by GitHub
parent 645f5c4a73
commit e6110efd03
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 24 additions and 4 deletions

View file

@ -143,6 +143,22 @@ def test_multiline_string_parsing(self):
output = kill_python(p)
self.assertEqual(p.returncode, 0)
@cpython_only
def test_lexer_buffer_realloc_with_null_start(self):
# gh-144759: NULL pointer arithmetic in the lexer when start and
# multi_line_start are NULL (uninitialized in tok_mode_stack[0])
# and the lexer buffer is reallocated while parsing long input.
long_value = "a" * 2000
user_input = dedent(f"""\
x = f'{{{long_value!r}}}'
print(x)
""")
p = spawn_repl()
p.stdin.write(user_input)
output = kill_python(p)
self.assertEqual(p.returncode, 0)
self.assertIn(long_value, output)
def test_close_stdin(self):
user_input = dedent('''
import os

View file

@ -0,0 +1,4 @@
Fix undefined behavior in the lexer when ``start`` and ``multi_line_start``
pointers are ``NULL`` in ``_PyLexer_remember_fstring_buffers()`` and
``_PyLexer_restore_fstring_buffers()``. The ``NULL`` pointer arithmetic
(``NULL - valid_pointer``) is now guarded with explicit ``NULL`` checks.

View file

@ -13,8 +13,8 @@ _PyLexer_remember_fstring_buffers(struct tok_state *tok)
for (index = tok->tok_mode_stack_index; index >= 0; --index) {
mode = &(tok->tok_mode_stack[index]);
mode->start_offset = mode->start - tok->buf;
mode->multi_line_start_offset = mode->multi_line_start - tok->buf;
mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf;
mode->multi_line_start_offset = mode->multi_line_start == NULL ? -1 : mode->multi_line_start - tok->buf;
}
}
@ -27,8 +27,8 @@ _PyLexer_restore_fstring_buffers(struct tok_state *tok)
for (index = tok->tok_mode_stack_index; index >= 0; --index) {
mode = &(tok->tok_mode_stack[index]);
mode->start = tok->buf + mode->start_offset;
mode->multi_line_start = tok->buf + mode->multi_line_start_offset;
mode->start = mode->start_offset < 0 ? NULL : tok->buf + mode->start_offset;
mode->multi_line_start = mode->multi_line_start_offset < 0 ? NULL : tok->buf + mode->multi_line_start_offset;
}
}