Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2026-06-28 11:50:50 +00:00
Code Issues Projects Releases Packages Wiki Activity

[3.15] gh-150207: Raise MemoryError on tokenizer allocation failure instead of crashing (GH-150275) (#150996)

Browse source 
gh-150207: Raise MemoryError on tokenizer allocation failure instead of crashing (GH-150275)
(cherry picked from commit 262625fa30)

Co-authored-by: Grant Herman <grantlouisherman041@gmail.com>
This commit is contained in:
Miss Islington (bot) 2026-06-06 04:46:55 +02:00 • committed by GitHub
parent 4a3d6f9793
commit 934ae3ed15
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 9 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-21-52-38.gh-issue-150207.l2BUtI.rst Normal file
View file

@ -0,0 +1 @@
Fix a crash when a memory allocation fails during tokenizer initialization. A proper :exc:`MemoryError` is now raised instead.

5
Parser/lexer/state.c
View file

@ -15,8 +15,11 @@ _PyTokenizer_tok_new(void)
struct tok_state *tok = (struct tok_state *)PyMem_Calloc(
1,
sizeof(struct tok_state));
if (tok == NULL)
if (tok == NULL) {
PyErr_NoMemory();
return NULL;
}
tok->buf = tok->cur = tok->inp = NULL;
tok->fp_interactive = 0;
tok->interactive_src_start = NULL;

1
Parser/tokenizer/file_tokenizer.c
View file

@ -378,6 +378,7 @@ _PyTokenizer_FromFile(FILE *fp, const char* enc,
return NULL;
if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) {
_PyTokenizer_Free(tok);
PyErr_NoMemory();
return NULL;
}
tok->cur = tok->inp = tok->buf;

2
Parser/tokenizer/helpers.c
View file

@ -193,6 +193,7 @@ _PyTokenizer_new_string(const char *s, Py_ssize_t len, struct tok_state *tok)
char* result = (char *)PyMem_Malloc(len + 1);
if (!result) {
tok->done = E_NOMEM;
PyErr_NoMemory();
return NULL;
}
memcpy(result, s, len);
@ -221,6 +222,7 @@ _PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf
buf = PyMem_Malloc(needed_length);
if (buf == NULL) {
tok->done = E_NOMEM;
PyErr_NoMemory();
return NULL;
}
for (current = buf; *s; s++, current++) {

1
Parser/tokenizer/readline_tokenizer.c
View file

@ -114,6 +114,7 @@ _PyTokenizer_FromReadline(PyObject* readline, const char* enc,
return NULL;
if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) {
_PyTokenizer_Free(tok);
PyErr_NoMemory();
return NULL;
}
tok->cur = tok->inp = tok->buf;