Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2026-01-06 15:32:22 +00:00
Code Issues Projects Releases Packages Wiki Activity

bpo-46521: Fix codeop to use a new partial-input mode of the parser (GH-31010)

Browse source
This commit is contained in:
Pablo Galindo Salgado 2022-02-08 11:54:37 +00:00 • committed by GitHub
parent 25db2b361b
commit 69e10976b2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 49 additions and 55 deletions
Download patch file Download diff file Expand all files Collapse all files

15
Parser/pegen.c
View file

@ -726,6 +726,9 @@ compute_parser_flags(PyCompilerFlags *flags)
if ((flags->cf_flags & PyCF_ONLY_AST) && flags->cf_feature_version < 7) {
parser_flags |= PyPARSE_ASYNC_HACKS;
}
if (flags->cf_flags & PyCF_ALLOW_INCOMPLETE_INPUT) {
parser_flags |= PyPARSE_ALLOW_INCOMPLETE_INPUT;
}
return parser_flags;
}
@ -811,16 +814,26 @@ reset_parser_state_for_error_pass(Parser *p)
p->tok->interactive_underflow = IUNDERFLOW_STOP;
}
static inline int
_is_end_of_source(Parser *p) {
int err = p->tok->done;
return err == E_EOF || err == E_EOFS || err == E_EOLS;
}
void *
_PyPegen_run_parser(Parser *p)
{
void *res = _PyPegen_parse(p);
assert(p->level == 0);
if (res == NULL) {
if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) && _is_end_of_source(p)) {
PyErr_Clear();
return RAISE_SYNTAX_ERROR("incomplete input");
}
if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_SyntaxError)) {
return NULL;
}
// Make a second parser pass. In this pass we activate heavier and slower checks
// Make a second parser pass. In this pass we activate heavier and slower checks
// to produce better error messages and more complete diagnostics. Extra "invalid_*"
// rules will be active during parsing.
Token *last_token = p->tokens[p->fill - 1];

1
Parser/pegen.h
View file

@ -22,6 +22,7 @@
#define PyPARSE_BARRY_AS_BDFL 0x0020
#define PyPARSE_TYPE_COMMENTS 0x0040
#define PyPARSE_ASYNC_HACKS 0x0080
#define PyPARSE_ALLOW_INCOMPLETE_INPUT 0x0100
#define CURRENT_POS (-5)

26
Parser/tokenizer.c
View file

@ -40,7 +40,7 @@
static struct tok_state *tok_new(void);
static int tok_nextc(struct tok_state *tok);
static void tok_backup(struct tok_state *tok, int c);
static int syntaxerror(struct tok_state *tok, const char *format, ...);
/* Spaces in this constant are treated as "zero or more spaces or tabs" when
tokenizing. */
@ -1031,8 +1031,9 @@ tok_nextc(struct tok_state *tok)
if (tok->cur != tok->inp) {
return Py_CHARMASK(*tok->cur++); /* Fast path */
}
if (tok->done != E_OK)
return EOF;
if (tok->done != E_OK) {
return EOF;
}
if (tok->fp == NULL) {
rc = tok_underflow_string(tok);
}
@ -1964,16 +1965,21 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
tok->line_start = tok->multi_line_start;
int start = tok->lineno;
tok->lineno = tok->first_lineno;
if (quote_size == 3) {
return syntaxerror(tok,
"unterminated triple-quoted string literal"
" (detected at line %d)", start);
syntaxerror(tok, "unterminated triple-quoted string literal"
" (detected at line %d)", start);
if (c != '\n') {
tok->done = E_EOFS;
}
return ERRORTOKEN;
}
else {
return syntaxerror(tok,
"unterminated string literal (detected at"
" line %d)", start);
syntaxerror(tok, "unterminated string literal (detected at"
" line %d)", start);
if (c != '\n') {
tok->done = E_EOLS;
}
return ERRORTOKEN;
}
}
if (c == quote) {