Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-19 16:03:42 +00:00
Code Issues Projects Releases Packages Wiki Activity

gh-132661: Disallow Template/str concatenation after PEP 750 spec update (#135996)

Browse source 
Co-authored-by: sobolevn <mail@sobolevn.me>
Co-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com>
This commit is contained in:
Dave Peck 2025-07-20 23:44:26 -07:00 • committed by GitHub
parent 246be21de1
commit c5e77af131
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 2097 additions and 2133 deletions
Download patch file Download diff file Expand all files Collapse all files

11
Grammar/python.gram
View file

@ -987,7 +987,10 @@ tstring[expr_ty] (memo):
_PyPegen_template_str(p, a, (asdl_expr_seq*)b, c)) }
string[expr_ty]: s[Token*]=STRING { _PyPegen_constant_from_string(p, s) }
strings[expr_ty] (memo): a[asdl_expr_seq*]=(fstring|string|tstring)+ { _PyPegen_concatenate_strings(p, a, EXTRA) }
strings[expr_ty] (memo):
| invalid_string_tstring_concat
| a[asdl_expr_seq*]=(fstring|string)+ { _PyPegen_concatenate_strings(p, a, EXTRA) }
| a[asdl_expr_seq*]=tstring+ { _PyPegen_concatenate_tstrings(p, a, EXTRA) }
list[expr_ty]:
| '[' a=[star_named_expressions] ']' { _PyAST_List(a, Load, EXTRA) }
@ -1553,6 +1556,12 @@ invalid_tstring_conversion_character:
| '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: missing conversion character") }
| '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: invalid conversion character") }
invalid_string_tstring_concat:
| a=(fstring|string)+ b[expr_ty]=tstring {
RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") }
| a=tstring+ b[expr_ty]=(fstring|string) {
RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") }
invalid_arithmetic:
| sum ('+'|'-'|'*'|'/'|'%'|'//'|'@') a='not' b=inversion { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") }
invalid_factor:

26
Lib/_ast_unparse.py
View file

@ -626,35 +626,11 @@ def _write_ftstring(self, values, prefix):
)
self._ftstring_helper(fstring_parts)
def _tstring_helper(self, node):
if not node.values:
self._write_ftstring([], "t")
return
last_idx = 0
for i, value in enumerate(node.values):
# This can happen if we have an implicit concat of a t-string
# with an f-string
if isinstance(value, FormattedValue):
if i > last_idx:
# Write t-string until here
self._write_ftstring(node.values[last_idx:i], "t")
self.write(" ")
# Write f-string with the current formatted value
self._write_ftstring([node.values[i]], "f")
if i + 1 < len(node.values):
# Only add a space if there are more values after this
self.write(" ")
last_idx = i + 1
if last_idx < len(node.values):
# Write t-string from last_idx to end
self._write_ftstring(node.values[last_idx:], "t")
def visit_JoinedStr(self, node):
self._write_ftstring(node.values, "f")
def visit_TemplateStr(self, node):
self._tstring_helper(node)
self._write_ftstring(node.values, "t")
def _write_ftstring_inner(self, node, is_format_spec=False):
if isinstance(node, JoinedStr):

7
Lib/test/test_ast/test_ast.py
View file

@ -999,13 +999,6 @@ def test_tstring(self):
self.assertIsInstance(tree.body[0].value.values[0], ast.Constant)
self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation)
# Test AST for implicit concat of t-string with f-string
tree = ast.parse('t"Hello {name}" f"{name}"')
self.assertIsInstance(tree.body[0].value, ast.TemplateStr)
self.assertIsInstance(tree.body[0].value.values[0], ast.Constant)
self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation)
self.assertIsInstance(tree.body[0].value.values[2], ast.FormattedValue)
class CopyTests(unittest.TestCase):
"""Test copying and pickling AST nodes."""

73
Lib/test/test_tstring.py
View file

@ -150,7 +150,6 @@ def test_raw_tstrings(self):
t = tr"{path}\Documents"
self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")])
def test_template_concatenation(self):
# Test template + template
t1 = t"Hello, "
@ -161,9 +160,10 @@ def test_template_concatenation(self):
# Test template + string
t1 = t"Hello"
combined = t1 + ", world"
self.assertTStringEqual(combined, ("Hello, world",), ())
self.assertEqual(fstring(combined), "Hello, world")
expected_msg = 'can only concatenate string.templatelib.Template ' \
'\\(not "str"\\) to string.templatelib.Template'
with self.assertRaisesRegex(TypeError, expected_msg):
t1 + ", world"
# Test template + template with interpolation
name = "Python"
@ -174,9 +174,10 @@ def test_template_concatenation(self):
self.assertEqual(fstring(combined), "Hello, Python")
# Test string + template
t = "Hello, " + t"{name}"
self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")])
self.assertEqual(fstring(t), "Hello, Python")
expected_msg = 'can only concatenate str ' \
'\\(not "string.templatelib.Template"\\) to str'
with self.assertRaisesRegex(TypeError, expected_msg):
"Hello, " + t"{name}"
def test_nested_templates(self):
# Test a template inside another template expression
@ -241,52 +242,28 @@ def test_literal_concatenation(self):
self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")])
self.assertEqual(fstring(t), "Hello, Python")
# Test concatenation with string literal
name = "Python"
t = t"Hello, {name}" "and welcome!"
self.assertTStringEqual(
t, ("Hello, ", "and welcome!"), [(name, "name")]
)
self.assertEqual(fstring(t), "Hello, Pythonand welcome!")
# Test concatenation with Unicode literal
name = "Python"
t = t"Hello, {name}" u"and welcome!"
self.assertTStringEqual(
t, ("Hello, ", "and welcome!"), [(name, "name")]
)
self.assertEqual(fstring(t), "Hello, Pythonand welcome!")
# Test concatenation with f-string literal
tab = '\t'
t = t"Tab: {tab}. " f"f-tab: {tab}."
self.assertTStringEqual(t, ("Tab: ", ". f-tab: \t."), [(tab, "tab")])
self.assertEqual(fstring(t), "Tab: \t. f-tab: \t.")
# Test concatenation with raw string literal
tab = '\t'
t = t"Tab: {tab}. " r"Raw tab: \t."
self.assertTStringEqual(
t, ("Tab: ", r". Raw tab: \t."), [(tab, "tab")]
)
self.assertEqual(fstring(t), "Tab: \t. Raw tab: \\t.")
# Test concatenation with raw f-string literal
tab = '\t'
t = t"Tab: {tab}. " rf"f-tab: {tab}. Raw tab: \t."
self.assertTStringEqual(
t, ("Tab: ", ". f-tab: \t. Raw tab: \\t."), [(tab, "tab")]
)
self.assertEqual(fstring(t), "Tab: \t. f-tab: \t. Raw tab: \\t.")
# Test disallowed mix of t-string and string/f-string (incl. bytes)
what = 't'
expected_msg = 'cannot mix bytes and nonbytes literals'
expected_msg = 'cannot mix t-string literals with string or bytes literals'
for case in (
"t'{what}-string literal' 'str literal'",
"t'{what}-string literal' u'unicode literal'",
"t'{what}-string literal' f'f-string literal'",
"t'{what}-string literal' r'raw string literal'",
"t'{what}-string literal' rf'raw f-string literal'",
"t'{what}-string literal' b'bytes literal'",
"t'{what}-string literal' br'raw bytes literal'",
"'str literal' t'{what}-string literal'",
"u'unicode literal' t'{what}-string literal'",
"f'f-string literal' t'{what}-string literal'",
"r'raw string literal' t'{what}-string literal'",
"rf'raw f-string literal' t'{what}-string literal'",
"b'bytes literal' t'{what}-string literal'",
"br'raw bytes literal' t'{what}-string literal'",
):
with self.assertRaisesRegex(SyntaxError, expected_msg):
eval(case)
with self.subTest(case):
with self.assertRaisesRegex(SyntaxError, expected_msg):
eval(case)
def test_triple_quoted(self):
# Test triple-quoted t-strings

4
Lib/test/test_unparse.py
View file

@ -206,10 +206,6 @@ def test_tstrings(self):
self.check_ast_roundtrip("t'foo'")
self.check_ast_roundtrip("t'foo {bar}'")
self.check_ast_roundtrip("t'foo {bar!s:.2f}'")
self.check_ast_roundtrip("t'foo {bar}' f'{bar}'")
self.check_ast_roundtrip("f'{bar}' t'foo {bar}'")
self.check_ast_roundtrip("t'foo {bar}' fr'\\hello {bar}'")
self.check_ast_roundtrip("t'foo {bar}' u'bar'")
def test_strings(self):
self.check_ast_roundtrip("u'foo'")

5
Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-22-08.gh-issue-132661.34ftJl.rst Normal file
View file

@ -0,0 +1,5 @@
Reflect recent :pep:`750` change.
Disallow concatenation of ``string.templatelib.Template`` and :class:`str`.
Also, disallow implicit concatenation of t-string literals with string or
f-string literals.

91
Objects/templateobject.c
View file

@ -30,7 +30,8 @@ templateiter_next(PyObject *op)
Py_SETREF(item, PyIter_Next(self->interpolationsiter));
self->from_strings = 1;
}
} else {
}
else {
item = PyIter_Next(self->interpolationsiter);
self->from_strings = 1;
}
@ -245,54 +246,6 @@ template_iter(PyObject *op)
return (PyObject *)iter;
}
static PyObject *
template_strings_append_str(PyObject *strings, PyObject *str)
{
Py_ssize_t stringslen = PyTuple_GET_SIZE(strings);
PyObject *string = PyTuple_GET_ITEM(strings, stringslen - 1);
PyObject *concat = PyUnicode_Concat(string, str);
if (concat == NULL) {
return NULL;
}
PyObject *newstrings = PyTuple_New(stringslen);
if (newstrings == NULL) {
Py_DECREF(concat);
return NULL;
}
for (Py_ssize_t i = 0; i < stringslen - 1; i++) {
PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i)));
}
PyTuple_SET_ITEM(newstrings, stringslen - 1, concat);
return newstrings;
}
static PyObject *
template_strings_prepend_str(PyObject *strings, PyObject *str)
{
Py_ssize_t stringslen = PyTuple_GET_SIZE(strings);
PyObject *string = PyTuple_GET_ITEM(strings, 0);
PyObject *concat = PyUnicode_Concat(str, string);
if (concat == NULL) {
return NULL;
}
PyObject *newstrings = PyTuple_New(stringslen);
if (newstrings == NULL) {
Py_DECREF(concat);
return NULL;
}
PyTuple_SET_ITEM(newstrings, 0, concat);
for (Py_ssize_t i = 1; i < stringslen; i++) {
PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i)));
}
return newstrings;
}
static PyObject *
template_strings_concat(PyObject *left, PyObject *right)
{
@ -344,47 +297,17 @@ template_concat_templates(templateobject *self, templateobject *other)
return newtemplate;
}
static PyObject *
template_concat_template_str(templateobject *self, PyObject *other)
{
PyObject *newstrings = template_strings_append_str(self->strings, other);
if (newstrings == NULL) {
return NULL;
}
PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations);
Py_DECREF(newstrings);
return newtemplate;
}
static PyObject *
template_concat_str_template(templateobject *self, PyObject *other)
{
PyObject *newstrings = template_strings_prepend_str(self->strings, other);
if (newstrings == NULL) {
return NULL;
}
PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations);
Py_DECREF(newstrings);
return newtemplate;
}
PyObject *
_PyTemplate_Concat(PyObject *self, PyObject *other)
{
if (_PyTemplate_CheckExact(self) && _PyTemplate_CheckExact(other)) {
return template_concat_templates((templateobject *) self, (templateobject *) other);
}
else if ((_PyTemplate_CheckExact(self)) && PyUnicode_Check(other)) {
return template_concat_template_str((templateobject *) self, other);
}
else if (PyUnicode_Check(self) && (_PyTemplate_CheckExact(other))) {
return template_concat_str_template((templateobject *) other, self);
}
else {
Py_RETURN_NOTIMPLEMENTED;
}
PyErr_Format(PyExc_TypeError,
"can only concatenate string.templatelib.Template (not \"%T\") to string.templatelib.Template",
other);
return NULL;
}
static PyObject *

15
Objects/unicodeobject.c
View file

@ -56,7 +56,6 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#include "pycore_pyhash.h" // _Py_HashSecret_t
#include "pycore_pylifecycle.h" // _Py_SetFileSystemEncoding()
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "pycore_template.h" // _PyTemplate_Concat()
#include "pycore_tuple.h" // _PyTuple_FromArray()
#include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI
#include "pycore_unicodeobject.h" // struct _Py_unicode_state
@ -11610,16 +11609,10 @@ PyUnicode_Concat(PyObject *left, PyObject *right)
return NULL;
if (!PyUnicode_Check(right)) {
if (_PyTemplate_CheckExact(right)) {
// str + tstring is implemented in the tstring type
return _PyTemplate_Concat(left, right);
}
else {
PyErr_Format(PyExc_TypeError,
"can only concatenate str (not \"%.200s\") to str",
Py_TYPE(right)->tp_name);
return NULL;
}
PyErr_Format(PyExc_TypeError,
"can only concatenate str (not \"%.200s\") to str",
Py_TYPE(right)->tp_name);
return NULL;
}
/* Shortcuts */

17
Parser/action_helpers.c
View file

@ -1834,8 +1834,8 @@ _build_concatenated_joined_str(Parser *p, asdl_expr_seq *strings,
return _PyAST_JoinedStr(values, lineno, col_offset, end_lineno, end_col_offset, p->arena);
}
static expr_ty
_build_concatenated_template_str(Parser *p, asdl_expr_seq *strings,
expr_ty
_PyPegen_concatenate_tstrings(Parser *p, asdl_expr_seq *strings,
int lineno, int col_offset, int end_lineno,
int end_col_offset, PyArena *arena)
{
@ -1853,7 +1853,6 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings,
Py_ssize_t len = asdl_seq_LEN(strings);
assert(len > 0);
int t_string_found = 0;
int f_string_found = 0;
int unicode_string_found = 0;
int bytes_found = 0;
@ -1873,7 +1872,8 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings,
f_string_found = 1;
break;
case TemplateStr_kind:
t_string_found = 1;
// python.gram handles this; we should never get here
assert(0);
break;
default:
f_string_found = 1;
@ -1882,13 +1882,13 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings,
}
// Cannot mix unicode and bytes
if ((unicode_string_found || f_string_found || t_string_found) && bytes_found) {
if ((unicode_string_found || f_string_found) && bytes_found) {
RAISE_SYNTAX_ERROR("cannot mix bytes and nonbytes literals");
return NULL;
}
// If it's only bytes or only unicode string, do a simple concat
if (!f_string_found && !t_string_found) {
if (!f_string_found) {
if (len == 1) {
return asdl_seq_GET(strings, 0);
}
@ -1902,11 +1902,6 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings,
}
}
if (t_string_found) {
return _build_concatenated_template_str(p, strings, lineno,
col_offset, end_lineno, end_col_offset, arena);
}
return _build_concatenated_joined_str(p, strings, lineno,
col_offset, end_lineno, end_col_offset, arena);
}

3884
Parser/parser.c generated
View file

File diff suppressed because it is too large Load diff

1
Parser/pegen.h
View file

@ -351,6 +351,7 @@ expr_ty _PyPegen_collect_call_seqs(Parser *, asdl_expr_seq *, asdl_seq *,
expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok);
expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok);
expr_ty _PyPegen_constant_from_string(Parser* p, Token* tok);
expr_ty _PyPegen_concatenate_tstrings(Parser *p, asdl_expr_seq *, int, int, int, int, PyArena *);
expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *, int, int, int, int, PyArena *);
expr_ty _PyPegen_FetchRawForm(Parser *p, int, int, int, int);
expr_ty _PyPegen_ensure_imaginary(Parser *p, expr_ty);

86
Python/ast_unparse.c
View file

@ -663,27 +663,15 @@ build_ftstring_body(asdl_expr_seq *values, bool is_format_spec)
}
static int
_write_values_subarray(PyUnicodeWriter *writer, asdl_expr_seq *values, Py_ssize_t first_idx,
Py_ssize_t last_idx, char prefix, PyArena *arena)
append_templatestr(PyUnicodeWriter *writer, expr_ty e)
{
int result = -1;
asdl_expr_seq *new_values = _Py_asdl_expr_seq_new(last_idx - first_idx + 1, arena);
if (!new_values) {
return result;
}
Py_ssize_t j = 0;
for (Py_ssize_t i = first_idx; i <= last_idx; ++i) {
asdl_seq_SET(new_values, j++, asdl_seq_GET(values, i));
}
PyObject *body = build_ftstring_body(new_values, false);
PyObject *body = build_ftstring_body(e->v.TemplateStr.values, false);
if (!body) {
return result;
return -1;
}
if (-1 != append_char(writer, prefix) &&
if (-1 != append_charp(writer, "t") &&
-1 != append_repr(writer, body))
{
result = 0;
@ -692,72 +680,6 @@ _write_values_subarray(PyUnicodeWriter *writer, asdl_expr_seq *values, Py_ssize_
return result;
}
static int
append_templatestr(PyUnicodeWriter *writer, expr_ty e)
{
PyArena *arena = _PyArena_New();
if (!arena) {
return -1;
}
Py_ssize_t last_idx = 0;
Py_ssize_t len = asdl_seq_LEN(e->v.TemplateStr.values);
if (len == 0) {
int result = _write_values_subarray(writer, e->v.TemplateStr.values,
0, len - 1, 't', arena);
_PyArena_Free(arena);
return result;
}
for (Py_ssize_t i = 0; i < len; i++) {
expr_ty value = asdl_seq_GET(e->v.TemplateStr.values, i);
// Handle implicit concat of t-strings with f-strings
if (value->kind == FormattedValue_kind) {
if (i > last_idx) {
// Create a new TemplateStr with the values between last_idx and i
// and append it to the writer.
if (_write_values_subarray(writer, e->v.TemplateStr.values,
last_idx, i - 1, 't', arena) == -1) {
goto error;
}
if (append_charp(writer, " ") == -1) {
goto error;
}
}
// Append the FormattedValue to the writer.
if (_write_values_subarray(writer, e->v.TemplateStr.values,
i, i, 'f', arena) == -1) {
goto error;
}
if (i + 1 < len) {
if (append_charp(writer, " ") == -1) {
goto error;
}
}
last_idx = i + 1;
}
}
if (last_idx < len) {
if (_write_values_subarray(writer, e->v.TemplateStr.values,
last_idx, len - 1, 't', arena) == -1) {
goto error;
}
}
_PyArena_Free(arena);
return 0;
error:
_PyArena_Free(arena);
return -1;
}
static int
append_joinedstr(PyUnicodeWriter *writer, expr_ty e, bool is_format_spec)
{

10
Python/codegen.c
View file

@ -4081,16 +4081,6 @@ codegen_template_str(compiler *c, expr_ty e)
}
else {
VISIT(c, expr, value);
Py_ssize_t j;
for (j = i + 1; j < value_count; j++) {
value = asdl_seq_GET(e->v.TemplateStr.values, j);
if (value->kind == Interpolation_kind) {
break;
}
VISIT(c, expr, value);
ADDOP_INPLACE(c, loc, Add);
}
i = j - 1;
stringslen++;
last_was_interpolation = 0;
}