gh-132661: Disallow Template/str concatenation after PEP 750 spec update (#135996)

Co-authored-by: sobolevn <mail@sobolevn.me>
Co-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com>

Grammar/python.gram

@@ -987,7 +987,10 @@ tstring[expr_ty] (memo):
             _PyPegen_template_str(p, a, (asdl_expr_seq*)b, c)) }
 
 string[expr_ty]: s[Token*]=STRING { _PyPegen_constant_from_string(p, s) }
-strings[expr_ty] (memo): a[asdl_expr_seq*]=(fstring|string|tstring)+ { _PyPegen_concatenate_strings(p, a, EXTRA) }
+strings[expr_ty] (memo):
+    | invalid_string_tstring_concat
+    | a[asdl_expr_seq*]=(fstring|string)+ { _PyPegen_concatenate_strings(p, a, EXTRA) }
+    | a[asdl_expr_seq*]=tstring+ { _PyPegen_concatenate_tstrings(p, a, EXTRA) }
 
 list[expr_ty]:
     | '[' a=[star_named_expressions] ']' { _PyAST_List(a, Load, EXTRA) }
@@ -1553,6 +1556,12 @@ invalid_tstring_conversion_character:
     | '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: missing conversion character") }
     | '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: invalid conversion character") }
 
+invalid_string_tstring_concat:
+    | a=(fstring|string)+ b[expr_ty]=tstring {
+        RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") }
+    | a=tstring+ b[expr_ty]=(fstring|string) {
+        RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") }
+
 invalid_arithmetic:
     | sum ('+'|'-'|'*'|'/'|'%'|'//'|'@') a='not' b=inversion { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") }
 invalid_factor:

Lib/_ast_unparse.py

@@ -626,35 +626,11 @@ def _write_ftstring(self, values, prefix):
             )
         self._ftstring_helper(fstring_parts)
 
-    def _tstring_helper(self, node):
-        if not node.values:
-            self._write_ftstring([], "t")
-            return
-        last_idx = 0
-        for i, value in enumerate(node.values):
-            # This can happen if we have an implicit concat of a t-string
-            # with an f-string
-            if isinstance(value, FormattedValue):
-                if i > last_idx:
-                    # Write t-string until here
-                    self._write_ftstring(node.values[last_idx:i], "t")
-                    self.write(" ")
-                # Write f-string with the current formatted value
-                self._write_ftstring([node.values[i]], "f")
-                if i + 1 < len(node.values):
-                    # Only add a space if there are more values after this
-                    self.write(" ")
-                last_idx = i + 1
-
-        if last_idx < len(node.values):
-            # Write t-string from last_idx to end
-            self._write_ftstring(node.values[last_idx:], "t")
-
     def visit_JoinedStr(self, node):
         self._write_ftstring(node.values, "f")
 
     def visit_TemplateStr(self, node):
-        self._tstring_helper(node)
+        self._write_ftstring(node.values, "t")
 
     def _write_ftstring_inner(self, node, is_format_spec=False):
         if isinstance(node, JoinedStr):

Lib/test/test_ast/test_ast.py

@@ -999,13 +999,6 @@ def test_tstring(self):
         self.assertIsInstance(tree.body[0].value.values[0], ast.Constant)
         self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation)
 
-        # Test AST for implicit concat of t-string with f-string
-        tree = ast.parse('t"Hello {name}" f"{name}"')
-        self.assertIsInstance(tree.body[0].value, ast.TemplateStr)
-        self.assertIsInstance(tree.body[0].value.values[0], ast.Constant)
-        self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation)
-        self.assertIsInstance(tree.body[0].value.values[2], ast.FormattedValue)
-
 
 class CopyTests(unittest.TestCase):
     """Test copying and pickling AST nodes."""

Lib/test/test_tstring.py

@@ -150,7 +150,6 @@ def test_raw_tstrings(self):
         t = tr"{path}\Documents"
         self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")])
 
-
     def test_template_concatenation(self):
         # Test template + template
         t1 = t"Hello, "
@@ -161,9 +160,10 @@ def test_template_concatenation(self):
 
         # Test template + string
         t1 = t"Hello"
-        combined = t1 + ", world"
+        expected_msg = 'can only concatenate string.templatelib.Template ' \
-        self.assertTStringEqual(combined, ("Hello, world",), ())
+            '\\(not "str"\\) to string.templatelib.Template'
-        self.assertEqual(fstring(combined), "Hello, world")
+        with self.assertRaisesRegex(TypeError, expected_msg):
+            t1 + ", world"
 
         # Test template + template with interpolation
         name = "Python"
@@ -174,9 +174,10 @@ def test_template_concatenation(self):
         self.assertEqual(fstring(combined), "Hello, Python")
 
         # Test string + template
-        t = "Hello, " + t"{name}"
+        expected_msg = 'can only concatenate str ' \
-        self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")])
+            '\\(not "string.templatelib.Template"\\) to str'
-        self.assertEqual(fstring(t), "Hello, Python")
+        with self.assertRaisesRegex(TypeError, expected_msg):
+            "Hello, " + t"{name}"
 
     def test_nested_templates(self):
         # Test a template inside another template expression
@@ -241,52 +242,28 @@ def test_literal_concatenation(self):
         self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")])
         self.assertEqual(fstring(t), "Hello, Python")
 
-        # Test concatenation with string literal
+        # Test disallowed mix of t-string and string/f-string (incl. bytes)
-        name = "Python"
-        t = t"Hello, {name}" "and welcome!"
-        self.assertTStringEqual(
-            t, ("Hello, ", "and welcome!"), [(name, "name")]
-        )
-        self.assertEqual(fstring(t), "Hello, Pythonand welcome!")
-
-        # Test concatenation with Unicode literal
-        name = "Python"
-        t = t"Hello, {name}" u"and welcome!"
-        self.assertTStringEqual(
-            t, ("Hello, ", "and welcome!"), [(name, "name")]
-        )
-        self.assertEqual(fstring(t), "Hello, Pythonand welcome!")
-
-        # Test concatenation with f-string literal
-        tab = '\t'
-        t = t"Tab: {tab}. " f"f-tab: {tab}."
-        self.assertTStringEqual(t, ("Tab: ", ". f-tab: \t."), [(tab, "tab")])
-        self.assertEqual(fstring(t), "Tab: \t. f-tab: \t.")
-
-        # Test concatenation with raw string literal
-        tab = '\t'
-        t = t"Tab: {tab}. " r"Raw tab: \t."
-        self.assertTStringEqual(
-            t, ("Tab: ", r". Raw tab: \t."), [(tab, "tab")]
-        )
-        self.assertEqual(fstring(t), "Tab: \t. Raw tab: \\t.")
-
-        # Test concatenation with raw f-string literal
-        tab = '\t'
-        t = t"Tab: {tab}. " rf"f-tab: {tab}. Raw tab: \t."
-        self.assertTStringEqual(
-            t, ("Tab: ", ". f-tab: \t. Raw tab: \\t."), [(tab, "tab")]
-        )
-        self.assertEqual(fstring(t), "Tab: \t. f-tab: \t. Raw tab: \\t.")
-
         what = 't'
-        expected_msg = 'cannot mix bytes and nonbytes literals'
+        expected_msg = 'cannot mix t-string literals with string or bytes literals'
         for case in (
+            "t'{what}-string literal' 'str literal'",
+            "t'{what}-string literal' u'unicode literal'",
+            "t'{what}-string literal' f'f-string literal'",
+            "t'{what}-string literal' r'raw string literal'",
+            "t'{what}-string literal' rf'raw f-string literal'",
             "t'{what}-string literal' b'bytes literal'",
             "t'{what}-string literal' br'raw bytes literal'",
+            "'str literal' t'{what}-string literal'",
+            "u'unicode literal' t'{what}-string literal'",
+            "f'f-string literal' t'{what}-string literal'",
+            "r'raw string literal' t'{what}-string literal'",
+            "rf'raw f-string literal' t'{what}-string literal'",
+            "b'bytes literal' t'{what}-string literal'",
+            "br'raw bytes literal' t'{what}-string literal'",
         ):
-            with self.assertRaisesRegex(SyntaxError, expected_msg):
+            with self.subTest(case):
-                eval(case)
+                with self.assertRaisesRegex(SyntaxError, expected_msg):
+                    eval(case)
 
     def test_triple_quoted(self):
         # Test triple-quoted t-strings

Lib/test/test_unparse.py

@@ -206,10 +206,6 @@ def test_tstrings(self):
         self.check_ast_roundtrip("t'foo'")
         self.check_ast_roundtrip("t'foo {bar}'")
         self.check_ast_roundtrip("t'foo {bar!s:.2f}'")
-        self.check_ast_roundtrip("t'foo {bar}' f'{bar}'")
-        self.check_ast_roundtrip("f'{bar}' t'foo {bar}'")
-        self.check_ast_roundtrip("t'foo {bar}' fr'\\hello {bar}'")
-        self.check_ast_roundtrip("t'foo {bar}' u'bar'")
 
     def test_strings(self):
         self.check_ast_roundtrip("u'foo'")

Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-22-08.gh-issue-132661.34ftJl.rst

@@ -0,0 +1,5 @@
+Reflect recent :pep:`750` change.
+
+Disallow concatenation of ``string.templatelib.Template`` and :class:`str`.
+Also, disallow implicit concatenation of t-string literals with string or
+f-string literals.

Objects/templateobject.c

@@ -30,7 +30,8 @@ templateiter_next(PyObject *op)
             Py_SETREF(item, PyIter_Next(self->interpolationsiter));
             self->from_strings = 1;
         }
-    } else {
+    }
+    else {
         item = PyIter_Next(self->interpolationsiter);
         self->from_strings = 1;
     }
@@ -245,54 +246,6 @@ template_iter(PyObject *op)
     return (PyObject *)iter;
 }
 
-static PyObject *
-template_strings_append_str(PyObject *strings, PyObject *str)
-{
-    Py_ssize_t stringslen = PyTuple_GET_SIZE(strings);
-    PyObject *string = PyTuple_GET_ITEM(strings, stringslen - 1);
-    PyObject *concat = PyUnicode_Concat(string, str);
-    if (concat == NULL) {
-        return NULL;
-    }
-
-    PyObject *newstrings = PyTuple_New(stringslen);
-    if (newstrings == NULL) {
-        Py_DECREF(concat);
-        return NULL;
-    }
-
-    for (Py_ssize_t i = 0; i < stringslen - 1; i++) {
-        PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i)));
-    }
-    PyTuple_SET_ITEM(newstrings, stringslen - 1, concat);
-
-    return newstrings;
-}
-
-static PyObject *
-template_strings_prepend_str(PyObject *strings, PyObject *str)
-{
-    Py_ssize_t stringslen = PyTuple_GET_SIZE(strings);
-    PyObject *string = PyTuple_GET_ITEM(strings, 0);
-    PyObject *concat = PyUnicode_Concat(str, string);
-    if (concat == NULL) {
-        return NULL;
-    }
-
-    PyObject *newstrings = PyTuple_New(stringslen);
-    if (newstrings == NULL) {
-        Py_DECREF(concat);
-        return NULL;
-    }
-
-    PyTuple_SET_ITEM(newstrings, 0, concat);
-    for (Py_ssize_t i = 1; i < stringslen; i++) {
-        PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i)));
-    }
-
-    return newstrings;
-}
-
 static PyObject *
 template_strings_concat(PyObject *left, PyObject *right)
 {
@@ -344,47 +297,17 @@ template_concat_templates(templateobject *self, templateobject *other)
     return newtemplate;
 }
 
-static PyObject *
-template_concat_template_str(templateobject *self, PyObject *other)
-{
-    PyObject *newstrings = template_strings_append_str(self->strings, other);
-    if (newstrings == NULL) {
-        return NULL;
-    }
-
-    PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations);
-    Py_DECREF(newstrings);
-    return newtemplate;
-}
-
-static PyObject *
-template_concat_str_template(templateobject *self, PyObject *other)
-{
-    PyObject *newstrings = template_strings_prepend_str(self->strings, other);
-    if (newstrings == NULL) {
-        return NULL;
-    }
-
-    PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations);
-    Py_DECREF(newstrings);
-    return newtemplate;
-}
-
 PyObject *
 _PyTemplate_Concat(PyObject *self, PyObject *other)
 {
     if (_PyTemplate_CheckExact(self) && _PyTemplate_CheckExact(other)) {
         return template_concat_templates((templateobject *) self, (templateobject *) other);
     }
-    else if ((_PyTemplate_CheckExact(self)) && PyUnicode_Check(other)) {
+
-        return template_concat_template_str((templateobject *) self, other);
+    PyErr_Format(PyExc_TypeError,
-    }
+        "can only concatenate string.templatelib.Template (not \"%T\") to string.templatelib.Template",
-    else if (PyUnicode_Check(self) && (_PyTemplate_CheckExact(other))) {
+        other);
-        return template_concat_str_template((templateobject *) other, self);
+    return NULL;
-    }
-    else {
-        Py_RETURN_NOTIMPLEMENTED;
-    }
 }
 
 static PyObject *

Objects/unicodeobject.c

@@ -56,7 +56,6 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 #include "pycore_pyhash.h"        // _Py_HashSecret_t
 #include "pycore_pylifecycle.h"   // _Py_SetFileSystemEncoding()
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
-#include "pycore_template.h"      // _PyTemplate_Concat()
 #include "pycore_tuple.h"         // _PyTuple_FromArray()
 #include "pycore_ucnhash.h"       // _PyUnicode_Name_CAPI
 #include "pycore_unicodeobject.h" // struct _Py_unicode_state
@@ -11610,16 +11609,10 @@ PyUnicode_Concat(PyObject *left, PyObject *right)
         return NULL;
 
     if (!PyUnicode_Check(right)) {
-        if (_PyTemplate_CheckExact(right)) {
+        PyErr_Format(PyExc_TypeError,
-            // str + tstring is implemented in the tstring type
+            "can only concatenate str (not \"%.200s\") to str",
-            return _PyTemplate_Concat(left, right);
+            Py_TYPE(right)->tp_name);
-        }
+        return NULL;
-        else {
-            PyErr_Format(PyExc_TypeError,
-                "can only concatenate str (not \"%.200s\") to str",
-                Py_TYPE(right)->tp_name);
-            return NULL;
-        }
     }
 
     /* Shortcuts */

Parser/action_helpers.c

@@ -1834,8 +1834,8 @@ _build_concatenated_joined_str(Parser *p, asdl_expr_seq *strings,
     return _PyAST_JoinedStr(values, lineno, col_offset, end_lineno, end_col_offset, p->arena);
 }
 
-static expr_ty
+expr_ty
-_build_concatenated_template_str(Parser *p, asdl_expr_seq *strings,
+_PyPegen_concatenate_tstrings(Parser *p, asdl_expr_seq *strings,
                                int lineno, int col_offset, int end_lineno,
                                int end_col_offset, PyArena *arena)
 {
@@ -1853,7 +1853,6 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings,
     Py_ssize_t len = asdl_seq_LEN(strings);
     assert(len > 0);
 
-    int t_string_found = 0;
     int f_string_found = 0;
     int unicode_string_found = 0;
     int bytes_found = 0;
@@ -1873,7 +1872,8 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings,
                 f_string_found = 1;
                 break;
             case TemplateStr_kind:
-                t_string_found = 1;
+                // python.gram handles this; we should never get here
+                assert(0);
                 break;
             default:
                 f_string_found = 1;
@@ -1882,13 +1882,13 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings,
     }
 
     // Cannot mix unicode and bytes
-    if ((unicode_string_found || f_string_found || t_string_found) && bytes_found) {
+    if ((unicode_string_found || f_string_found) && bytes_found) {
         RAISE_SYNTAX_ERROR("cannot mix bytes and nonbytes literals");
         return NULL;
     }
 
     // If it's only bytes or only unicode string, do a simple concat
-    if (!f_string_found && !t_string_found) {
+    if (!f_string_found) {
         if (len == 1) {
             return asdl_seq_GET(strings, 0);
         }
@@ -1902,11 +1902,6 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings,
         }
     }
 
-    if (t_string_found) {
-        return _build_concatenated_template_str(p, strings, lineno,
-            col_offset, end_lineno, end_col_offset, arena);
-    }
-
     return _build_concatenated_joined_str(p, strings, lineno,
         col_offset, end_lineno, end_col_offset, arena);
 }

Parser/pegen.h

@@ -351,6 +351,7 @@ expr_ty _PyPegen_collect_call_seqs(Parser *, asdl_expr_seq *, asdl_seq *,
 expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok);
 expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok);
 expr_ty _PyPegen_constant_from_string(Parser* p, Token* tok);
+expr_ty _PyPegen_concatenate_tstrings(Parser *p, asdl_expr_seq *, int, int, int, int, PyArena *);
 expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *, int, int, int, int, PyArena *);
 expr_ty _PyPegen_FetchRawForm(Parser *p, int, int, int, int);
 expr_ty _PyPegen_ensure_imaginary(Parser *p, expr_ty);

Python/ast_unparse.c

@@ -663,27 +663,15 @@ build_ftstring_body(asdl_expr_seq *values, bool is_format_spec)
 }
 
 static int
-_write_values_subarray(PyUnicodeWriter *writer, asdl_expr_seq *values, Py_ssize_t first_idx,
+append_templatestr(PyUnicodeWriter *writer, expr_ty e)
-                       Py_ssize_t last_idx, char prefix, PyArena *arena)
 {
     int result = -1;
-
+    PyObject *body = build_ftstring_body(e->v.TemplateStr.values, false);
-    asdl_expr_seq *new_values = _Py_asdl_expr_seq_new(last_idx - first_idx + 1, arena);
-    if (!new_values) {
-        return result;
-    }
-
-    Py_ssize_t j = 0;
-    for (Py_ssize_t i = first_idx; i <= last_idx; ++i) {
-        asdl_seq_SET(new_values, j++, asdl_seq_GET(values, i));
-    }
-
-    PyObject *body = build_ftstring_body(new_values, false);
     if (!body) {
-        return result;
+        return -1;
     }
 
-    if (-1 != append_char(writer, prefix) &&
+    if (-1 != append_charp(writer, "t") &&
         -1 != append_repr(writer, body))
     {
         result = 0;
@@ -692,72 +680,6 @@ _write_values_subarray(PyUnicodeWriter *writer, asdl_expr_seq *values, Py_ssize_
     return result;
 }
 
-static int
-append_templatestr(PyUnicodeWriter *writer, expr_ty e)
-{
-    PyArena *arena = _PyArena_New();
-    if (!arena) {
-        return -1;
-    }
-
-    Py_ssize_t last_idx = 0;
-    Py_ssize_t len = asdl_seq_LEN(e->v.TemplateStr.values);
-    if (len == 0) {
-        int result = _write_values_subarray(writer, e->v.TemplateStr.values,
-                0, len - 1, 't', arena);
-        _PyArena_Free(arena);
-        return result;
-    }
-
-    for (Py_ssize_t i = 0; i < len; i++) {
-        expr_ty value = asdl_seq_GET(e->v.TemplateStr.values, i);
-
-        // Handle implicit concat of t-strings with f-strings
-        if (value->kind == FormattedValue_kind) {
-            if (i > last_idx) {
-                // Create a new TemplateStr with the values between last_idx and i
-                // and append it to the writer.
-                if (_write_values_subarray(writer, e->v.TemplateStr.values,
-                        last_idx, i - 1, 't', arena) == -1) {
-                    goto error;
-                }
-
-                if (append_charp(writer, " ") == -1) {
-                    goto error;
-                }
-            }
-
-            // Append the FormattedValue to the writer.
-            if (_write_values_subarray(writer, e->v.TemplateStr.values,
-                    i, i, 'f', arena) == -1) {
-                goto error;
-            }
-
-            if (i + 1 < len) {
-                if (append_charp(writer, " ") == -1) {
-                    goto error;
-                }
-            }
-
-            last_idx = i + 1;
-        }
-    }
-
-    if (last_idx < len) {
-        if (_write_values_subarray(writer, e->v.TemplateStr.values,
-                last_idx, len - 1, 't', arena) == -1) {
-            goto error;
-        }
-    }
-    _PyArena_Free(arena);
-
-    return 0;
-
-error:
-    _PyArena_Free(arena);
-    return -1;
-}
-
 static int
 append_joinedstr(PyUnicodeWriter *writer, expr_ty e, bool is_format_spec)
 {

Python/codegen.c

@@ -4081,16 +4081,6 @@ codegen_template_str(compiler *c, expr_ty e)
         }
         else {
             VISIT(c, expr, value);
-            Py_ssize_t j;
-            for (j = i + 1; j < value_count; j++) {
-                value = asdl_seq_GET(e->v.TemplateStr.values, j);
-                if (value->kind == Interpolation_kind) {
-                    break;
-                }
-                VISIT(c, expr, value);
-                ADDOP_INPLACE(c, loc, Add);
-            }
-            i = j - 1;
             stringslen++;
             last_was_interpolation = 0;
         }