Issue #24965: Implement PEP 498 "Literal String Interpolation". Documentation is still needed, I'll open an issue for that.

Include/Python-ast.h

@@ -201,9 +201,10 @@ enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
                   SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11,
                   Await_kind=12, Yield_kind=13, YieldFrom_kind=14,
                   Compare_kind=15, Call_kind=16, Num_kind=17, Str_kind=18,
-                  Bytes_kind=19, NameConstant_kind=20, Ellipsis_kind=21,
+                  FormattedValue_kind=19, JoinedStr_kind=20, Bytes_kind=21,
-                  Attribute_kind=22, Subscript_kind=23, Starred_kind=24,
+                  NameConstant_kind=22, Ellipsis_kind=23, Attribute_kind=24,
-                  Name_kind=25, List_kind=26, Tuple_kind=27};
+                  Subscript_kind=25, Starred_kind=26, Name_kind=27,
+                  List_kind=28, Tuple_kind=29};
 struct _expr {
     enum _expr_kind kind;
     union {
@@ -296,6 +297,16 @@ struct _expr {
             string s;
         } Str;
         
+        struct {
+            expr_ty value;
+            int conversion;
+            expr_ty format_spec;
+        } FormattedValue;
+        
+        struct {
+            asdl_seq *values;
+        } JoinedStr;
+        
         struct {
             bytes s;
         } Bytes;
@@ -543,6 +554,12 @@ expr_ty _Py_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int
 expr_ty _Py_Num(object n, int lineno, int col_offset, PyArena *arena);
 #define Str(a0, a1, a2, a3) _Py_Str(a0, a1, a2, a3)
 expr_ty _Py_Str(string s, int lineno, int col_offset, PyArena *arena);
+#define FormattedValue(a0, a1, a2, a3, a4, a5) _Py_FormattedValue(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_FormattedValue(expr_ty value, int conversion, expr_ty format_spec,
+                           int lineno, int col_offset, PyArena *arena);
+#define JoinedStr(a0, a1, a2, a3) _Py_JoinedStr(a0, a1, a2, a3)
+expr_ty _Py_JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena
+                      *arena);
 #define Bytes(a0, a1, a2, a3) _Py_Bytes(a0, a1, a2, a3)
 expr_ty _Py_Bytes(bytes s, int lineno, int col_offset, PyArena *arena);
 #define NameConstant(a0, a1, a2, a3) _Py_NameConstant(a0, a1, a2, a3)

Lib/test/test_fstring.py

@@ -0,0 +1,715 @@
+import ast
+import types
+import decimal
+import unittest
+
+a_global = 'global variable'
+
+# You could argue that I'm too strict in looking for specific error
+#  values with assertRaisesRegex, but without it it's way too easy to
+#  make a syntax error in the test strings. Especially with all of the
+#  triple quotes, raw strings, backslashes, etc. I think it's a
+#  worthwhile tradeoff. When I switched to this method, I found many
+#  examples where I wasn't testing what I thought I was.
+
+class TestCase(unittest.TestCase):
+    def assertAllRaise(self, exception_type, regex, error_strings):
+        for str in error_strings:
+            with self.subTest(str=str):
+                with self.assertRaisesRegex(exception_type, regex):
+                    eval(str)
+
+    def test__format__lookup(self):
+        # Make sure __format__ is looked up on the type, not the instance.
+        class X:
+            def __format__(self, spec):
+                return 'class'
+
+        x = X()
+
+        # Add a bound __format__ method to the 'y' instance, but not
+        #  the 'x' instance.
+        y = X()
+        y.__format__ = types.MethodType(lambda self, spec: 'instance', y)
+
+        self.assertEqual(f'{y}', format(y))
+        self.assertEqual(f'{y}', 'class')
+        self.assertEqual(format(x), format(y))
+
+        # __format__ is not called this way, but still make sure it
+        #  returns what we expect (so we can make sure we're bypassing
+        #  it).
+        self.assertEqual(x.__format__(''), 'class')
+        self.assertEqual(y.__format__(''), 'instance')
+
+        # This is how __format__ is actually called.
+        self.assertEqual(type(x).__format__(x, ''), 'class')
+        self.assertEqual(type(y).__format__(y, ''), 'class')
+
+    def test_ast(self):
+        # Inspired by http://bugs.python.org/issue24975
+        class X:
+            def __init__(self):
+                self.called = False
+            def __call__(self):
+                self.called = True
+                return 4
+        x = X()
+        expr = """
+a = 10
+f'{a * x()}'"""
+        t = ast.parse(expr)
+        c = compile(t, '', 'exec')
+
+        # Make sure x was not called.
+        self.assertFalse(x.called)
+
+        # Actually run the code.
+        exec(c)
+
+        # Make sure x was called.
+        self.assertTrue(x.called)
+
+    def test_literal_eval(self):
+        # With no expressions, an f-string is okay.
+        self.assertEqual(ast.literal_eval("f'x'"), 'x')
+        self.assertEqual(ast.literal_eval("f'x' 'y'"), 'xy')
+
+        # But this should raise an error.
+        with self.assertRaisesRegex(ValueError, 'malformed node or string'):
+            ast.literal_eval("f'x{3}'")
+
+        # As should this, which uses a different ast node
+        with self.assertRaisesRegex(ValueError, 'malformed node or string'):
+            ast.literal_eval("f'{3}'")
+
+    def test_ast_compile_time_concat(self):
+        x = ['']
+
+        expr = """x[0] = 'foo' f'{3}'"""
+        t = ast.parse(expr)
+        c = compile(t, '', 'exec')
+        exec(c)
+        self.assertEqual(x[0], 'foo3')
+
+    def test_literal(self):
+        self.assertEqual(f'', '')
+        self.assertEqual(f'a', 'a')
+        self.assertEqual(f' ', ' ')
+        self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}',
+                         '\N{GREEK CAPITAL LETTER DELTA}')
+        self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}',
+                         '\u0394')
+        self.assertEqual(f'\N{True}', '\u22a8')
+        self.assertEqual(rf'\N{True}', r'\NTrue')
+
+    def test_escape_order(self):
+        # note that hex(ord('{')) == 0x7b, so this
+        #  string becomes f'a{4*10}b'
+        self.assertEqual(f'a\u007b4*10}b', 'a40b')
+        self.assertEqual(f'a\x7b4*10}b', 'a40b')
+        self.assertEqual(f'a\x7b4*10\N{RIGHT CURLY BRACKET}b', 'a40b')
+        self.assertEqual(f'{"a"!\N{LATIN SMALL LETTER R}}', "'a'")
+        self.assertEqual(f'{10\x3a02X}', '0A')
+        self.assertEqual(f'{10:02\N{LATIN CAPITAL LETTER X}}', '0A')
+
+        self.assertAllRaise(SyntaxError, "f-string: single '}' is not allowed",
+                            [r"""f'a{\u007b4*10}b'""",    # mis-matched brackets
+                             ])
+        self.assertAllRaise(SyntaxError, 'unexpected character after line continuation character',
+                            [r"""f'{"a"\!r}'""",
+                             r"""f'{a\!r}'""",
+                             ])
+
+    def test_unterminated_string(self):
+        self.assertAllRaise(SyntaxError, 'f-string: unterminated string',
+                            [r"""f'{"x'""",
+                             r"""f'{"x}'""",
+                             r"""f'{("x'""",
+                             r"""f'{("x}'""",
+                             ])
+
+    def test_mismatched_parens(self):
+        self.assertAllRaise(SyntaxError, 'f-string: mismatched',
+                            ["f'{((}'",
+                             ])
+
+    def test_double_braces(self):
+        self.assertEqual(f'{{', '{')
+        self.assertEqual(f'a{{', 'a{')
+        self.assertEqual(f'{{b', '{b')
+        self.assertEqual(f'a{{b', 'a{b')
+        self.assertEqual(f'}}', '}')
+        self.assertEqual(f'a}}', 'a}')
+        self.assertEqual(f'}}b', '}b')
+        self.assertEqual(f'a}}b', 'a}b')
+
+        self.assertEqual(f'{{{10}', '{10')
+        self.assertEqual(f'}}{10}', '}10')
+        self.assertEqual(f'}}{{{10}', '}{10')
+        self.assertEqual(f'}}a{{{10}', '}a{10')
+
+        self.assertEqual(f'{10}{{', '10{')
+        self.assertEqual(f'{10}}}', '10}')
+        self.assertEqual(f'{10}}}{{', '10}{')
+        self.assertEqual(f'{10}}}a{{' '}', '10}a{}')
+
+        # Inside of strings, don't interpret doubled brackets.
+        self.assertEqual(f'{"{{}}"}', '{{}}')
+
+        self.assertAllRaise(TypeError, 'unhashable type',
+                            ["f'{ {{}} }'", # dict in a set
+                             ])
+
+    def test_compile_time_concat(self):
+        x = 'def'
+        self.assertEqual('abc' f'## {x}ghi', 'abc## defghi')
+        self.assertEqual('abc' f'{x}' 'ghi', 'abcdefghi')
+        self.assertEqual('abc' f'{x}' 'gh' f'i{x:4}', 'abcdefghidef ')
+        self.assertEqual('{x}' f'{x}', '{x}def')
+        self.assertEqual('{x' f'{x}', '{xdef')
+        self.assertEqual('{x}' f'{x}', '{x}def')
+        self.assertEqual('{{x}}' f'{x}', '{{x}}def')
+        self.assertEqual('{{x' f'{x}', '{{xdef')
+        self.assertEqual('x}}' f'{x}', 'x}}def')
+        self.assertEqual(f'{x}' 'x}}', 'defx}}')
+        self.assertEqual(f'{x}' '', 'def')
+        self.assertEqual('' f'{x}' '', 'def')
+        self.assertEqual('' f'{x}', 'def')
+        self.assertEqual(f'{x}' '2', 'def2')
+        self.assertEqual('1' f'{x}' '2', '1def2')
+        self.assertEqual('1' f'{x}', '1def')
+        self.assertEqual(f'{x}' f'-{x}', 'def-def')
+        self.assertEqual('' f'', '')
+        self.assertEqual('' f'' '', '')
+        self.assertEqual('' f'' '' f'', '')
+        self.assertEqual(f'', '')
+        self.assertEqual(f'' '', '')
+        self.assertEqual(f'' '' f'', '')
+        self.assertEqual(f'' '' f'' '', '')
+
+        self.assertAllRaise(SyntaxError, "f-string: expecting '}'",
+                            ["f'{3' f'}'",  # can't concat to get a valid f-string
+                             ])
+
+    def test_comments(self):
+        # These aren't comments, since they're in strings.
+        d = {'#': 'hash'}
+        self.assertEqual(f'{"#"}', '#')
+        self.assertEqual(f'{d["#"]}', 'hash')
+
+        self.assertAllRaise(SyntaxError, "f-string cannot include '#'",
+                            ["f'{1#}'",   # error because the expression becomes "(1#)"
+                             "f'{3(#)}'",
+                             ])
+
+    def test_many_expressions(self):
+        # Create a string with many expressions in it. Note that
+        #  because we have a space in here as a literal, we're actually
+        #  going to use twice as many ast nodes: one for each literal
+        #  plus one for each expression.
+        def build_fstr(n, extra=''):
+            return "f'" + ('{x} ' * n) + extra + "'"
+
+        x = 'X'
+        width = 1
+
+        # Test around 256.
+        for i in range(250, 260):
+            self.assertEqual(eval(build_fstr(i)), (x+' ')*i)
+
+        # Test concatenating 2 largs fstrings.
+        self.assertEqual(eval(build_fstr(255)*256), (x+' ')*(255*256))
+
+        s = build_fstr(253, '{x:{width}} ')
+        self.assertEqual(eval(s), (x+' ')*254)
+
+        # Test lots of expressions and constants, concatenated.
+        s = "f'{1}' 'x' 'y'" * 1024
+        self.assertEqual(eval(s), '1xy' * 1024)
+
+    def test_format_specifier_expressions(self):
+        width = 10
+        precision = 4
+        value = decimal.Decimal('12.34567')
+        self.assertEqual(f'result: {value:{width}.{precision}}', 'result:      12.35')
+        self.assertEqual(f'result: {value:{width!r}.{precision}}', 'result:      12.35')
+        self.assertEqual(f'result: {value:{width:0}.{precision:1}}', 'result:      12.35')
+        self.assertEqual(f'result: {value:{1}{0:0}.{precision:1}}', 'result:      12.35')
+        self.assertEqual(f'result: {value:{ 1}{ 0:0}.{ precision:1}}', 'result:      12.35')
+        self.assertEqual(f'{10:#{1}0x}', '       0xa')
+        self.assertEqual(f'{10:{"#"}1{0}{"x"}}', '       0xa')
+        self.assertEqual(f'{-10:-{"#"}1{0}x}', '      -0xa')
+        self.assertEqual(f'{-10:{"-"}#{1}0{"x"}}', '      -0xa')
+        self.assertEqual(f'{10:#{3 != {4:5} and width}x}', '       0xa')
+
+        self.assertAllRaise(SyntaxError, "f-string: expecting '}'",
+                            ["""f'{"s"!r{":10"}}'""",
+
+                             # This looks like a nested format spec.
+                             ])
+
+        self.assertAllRaise(SyntaxError, "invalid syntax",
+                            [# Invalid sytax inside a nested spec.
+                             "f'{4:{/5}}'",
+                             ])
+
+        self.assertAllRaise(SyntaxError, "f-string: expressions nested too deeply",
+                            [# Can't nest format specifiers.
+                             "f'result: {value:{width:{0}}.{precision:1}}'",
+                             ])
+
+        self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character',
+                            [# No expansion inside conversion or for
+                             #  the : or ! itself.
+                             """f'{"s"!{"r"}}'""",
+                             ])
+
+    def test_side_effect_order(self):
+        class X:
+            def __init__(self):
+                self.i = 0
+            def __format__(self, spec):
+                self.i += 1
+                return str(self.i)
+
+        x = X()
+        self.assertEqual(f'{x} {x}', '1 2')
+
+    def test_missing_expression(self):
+        self.assertAllRaise(SyntaxError, 'f-string: empty expression not allowed',
+                            ["f'{}'",
+                             "f'{ }'"
+                             "f' {} '",
+                             "f'{!r}'",
+                             "f'{ !r}'",
+                             "f'{10:{ }}'",
+                             "f' { } '",
+                             r"f'{\n}'",
+                             r"f'{\n \n}'",
+                             ])
+
+    def test_parens_in_expressions(self):
+        self.assertEqual(f'{3,}', '(3,)')
+
+        # Add these because when an expression is evaluated, parens
+        #  are added around it. But we shouldn't go from an invalid
+        #  expression to a valid one. The added parens are just
+        #  supposed to allow whitespace (including newlines).
+        self.assertAllRaise(SyntaxError, 'invalid syntax',
+                            ["f'{,}'",
+                             "f'{,}'",  # this is (,), which is an error
+                             ])
+
+        self.assertAllRaise(SyntaxError, "f-string: expecting '}'",
+                            ["f'{3)+(4}'",
+                             ])
+
+        self.assertAllRaise(SyntaxError, 'EOL while scanning string literal',
+                            ["f'{\n}'",
+                             ])
+
+    def test_newlines_in_expressions(self):
+        self.assertEqual(f'{0}', '0')
+        self.assertEqual(f'{0\n}', '0')
+        self.assertEqual(f'{0\r}', '0')
+        self.assertEqual(f'{\n0\n}', '0')
+        self.assertEqual(f'{\r0\r}', '0')
+        self.assertEqual(f'{\n0\r}', '0')
+        self.assertEqual(f'{\n0}', '0')
+        self.assertEqual(f'{3+\n4}', '7')
+        self.assertEqual(f'{3+\\\n4}', '7')
+        self.assertEqual(rf'''{3+
+4}''', '7')
+        self.assertEqual(f'''{3+\
+4}''', '7')
+
+        self.assertAllRaise(SyntaxError, 'f-string: empty expression not allowed',
+                            [r"f'{\n}'",
+                             ])
+
+    def test_lambda(self):
+        x = 5
+        self.assertEqual(f'{(lambda y:x*y)("8")!r}', "'88888'")
+        self.assertEqual(f'{(lambda y:x*y)("8")!r:10}', "'88888'   ")
+        self.assertEqual(f'{(lambda y:x*y)("8"):10}', "88888     ")
+
+        # lambda doesn't work without parens, because the colon
+        #  makes the parser think it's a format_spec
+        self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing',
+                            ["f'{lambda x:x}'",
+                             ])
+
+    def test_yield(self):
+        # Not terribly useful, but make sure the yield turns
+        #  a function into a generator
+        def fn(y):
+            f'y:{yield y*2}'
+
+        g = fn(4)
+        self.assertEqual(next(g), 8)
+
+    def test_yield_send(self):
+        def fn(x):
+            yield f'x:{yield (lambda i: x * i)}'
+
+        g = fn(10)
+        the_lambda = next(g)
+        self.assertEqual(the_lambda(4), 40)
+        self.assertEqual(g.send('string'), 'x:string')
+
+    def test_expressions_with_triple_quoted_strings(self):
+        self.assertEqual(f"{'''x'''}", 'x')
+        self.assertEqual(f"{'''eric's'''}", "eric's")
+        self.assertEqual(f'{"""eric\'s"""}', "eric's")
+        self.assertEqual(f"{'''eric\"s'''}", 'eric"s')
+        self.assertEqual(f'{"""eric"s"""}', 'eric"s')
+
+        # Test concatenation within an expression
+        self.assertEqual(f'{"x" """eric"s""" "y"}', 'xeric"sy')
+        self.assertEqual(f'{"x" """eric"s"""}', 'xeric"s')
+        self.assertEqual(f'{"""eric"s""" "y"}', 'eric"sy')
+        self.assertEqual(f'{"""x""" """eric"s""" "y"}', 'xeric"sy')
+        self.assertEqual(f'{"""x""" """eric"s""" """y"""}', 'xeric"sy')
+        self.assertEqual(f'{r"""x""" """eric"s""" """y"""}', 'xeric"sy')
+
+    def test_multiple_vars(self):
+        x = 98
+        y = 'abc'
+        self.assertEqual(f'{x}{y}', '98abc')
+
+        self.assertEqual(f'X{x}{y}', 'X98abc')
+        self.assertEqual(f'{x}X{y}', '98Xabc')
+        self.assertEqual(f'{x}{y}X', '98abcX')
+
+        self.assertEqual(f'X{x}Y{y}', 'X98Yabc')
+        self.assertEqual(f'X{x}{y}Y', 'X98abcY')
+        self.assertEqual(f'{x}X{y}Y', '98XabcY')
+
+        self.assertEqual(f'X{x}Y{y}Z', 'X98YabcZ')
+
+    def test_closure(self):
+        def outer(x):
+            def inner():
+                return f'x:{x}'
+            return inner
+
+        self.assertEqual(outer('987')(), 'x:987')
+        self.assertEqual(outer(7)(), 'x:7')
+
+    def test_arguments(self):
+        y = 2
+        def f(x, width):
+            return f'x={x*y:{width}}'
+
+        self.assertEqual(f('foo', 10), 'x=foofoo    ')
+        x = 'bar'
+        self.assertEqual(f(10, 10), 'x=        20')
+
+    def test_locals(self):
+        value = 123
+        self.assertEqual(f'v:{value}', 'v:123')
+
+    def test_missing_variable(self):
+        with self.assertRaises(NameError):
+            f'v:{value}'
+
+    def test_missing_format_spec(self):
+        class O:
+            def __format__(self, spec):
+                if not spec:
+                    return '*'
+                return spec
+
+        self.assertEqual(f'{O():x}', 'x')
+        self.assertEqual(f'{O()}', '*')
+        self.assertEqual(f'{O():}', '*')
+
+        self.assertEqual(f'{3:}', '3')
+        self.assertEqual(f'{3!s:}', '3')
+
+    def test_global(self):
+        self.assertEqual(f'g:{a_global}', 'g:global variable')
+        self.assertEqual(f'g:{a_global!r}', "g:'global variable'")
+
+        a_local = 'local variable'
+        self.assertEqual(f'g:{a_global} l:{a_local}',
+                         'g:global variable l:local variable')
+        self.assertEqual(f'g:{a_global!r}',
+                         "g:'global variable'")
+        self.assertEqual(f'g:{a_global} l:{a_local!r}',
+                         "g:global variable l:'local variable'")
+
+        self.assertIn("module 'unittest' from", f'{unittest}')
+
+    def test_shadowed_global(self):
+        a_global = 'really a local'
+        self.assertEqual(f'g:{a_global}', 'g:really a local')
+        self.assertEqual(f'g:{a_global!r}', "g:'really a local'")
+
+        a_local = 'local variable'
+        self.assertEqual(f'g:{a_global} l:{a_local}',
+                         'g:really a local l:local variable')
+        self.assertEqual(f'g:{a_global!r}',
+                         "g:'really a local'")
+        self.assertEqual(f'g:{a_global} l:{a_local!r}',
+                         "g:really a local l:'local variable'")
+
+    def test_call(self):
+        def foo(x):
+            return 'x=' + str(x)
+
+        self.assertEqual(f'{foo(10)}', 'x=10')
+
+    def test_nested_fstrings(self):
+        y = 5
+        self.assertEqual(f'{f"{0}"*3}', '000')
+        self.assertEqual(f'{f"{y}"*3}', '555')
+        self.assertEqual(f'{f"{\'x\'}"*3}', 'xxx')
+
+        self.assertEqual(f"{r'x' f'{\"s\"}'}", 'xs')
+        self.assertEqual(f"{r'x'rf'{\"s\"}'}", 'xs')
+
+    def test_invalid_string_prefixes(self):
+        self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing',
+                            ["fu''",
+                             "uf''",
+                             "Fu''",
+                             "fU''",
+                             "Uf''",
+                             "uF''",
+                             "ufr''",
+                             "urf''",
+                             "fur''",
+                             "fru''",
+                             "rfu''",
+                             "ruf''",
+                             "FUR''",
+                             "Fur''",
+                             ])
+
+    def test_leading_trailing_spaces(self):
+        self.assertEqual(f'{ 3}', '3')
+        self.assertEqual(f'{  3}', '3')
+        self.assertEqual(f'{\t3}', '3')
+        self.assertEqual(f'{\t\t3}', '3')
+        self.assertEqual(f'{3 }', '3')
+        self.assertEqual(f'{3  }', '3')
+        self.assertEqual(f'{3\t}', '3')
+        self.assertEqual(f'{3\t\t}', '3')
+
+        self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]}}',
+                         'expr={1: 2}')
+        self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]} }',
+                         'expr={1: 2}')
+
+    def test_character_name(self):
+        self.assertEqual(f'{4}\N{GREEK CAPITAL LETTER DELTA}{3}',
+                         '4\N{GREEK CAPITAL LETTER DELTA}3')
+        self.assertEqual(f'{{}}\N{GREEK CAPITAL LETTER DELTA}{3}',
+                         '{}\N{GREEK CAPITAL LETTER DELTA}3')
+
+    def test_not_equal(self):
+        # There's a special test for this because there's a special
+        #  case in the f-string parser to look for != as not ending an
+        #  expression. Normally it would, while looking for !s or !r.
+
+        self.assertEqual(f'{3!=4}', 'True')
+        self.assertEqual(f'{3!=4:}', 'True')
+        self.assertEqual(f'{3!=4!s}', 'True')
+        self.assertEqual(f'{3!=4!s:.3}', 'Tru')
+
+    def test_conversions(self):
+        self.assertEqual(f'{3.14:10.10}', '      3.14')
+        self.assertEqual(f'{3.14!s:10.10}', '3.14      ')
+        self.assertEqual(f'{3.14!r:10.10}', '3.14      ')
+        self.assertEqual(f'{3.14!a:10.10}', '3.14      ')
+
+        self.assertEqual(f'{"a"}', 'a')
+        self.assertEqual(f'{"a"!r}', "'a'")
+        self.assertEqual(f'{"a"!a}', "'a'")
+
+        # Not a conversion.
+        self.assertEqual(f'{"a!r"}', "a!r")
+
+        # Not a conversion, but show that ! is allowed in a format spec.
+        self.assertEqual(f'{3.14:!<10.10}', '3.14!!!!!!')
+
+        self.assertEqual(f'{"\N{GREEK CAPITAL LETTER DELTA}"}', '\u0394')
+        self.assertEqual(f'{"\N{GREEK CAPITAL LETTER DELTA}"!r}', "'\u0394'")
+        self.assertEqual(f'{"\N{GREEK CAPITAL LETTER DELTA}"!a}', "'\\u0394'")
+
+        self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character',
+                            ["f'{3!g}'",
+                             "f'{3!A}'",
+                             "f'{3!A}'",
+                             "f'{3!A}'",
+                             "f'{3!!}'",
+                             "f'{3!:}'",
+                             "f'{3!\N{GREEK CAPITAL LETTER DELTA}}'",
+                             "f'{3! s}'",  # no space before conversion char
+                             "f'{x!\\x00:.<10}'",
+                             ])
+
+        self.assertAllRaise(SyntaxError, "f-string: expecting '}'",
+                            ["f'{x!s{y}}'",
+                             "f'{3!ss}'",
+                             "f'{3!ss:}'",
+                             "f'{3!ss:s}'",
+                             ])
+
+    def test_assignment(self):
+        self.assertAllRaise(SyntaxError, 'invalid syntax',
+                            ["f'' = 3",
+                             "f'{0}' = x",
+                             "f'{x}' = x",
+                             ])
+
+    def test_del(self):
+        self.assertAllRaise(SyntaxError, 'invalid syntax',
+                            ["del f''",
+                             "del '' f''",
+                             ])
+
+    def test_mismatched_braces(self):
+        self.assertAllRaise(SyntaxError, "f-string: single '}' is not allowed",
+                            ["f'{{}'",
+                             "f'{{}}}'",
+                             "f'}'",
+                             "f'x}'",
+                             "f'x}x'",
+
+                             # Can't have { or } in a format spec.
+                             "f'{3:}>10}'",
+                             r"f'{3:\\}>10}'",
+                             "f'{3:}}>10}'",
+                             ])
+
+        self.assertAllRaise(SyntaxError, "f-string: expecting '}'",
+                            ["f'{3:{{>10}'",
+                             "f'{3'",
+                             "f'{3!'",
+                             "f'{3:'",
+                             "f'{3!s'",
+                             "f'{3!s:'",
+                             "f'{3!s:3'",
+                             "f'x{'",
+                             "f'x{x'",
+                             "f'{3:s'",
+                             "f'{{{'",
+                             "f'{{}}{'",
+                             "f'{'",
+                             ])
+
+        self.assertAllRaise(SyntaxError, 'invalid syntax',
+                            [r"f'{3:\\{>10}'",
+                             ])
+
+        # But these are just normal strings.
+        self.assertEqual(f'{"{"}', '{')
+        self.assertEqual(f'{"}"}', '}')
+        self.assertEqual(f'{3:{"}"}>10}', '}}}}}}}}}3')
+        self.assertEqual(f'{2:{"{"}>10}', '{{{{{{{{{2')
+
+    def test_if_conditional(self):
+        # There's special logic in compile.c to test if the
+        #  conditional for an if (and while) are constants. Exercise
+        #  that code.
+
+        def test_fstring(x, expected):
+            flag = 0
+            if f'{x}':
+                flag = 1
+            else:
+                flag = 2
+            self.assertEqual(flag, expected)
+
+        def test_concat_empty(x, expected):
+            flag = 0
+            if '' f'{x}':
+                flag = 1
+            else:
+                flag = 2
+            self.assertEqual(flag, expected)
+
+        def test_concat_non_empty(x, expected):
+            flag = 0
+            if ' ' f'{x}':
+                flag = 1
+            else:
+                flag = 2
+            self.assertEqual(flag, expected)
+
+        test_fstring('', 2)
+        test_fstring(' ', 1)
+
+        test_concat_empty('', 2)
+        test_concat_empty(' ', 1)
+
+        test_concat_non_empty('', 1)
+        test_concat_non_empty(' ', 1)
+
+    def test_empty_format_specifier(self):
+        x = 'test'
+        self.assertEqual(f'{x}', 'test')
+        self.assertEqual(f'{x:}', 'test')
+        self.assertEqual(f'{x!s:}', 'test')
+        self.assertEqual(f'{x!r:}', "'test'")
+
+    def test_str_format_differences(self):
+        d = {'a': 'string',
+             0: 'integer',
+             }
+        a = 0
+        self.assertEqual(f'{d[0]}', 'integer')
+        self.assertEqual(f'{d["a"]}', 'string')
+        self.assertEqual(f'{d[a]}', 'integer')
+        self.assertEqual('{d[a]}'.format(d=d), 'string')
+        self.assertEqual('{d[0]}'.format(d=d), 'integer')
+
+    def test_invalid_expressions(self):
+        self.assertAllRaise(SyntaxError, 'invalid syntax',
+                            [r"f'{a[4)}'",
+                             r"f'{a(4]}'",
+                            ])
+
+    def test_loop(self):
+        for i in range(1000):
+            self.assertEqual(f'i:{i}', 'i:' + str(i))
+
+    def test_dict(self):
+        d = {'"': 'dquote',
+             "'": 'squote',
+             'foo': 'bar',
+             }
+        self.assertEqual(f'{d["\'"]}', 'squote')
+        self.assertEqual(f"{d['\"']}", 'dquote')
+
+        self.assertEqual(f'''{d["'"]}''', 'squote')
+        self.assertEqual(f"""{d['"']}""", 'dquote')
+
+        self.assertEqual(f'{d["foo"]}', 'bar')
+        self.assertEqual(f"{d['foo']}", 'bar')
+        self.assertEqual(f'{d[\'foo\']}', 'bar')
+        self.assertEqual(f"{d[\"foo\"]}", 'bar')
+
+    def test_escaped_quotes(self):
+        d = {'"': 'a',
+             "'": 'b'}
+
+        self.assertEqual(fr"{d['\"']}", 'a')
+        self.assertEqual(fr'{d["\'"]}', 'b')
+        self.assertEqual(fr"{'\"'}", '"')
+        self.assertEqual(fr'{"\'"}', "'")
+        self.assertEqual(f'{"\\"3"}', '"3')
+
+        self.assertAllRaise(SyntaxError, 'f-string: unterminated string',
+                            [r'''f'{"""\\}' ''',  # Backslash at end of expression
+                             ])
+        self.assertAllRaise(SyntaxError, 'unexpected character after line continuation',
+                            [r"rf'{3\}'",
+                             ])
+
+
+if __name__ == '__main__':
+    unittest.main()

Misc/NEWS

@@ -19,6 +19,11 @@ Core and Builtins
   argument list of a function declaration.  For example, "def f(*, a =
   3,): pass" is now legal. Patch from Mark Dickinson.
 
+- Issue #24965: Implement PEP 498 "Literal String Interpolation". This
+  allows you to embed expressions inside f-strings, which are
+  converted to normal strings at run time. Given x=3, then
+  f'value={x}' == 'value=3'. Patch by Eric V. Smith.
+
 Library
 -------
 

Parser/Python.asdl

@@ -71,6 +71,8 @@ module Python
          | Call(expr func, expr* args, keyword* keywords)
          | Num(object n) -- a number as a PyObject.
          | Str(string s) -- need to specify raw, unicode, etc?
+         | FormattedValue(expr value, int? conversion, expr? format_spec)
+         | JoinedStr(expr* values)
          | Bytes(bytes s)
          | NameConstant(singleton value)
          | Ellipsis

Parser/tokenizer.c

@@ -1477,17 +1477,19 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
     nonascii = 0;
     if (is_potential_identifier_start(c)) {
         /* Process b"", r"", u"", br"" and rb"" */
-        int saw_b = 0, saw_r = 0, saw_u = 0;
+        int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0;
         while (1) {
-            if (!(saw_b || saw_u) && (c == 'b' || c == 'B'))
+            if (!(saw_b || saw_u || saw_f) && (c == 'b' || c == 'B'))
                 saw_b = 1;
             /* Since this is a backwards compatibility support literal we don't
                want to support it in arbitrary order like byte literals. */
-            else if (!(saw_b || saw_u || saw_r) && (c == 'u' || c == 'U'))
+            else if (!(saw_b || saw_u || saw_r || saw_f) && (c == 'u' || c == 'U'))
                 saw_u = 1;
             /* ur"" and ru"" are not supported */
             else if (!(saw_r || saw_u) && (c == 'r' || c == 'R'))
                 saw_r = 1;
+            else if (!(saw_f || saw_b || saw_u) && (c == 'f' || c == 'F'))
+                saw_f = 1;
             else
                 break;
             c = tok_nextc(tok);

Python/Python-ast.c

@@ -285,6 +285,18 @@ _Py_IDENTIFIER(s);
 static char *Str_fields[]={
     "s",
 };
+static PyTypeObject *FormattedValue_type;
+_Py_IDENTIFIER(conversion);
+_Py_IDENTIFIER(format_spec);
+static char *FormattedValue_fields[]={
+    "value",
+    "conversion",
+    "format_spec",
+};
+static PyTypeObject *JoinedStr_type;
+static char *JoinedStr_fields[]={
+    "values",
+};
 static PyTypeObject *Bytes_type;
 static char *Bytes_fields[]={
     "s",
@@ -917,6 +929,11 @@ static int init_types(void)
     if (!Num_type) return 0;
     Str_type = make_type("Str", expr_type, Str_fields, 1);
     if (!Str_type) return 0;
+    FormattedValue_type = make_type("FormattedValue", expr_type,
+                                    FormattedValue_fields, 3);
+    if (!FormattedValue_type) return 0;
+    JoinedStr_type = make_type("JoinedStr", expr_type, JoinedStr_fields, 1);
+    if (!JoinedStr_type) return 0;
     Bytes_type = make_type("Bytes", expr_type, Bytes_fields, 1);
     if (!Bytes_type) return 0;
     NameConstant_type = make_type("NameConstant", expr_type,
@@ -2062,6 +2079,42 @@ Str(string s, int lineno, int col_offset, PyArena *arena)
     return p;
 }
 
+expr_ty
+FormattedValue(expr_ty value, int conversion, expr_ty format_spec, int lineno,
+               int col_offset, PyArena *arena)
+{
+    expr_ty p;
+    if (!value) {
+        PyErr_SetString(PyExc_ValueError,
+                        "field value is required for FormattedValue");
+        return NULL;
+    }
+    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
+    if (!p)
+        return NULL;
+    p->kind = FormattedValue_kind;
+    p->v.FormattedValue.value = value;
+    p->v.FormattedValue.conversion = conversion;
+    p->v.FormattedValue.format_spec = format_spec;
+    p->lineno = lineno;
+    p->col_offset = col_offset;
+    return p;
+}
+
+expr_ty
+JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena *arena)
+{
+    expr_ty p;
+    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
+    if (!p)
+        return NULL;
+    p->kind = JoinedStr_kind;
+    p->v.JoinedStr.values = values;
+    p->lineno = lineno;
+    p->col_offset = col_offset;
+    return p;
+}
+
 expr_ty
 Bytes(bytes s, int lineno, int col_offset, PyArena *arena)
 {
@@ -3161,6 +3214,34 @@ ast2obj_expr(void* _o)
             goto failed;
         Py_DECREF(value);
         break;
+    case FormattedValue_kind:
+        result = PyType_GenericNew(FormattedValue_type, NULL, NULL);
+        if (!result) goto failed;
+        value = ast2obj_expr(o->v.FormattedValue.value);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        value = ast2obj_int(o->v.FormattedValue.conversion);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_conversion, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        value = ast2obj_expr(o->v.FormattedValue.format_spec);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_format_spec, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        break;
+    case JoinedStr_kind:
+        result = PyType_GenericNew(JoinedStr_type, NULL, NULL);
+        if (!result) goto failed;
+        value = ast2obj_list(o->v.JoinedStr.values, ast2obj_expr);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_values, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        break;
     case Bytes_kind:
         result = PyType_GenericNew(Bytes_type, NULL, NULL);
         if (!result) goto failed;
@@ -6022,6 +6103,86 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
+    isinstance = PyObject_IsInstance(obj, (PyObject*)FormattedValue_type);
+    if (isinstance == -1) {
+        return 1;
+    }
+    if (isinstance) {
+        expr_ty value;
+        int conversion;
+        expr_ty format_spec;
+
+        if (_PyObject_HasAttrId(obj, &PyId_value)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_value);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_expr(tmp, &value, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from FormattedValue");
+            return 1;
+        }
+        if (exists_not_none(obj, &PyId_conversion)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_conversion);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_int(tmp, &conversion, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            conversion = 0;
+        }
+        if (exists_not_none(obj, &PyId_format_spec)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_format_spec);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_expr(tmp, &format_spec, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            format_spec = NULL;
+        }
+        *out = FormattedValue(value, conversion, format_spec, lineno,
+                              col_offset, arena);
+        if (*out == NULL) goto failed;
+        return 0;
+    }
+    isinstance = PyObject_IsInstance(obj, (PyObject*)JoinedStr_type);
+    if (isinstance == -1) {
+        return 1;
+    }
+    if (isinstance) {
+        asdl_seq* values;
+
+        if (_PyObject_HasAttrId(obj, &PyId_values)) {
+            int res;
+            Py_ssize_t len;
+            Py_ssize_t i;
+            tmp = _PyObject_GetAttrId(obj, &PyId_values);
+            if (tmp == NULL) goto failed;
+            if (!PyList_Check(tmp)) {
+                PyErr_Format(PyExc_TypeError, "JoinedStr field \"values\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                goto failed;
+            }
+            len = PyList_GET_SIZE(tmp);
+            values = _Py_asdl_seq_new(len, arena);
+            if (values == NULL) goto failed;
+            for (i = 0; i < len; i++) {
+                expr_ty value;
+                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena);
+                if (res != 0) goto failed;
+                asdl_seq_SET(values, i, value);
+            }
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"values\" missing from JoinedStr");
+            return 1;
+        }
+        *out = JoinedStr(values, lineno, col_offset, arena);
+        if (*out == NULL) goto failed;
+        return 0;
+    }
     isinstance = PyObject_IsInstance(obj, (PyObject*)Bytes_type);
     if (isinstance == -1) {
         return 1;
@@ -7319,6 +7480,10 @@ PyInit__ast(void)
     if (PyDict_SetItemString(d, "Call", (PyObject*)Call_type) < 0) return NULL;
     if (PyDict_SetItemString(d, "Num", (PyObject*)Num_type) < 0) return NULL;
     if (PyDict_SetItemString(d, "Str", (PyObject*)Str_type) < 0) return NULL;
+    if (PyDict_SetItemString(d, "FormattedValue",
+        (PyObject*)FormattedValue_type) < 0) return NULL;
+    if (PyDict_SetItemString(d, "JoinedStr", (PyObject*)JoinedStr_type) < 0)
+        return NULL;
     if (PyDict_SetItemString(d, "Bytes", (PyObject*)Bytes_type) < 0) return
         NULL;
     if (PyDict_SetItemString(d, "NameConstant", (PyObject*)NameConstant_type) <

Python/compile.c

@@ -731,6 +731,7 @@ compiler_set_qualname(struct compiler *c)
     return 1;
 }
 
+
 /* Allocate a new block and return a pointer to it.
    Returns NULL on error.
 */
@@ -3209,6 +3210,117 @@ compiler_call(struct compiler *c, expr_ty e)
                                 e->v.Call.keywords);
 }
 
+static int
+compiler_joined_str(struct compiler *c, expr_ty e)
+{
+    /* Concatenate parts of a string using ''.join(parts). There are
+       probably better ways of doing this.
+
+       This is used for constructs like "'x=' f'{42}'", which have to
+       be evaluated at compile time. */
+
+    static PyObject *empty_string;
+    static PyObject *join_string;
+
+    if (!empty_string) {
+        empty_string = PyUnicode_FromString("");
+        if (!empty_string)
+            return 0;
+    }
+    if (!join_string) {
+        join_string = PyUnicode_FromString("join");
+        if (!join_string)
+            return 0;
+    }
+
+    ADDOP_O(c, LOAD_CONST, empty_string, consts);
+    ADDOP_NAME(c, LOAD_ATTR, join_string, names);
+    VISIT_SEQ(c, expr, e->v.JoinedStr.values);
+    ADDOP_I(c, BUILD_LIST, asdl_seq_LEN(e->v.JoinedStr.values));
+    ADDOP_I(c, CALL_FUNCTION, 1);
+    return 1;
+}
+
+/* Note that this code uses the builtin functions format(), str(),
+   repr(), and ascii(). You can break this code, or make it do odd
+   things, by redefining those functions. */
+static int
+compiler_formatted_value(struct compiler *c, expr_ty e)
+{
+    PyObject *conversion_name = NULL;
+
+    static PyObject *format_string;
+    static PyObject *str_string;
+    static PyObject *repr_string;
+    static PyObject *ascii_string;
+
+    if (!format_string) {
+        format_string = PyUnicode_InternFromString("format");
+        if (!format_string)
+            return 0;
+    }
+
+    if (!str_string) {
+        str_string = PyUnicode_InternFromString("str");
+        if (!str_string)
+            return 0;
+    }
+
+    if (!repr_string) {
+        repr_string = PyUnicode_InternFromString("repr");
+        if (!repr_string)
+            return 0;
+    }
+    if (!ascii_string) {
+        ascii_string = PyUnicode_InternFromString("ascii");
+        if (!ascii_string)
+            return 0;
+    }
+
+    ADDOP_NAME(c, LOAD_GLOBAL, format_string, names);
+
+    /* If needed, convert via str, repr, or ascii. */
+    if (e->v.FormattedValue.conversion != -1) {
+        switch (e->v.FormattedValue.conversion) {
+        case 's':
+            conversion_name = str_string;
+            break;
+        case 'r':
+            conversion_name = repr_string;
+            break;
+        case 'a':
+            conversion_name = ascii_string;
+            break;
+        default:
+            PyErr_SetString(PyExc_SystemError,
+                            "Unrecognized conversion character");
+            return 0;
+        }
+        ADDOP_NAME(c, LOAD_GLOBAL, conversion_name, names);
+    }
+
+    /* Evaluate the value. */
+    VISIT(c, expr, e->v.FormattedValue.value);
+
+    /* If needed, convert via str, repr, or ascii. */
+    if (conversion_name) {
+        /* Call the function we previously pushed. */
+        ADDOP_I(c, CALL_FUNCTION, 1);
+    }
+
+    /* If we have a format spec, use format(value, format_spec). Otherwise,
+       use the single argument form. */
+    if (e->v.FormattedValue.format_spec) {
+        VISIT(c, expr, e->v.FormattedValue.format_spec);
+        ADDOP_I(c, CALL_FUNCTION, 2);
+    } else {
+        /* No format spec specified, call format(value). */
+        ADDOP_I(c, CALL_FUNCTION, 1);
+    }
+
+    return 1;
+}
+
 /* shared code between compiler_call and compiler_class */
 static int
 compiler_call_helper(struct compiler *c,
@@ -3878,6 +3990,10 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
     case Str_kind:
         ADDOP_O(c, LOAD_CONST, e->v.Str.s, consts);
         break;
+    case JoinedStr_kind:
+        return compiler_joined_str(c, e);
+    case FormattedValue_kind:
+        return compiler_formatted_value(c, e);
     case Bytes_kind:
         ADDOP_O(c, LOAD_CONST, e->v.Bytes.s, consts);
         break;
@@ -4784,4 +4900,3 @@ PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags,
 {
     return PyAST_CompileEx(mod, filename, flags, -1, arena);
 }
-

Python/symtable.c

@@ -1439,6 +1439,14 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
         VISIT_SEQ(st, expr, e->v.Call.args);
         VISIT_SEQ_WITH_NULL(st, keyword, e->v.Call.keywords);
         break;
+    case FormattedValue_kind:
+        VISIT(st, expr, e->v.FormattedValue.value);
+        if (e->v.FormattedValue.format_spec)
+            VISIT(st, expr, e->v.FormattedValue.format_spec);
+        break;
+    case JoinedStr_kind:
+        VISIT_SEQ(st, expr, e->v.JoinedStr.values);
+        break;
     case Num_kind:
     case Str_kind:
     case Bytes_kind: