t-strings: Various changes (python#74)

Author: AA-Turner

Doc/library/token.rst

@@ -133,7 +133,7 @@ The token constants are:
 
 .. data:: TSTRING_START
 
-   Token value used to indicate the beginning of a t-string literal.
+   Token value used to indicate the beginning of a template string literal.
 
    .. impl-detail::
 
@@ -144,7 +144,7 @@ The token constants are:
 
 .. data:: TSTRING_MIDDLE
 
-   Token value used for literal text inside a t-string literal
+   Token value used for literal text inside a template string literal
    including format specifications.
 
    .. impl-detail::
@@ -158,7 +158,7 @@ The token constants are:
 
 .. data:: TSTRING_END
 
-   Token value used to indicate the end of a t-string.
+   Token value used to indicate the end of a template string literal.
 
    .. impl-detail::
 

Doc/whatsnew/3.14.rst

@@ -160,7 +160,8 @@ safe shell operations, improve logging, tackle modern ideas in web development
 See :pep:`750` for more details.
 
 (Contributed by Jim Baker, Guido van Rossum, Paul Everitt, Koudai Aono,
-Lysandros Nikolaou, and Dave Peck in :gh:`132661`.)
+Lysandros Nikolaou, Dave Peck, Adam Turner, Jelle Zijlstra, Bénédikt Tran,
+and Pablo Galindo Salgado in :gh:`132661`.)
 
 .. _whatsnew314-pep768:
 

Include/internal/pycore_interpolation.h

@@ -9,8 +9,6 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_stackref.h"    // _PyStackRef
-
 extern PyTypeObject _PyInterpolation_Type;
 
 #define _PyInterpolation_CheckExact(op) Py_IS_TYPE((op), &_PyInterpolation_Type)

Lib/_ast_unparse.py

@@ -573,11 +573,11 @@ def _write_str_avoiding_backslashes(self, string, *, quote_types=_ALL_QUOTES):
         quote_type = quote_types[0]
         self.write(f"{quote_type}{string}{quote_type}")
 
-    def _ftstring_helper(self, ftstring_parts):
-        new_ftstring_parts = []
+    def _ftstring_helper(self, parts):
+        new_parts = []
         quote_types = list(_ALL_QUOTES)
         fallback_to_repr = False
-        for value, is_constant in ftstring_parts:
+        for value, is_constant in parts:
             if is_constant:
                 value, new_quote_types = self._str_literal_helper(
                     value,
@@ -596,22 +596,22 @@ def _ftstring_helper(self, ftstring_parts):
                 new_quote_types = [q for q in quote_types if q not in value]
                 if new_quote_types:
                     quote_types = new_quote_types
-            new_ftstring_parts.append(value)
+            new_parts.append(value)
 
         if fallback_to_repr:
             # If we weren't able to find a quote type that works for all parts
             # of the JoinedStr, fallback to using repr and triple single quotes.
             quote_types = ["'''"]
-            new_ftstring_parts.clear()
-            for value, is_constant in ftstring_parts:
+            new_parts.clear()
+            for value, is_constant in parts:
                 if is_constant:
                     value = repr('"' + value)  # force repr to use single quotes
                     expected_prefix = "'\""
                     assert value.startswith(expected_prefix), repr(value)
                     value = value[len(expected_prefix):-1]
-                new_ftstring_parts.append(value)
+                new_parts.append(value)
 
-        value = "".join(new_ftstring_parts)
+        value = "".join(new_parts)
         quote_type = quote_types[0]
         self.write(f"{quote_type}{value}{quote_type}")
 

Lib/tokenize.py

@@ -251,7 +251,7 @@ def untokenize(self, iterable):
                     self.tokens.append(indent)
                     self.prev_col = len(indent)
                 startline = False
-            elif tok_type in (FSTRING_MIDDLE, TSTRING_MIDDLE):
+            elif tok_type in {FSTRING_MIDDLE, TSTRING_MIDDLE}:
                 if '{' in token or '}' in token:
                     token = self.escape_brackets(token)
                     last_line = token.splitlines()[-1]
@@ -308,7 +308,7 @@ def compat(self, token, iterable):
             elif startline and indents:
                 toks_append(indents[-1])
                 startline = False
-            elif toknum in (FSTRING_MIDDLE, TSTRING_MIDDLE):
+            elif toknum in {FSTRING_MIDDLE, TSTRING_MIDDLE}:
                 tokval = self.escape_brackets(tokval)
 
             # Insert a space between two consecutive brackets if we are in an f-string

Objects/interpolationobject.c

@@ -1,8 +1,9 @@
 /* t-string Interpolation object implementation */
 
 #include "Python.h"
-#include "pycore_initconfig.h"      // _PyStatus_OK
+#include "pycore_initconfig.h"    // _PyStatus_OK
 #include "pycore_interpolation.h"
+#include "pycore_typeobject.h"    // _PyType_GetDict
 
 static int
 _conversion_converter(PyObject *arg, PyObject **conversion)

Objects/templateobject.c

@@ -2,6 +2,7 @@
 
 #include "Python.h"
 #include "pycore_interpolation.h" // _PyInterpolation_CheckExact()
+#include "pycore_runtime.h"       // _Py_STR()
 #include "pycore_template.h"
 
 typedef struct {
@@ -155,13 +156,15 @@ template_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
         }
         else if (_PyInterpolation_CheckExact(item)) {
             if (!last_was_str) {
+                _Py_DECLARE_STR(empty, "");
                 PyTuple_SET_ITEM(strings, stringsidx++, &_Py_STR(empty));
             }
             PyTuple_SET_ITEM(interpolations, interpolationsidx++, Py_NewRef(item));
             last_was_str = 0;
         }
     }
     if (!last_was_str) {
+        _Py_DECLARE_STR(empty, "");
         PyTuple_SET_ITEM(strings, stringsidx++, &_Py_STR(empty));
     }
 

Objects/unicodeobject.c

@@ -56,7 +56,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 #include "pycore_pyhash.h"        // _Py_HashSecret_t
 #include "pycore_pylifecycle.h"   // _Py_SetFileSystemEncoding()
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
-#include "pycore_template.h"      // _PyTemplate_Type
+#include "pycore_template.h"      // _PyTemplate_Concat()
 #include "pycore_tuple.h"         // _PyTuple_FromArray()
 #include "pycore_ucnhash.h"       // _PyUnicode_Name_CAPI
 #include "pycore_unicodeobject.h" // struct _Py_unicode_state

Python/bytecodes.c

@@ -16,7 +16,7 @@
 #include "pycore_emscripten_signal.h"  // _Py_CHECK_EMSCRIPTEN_SIGNALS
 #include "pycore_function.h"
 #include "pycore_instruments.h"
-#include "pycore_interpolation.h" // _PyInterpolation_FromStackRefStealOnSuccess()
+#include "pycore_interpolation.h" // _PyInterpolation_Build()
 #include "pycore_intrinsics.h"
 #include "pycore_long.h"          // _PyLong_GetZero()
 #include "pycore_moduleobject.h"  // PyModuleObject
@@ -31,7 +31,7 @@
 #include "pycore_setobject.h"     // _PySet_NextEntry()
 #include "pycore_sliceobject.h"   // _PyBuildSlice_ConsumeRefs
 #include "pycore_stackref.h"
-#include "pycore_template.h"      // _PyTemplate_From{List,Values}()
+#include "pycore_template.h"      // _PyTemplate_Build()
 #include "pycore_tuple.h"         // _PyTuple_ITEMS()
 #include "pycore_typeobject.h"    // _PySuper_Lookup()
 

Python/ceval.c

@@ -19,7 +19,7 @@
 #include "pycore_import.h"        // _PyImport_IsDefaultImportFunc()
 #include "pycore_instruments.h"
 #include "pycore_interpframe.h"   // _PyFrame_SetStackPointer()
-#include "pycore_interpolation.h" // _PyInterpolation_FromStackRefStealOnSuccess()
+#include "pycore_interpolation.h" // _PyInterpolation_Build()
 #include "pycore_intrinsics.h"
 #include "pycore_jit.h"
 #include "pycore_list.h"          // _PyList_GetItemRef()
@@ -37,7 +37,7 @@
 #include "pycore_setobject.h"     // _PySet_Update()
 #include "pycore_sliceobject.h"   // _PyBuildSlice_ConsumeRefs
 #include "pycore_sysmodule.h"     // _PySys_GetOptionalAttrString()
-#include "pycore_template.h"      // _PyTemplate_From{List,Values}()
+#include "pycore_template.h"      // _PyTemplate_Build()
 #include "pycore_traceback.h"     // _PyTraceBack_FromFrame
 #include "pycore_tuple.h"         // _PyTuple_ITEMS()
 #include "pycore_uop_ids.h"       // Uops

Tools/cases_generator/analyzer.py

@@ -630,7 +630,6 @@ def has_error_without_pop(op: parser.CodeDef) -> bool:
     "_PyFrame_StackPush",
     "_PyFunction_SetVersion",
     "_PyGen_GetGeneratorFromFrame",
-    "_PyInterpolation_FromStackRefStealOnSuccess",
     "_PyInterpreterState_GET",
     "_PyList_AppendTakeRef",
     "_PyList_ITEMS",
@@ -649,7 +648,6 @@ def has_error_without_pop(op: parser.CodeDef) -> bool:
     "_PyObject_InlineValues",
     "_PyObject_IsUniquelyReferenced",
     "_PyObject_ManagedDictPointer",
-    "_PyTemplate_FromList",
     "_PyThreadState_HasStackSpace",
     "_PyTuple_FromStackRefStealOnSuccess",
     "_PyTuple_ITEMS",