Merge branch 'master' into fstring-error-message

Author: lysnikolaou

Doc/glossary.rst

@@ -189,6 +189,10 @@ Glossary
       A list of bytecode instructions can be found in the documentation for
       :ref:`the dis module <bytecodes>`.
 
+   callback
+      A subroutine function which is passed as an argument to be executed at
+      some point in the future.
+
    class
       A template for creating user-defined objects. Class definitions
       normally contain method definitions which operate on instances of the

Doc/library/asyncio-dev.rst

@@ -73,7 +73,7 @@ event loop, no other Tasks can run in the same thread.  When a Task
 executes an ``await`` expression, the running Task gets suspended, and
 the event loop executes the next Task.
 
-To schedule a callback from a different OS thread, the
+To schedule a :term:`callback` from another OS thread, the
 :meth:`loop.call_soon_threadsafe` method should be used. Example::
 
     loop.call_soon_threadsafe(callback, *args)

Doc/library/asyncio-eventloop.rst

@@ -191,8 +191,8 @@ Scheduling callbacks
 
 .. method:: loop.call_soon(callback, *args, context=None)
 
-   Schedule a *callback* to be called with *args* arguments at
-   the next iteration of the event loop.
+   Schedule the *callback* :term:`callback` to be called with
+   *args* arguments at the next iteration of the event loop.
 
    Callbacks are called in the order in which they are registered.
    Each callback will be called exactly once.

Include/cpython/listobject.h

@@ -32,4 +32,3 @@ PyAPI_FUNC(void) _PyList_DebugMallocStats(FILE *out);
 #define PyList_GET_ITEM(op, i) (_PyList_CAST(op)->ob_item[i])
 #define PyList_SET_ITEM(op, i, v) (_PyList_CAST(op)->ob_item[i] = (v))
 #define PyList_GET_SIZE(op)    Py_SIZE(_PyList_CAST(op))
-#define _PyList_ITEMS(op)      (_PyList_CAST(op)->ob_item)

Include/internal/pycore_gc.h

@@ -169,7 +169,7 @@ extern void _PyFrame_ClearFreeList(PyThreadState *tstate);
 extern void _PyTuple_ClearFreeList(PyThreadState *tstate);
 extern void _PyFloat_ClearFreeList(PyThreadState *tstate);
 extern void _PyList_ClearFreeList(PyThreadState *tstate);
-extern void _PyDict_ClearFreeList(void);
+extern void _PyDict_ClearFreeList(PyThreadState *tstate);
 extern void _PyAsyncGen_ClearFreeLists(PyThreadState *tstate);
 extern void _PyContext_ClearFreeList(PyThreadState *tstate);
 

Include/internal/pycore_interp.h

@@ -65,10 +65,23 @@ struct _Py_unicode_fs_codec {
     _Py_error_handler error_handler;
 };
 
+struct _Py_bytes_state {
+    PyBytesObject *characters[256];
+    PyBytesObject *empty_string;
+};
+
 struct _Py_unicode_state {
     struct _Py_unicode_fs_codec fs_codec;
 };
 
+struct _Py_float_state {
+    /* Special free list
+       free_list is a singly-linked list of available PyFloatObjects,
+       linked via abuse of their ob_type members. */
+    int numfree;
+    PyFloatObject *free_list;
+};
+
 /* Speed optimization to avoid frequent malloc/free of small tuples */
 #ifndef PyTuple_MAXSAVESIZE
    // Largest tuple to save on free list
@@ -99,12 +112,16 @@ struct _Py_list_state {
     int numfree;
 };
 
-struct _Py_float_state {
-    /* Special free list
-       free_list is a singly-linked list of available PyFloatObjects,
-       linked via abuse of their ob_type members. */
+#ifndef PyDict_MAXFREELIST
+#  define PyDict_MAXFREELIST 80
+#endif
+
+struct _Py_dict_state {
+    /* Dictionary reuse scheme to save calls to malloc and free */
+    PyDictObject *free_list[PyDict_MAXFREELIST];
     int numfree;
-    PyFloatObject *free_list;
+    PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST];
+    int keys_numfree;
 };
 
 struct _Py_frame_state {
@@ -136,7 +153,6 @@ struct _Py_context_state {
 };
 
 
-
 /* interpreter state */
 
 #define _PY_NSMALLPOSINTS           257
@@ -182,8 +198,6 @@ struct _is {
     PyObject *codec_error_registry;
     int codecs_initialized;
 
-    struct _Py_unicode_state unicode;
-
     PyConfig config;
 #ifdef HAVE_DLOPEN
     int dlopenflags;
@@ -224,16 +238,19 @@ struct _is {
     */
     PyLongObject* small_ints[_PY_NSMALLNEGINTS + _PY_NSMALLPOSINTS];
 #endif
+    struct _Py_bytes_state bytes;
+    struct _Py_unicode_state unicode;
+    struct _Py_float_state float_state;
+    /* Using a cache is very effective since typically only a single slice is
+       created and then deleted again. */
+    PySliceObject *slice_cache;
+
     struct _Py_tuple_state tuple;
     struct _Py_list_state list;
-    struct _Py_float_state float_state;
+    struct _Py_dict_state dict_state;
     struct _Py_frame_state frame;
     struct _Py_async_gen_state async_gen;
     struct _Py_context_state context;
-
-    /* Using a cache is very effective since typically only a single slice is
-       created and then deleted again. */
-    PySliceObject *slice_cache;
 };
 
 /* Used by _PyImport_Cleanup() */

Include/internal/pycore_list.h

@@ -0,0 +1,20 @@
+#ifndef Py_INTERNAL_LIST_H
+#define Py_INTERNAL_LIST_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+#include "listobject.h"           // _PyList_CAST()
+
+
+#define _PyList_ITEMS(op) (_PyList_CAST(op)->ob_item)
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif   /* !Py_INTERNAL_LIST_H */

Include/internal/pycore_pylifecycle.h

@@ -59,11 +59,10 @@ extern PyStatus _PyGC_Init(PyThreadState *tstate);
 /* Various internal finalizers */
 
 extern void _PyFrame_Fini(PyThreadState *tstate);
-extern void _PyDict_Fini(void);
+extern void _PyDict_Fini(PyThreadState *tstate);
 extern void _PyTuple_Fini(PyThreadState *tstate);
 extern void _PyList_Fini(PyThreadState *tstate);
-extern void _PySet_Fini(void);
-extern void _PyBytes_Fini(void);
+extern void _PyBytes_Fini(PyThreadState *tstate);
 extern void _PyFloat_Fini(PyThreadState *tstate);
 extern void _PySlice_Fini(PyThreadState *tstate);
 extern void _PyAsyncGen_Fini(PyThreadState *tstate);

Include/internal/pycore_tupleobject.h renamed to Include/internal/pycore_tuple.h

@@ -1,5 +1,5 @@
-#ifndef Py_INTERNAL_TUPLEOBJECT_H
-#define Py_INTERNAL_TUPLEOBJECT_H
+#ifndef Py_INTERNAL_TUPLE_H
+#define Py_INTERNAL_TUPLE_H
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -11,9 +11,10 @@ extern "C" {
 #include "tupleobject.h"   /* _PyTuple_CAST() */
 
 #define _PyTuple_ITEMS(op) (_PyTuple_CAST(op)->ob_item)
+
 PyAPI_FUNC(PyObject *) _PyTuple_FromArray(PyObject *const *, Py_ssize_t);
 
 #ifdef __cplusplus
 }
 #endif
-#endif   /* !Py_INTERNAL_TUPLEOBJECT_H */
+#endif   /* !Py_INTERNAL_TUPLE_H */

Lib/random.py

@@ -39,7 +39,8 @@
 
 from warnings import warn as _warn
 from math import log as _log, exp as _exp, pi as _pi, e as _e, ceil as _ceil
-from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin, tau as TWOPI
+from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin
+from math import tau as TWOPI, floor as _floor
 from os import urandom as _urandom
 from _collections_abc import Set as _Set, Sequence as _Sequence
 from itertools import accumulate as _accumulate, repeat as _repeat
@@ -234,7 +235,7 @@ def __reduce__(self):
 
     ## -------------------- integer methods  -------------------
 
-    def randrange(self, start, stop=None, step=1, _int=int):
+    def randrange(self, start, stop=None, step=1):
         """Choose a random item from range(start, stop[, step]).
 
         This fixes the problem with randint() which includes the
@@ -244,7 +245,7 @@ def randrange(self, start, stop=None, step=1, _int=int):
 
         # This code is a bit messy to make it fast for the
         # common case while still doing adequate error checking.
-        istart = _int(start)
+        istart = int(start)
         if istart != start:
             raise ValueError("non-integer arg 1 for randrange()")
         if stop is None:
@@ -253,7 +254,7 @@ def randrange(self, start, stop=None, step=1, _int=int):
             raise ValueError("empty range for randrange()")
 
         # stop argument supplied.
-        istop = _int(stop)
+        istop = int(stop)
         if istop != stop:
             raise ValueError("non-integer stop for randrange()")
         width = istop - istart
@@ -263,7 +264,7 @@ def randrange(self, start, stop=None, step=1, _int=int):
             raise ValueError("empty range for randrange() (%d, %d, %d)" % (istart, istop, width))
 
         # Non-unit step argument supplied.
-        istep = _int(step)
+        istep = int(step)
         if istep != step:
             raise ValueError("non-integer step for randrange()")
         if istep > 0:
@@ -296,7 +297,7 @@ def _randbelow_with_getrandbits(self, n):
             r = getrandbits(k)
         return r
 
-    def _randbelow_without_getrandbits(self, n, int=int, maxsize=1<<BPF):
+    def _randbelow_without_getrandbits(self, n, maxsize=1<<BPF):
         """Return a random int in the range [0,n).  Returns 0 if n==0.
 
         The implementation does not use getrandbits, but only random.
@@ -307,15 +308,15 @@ def _randbelow_without_getrandbits(self, n, int=int, maxsize=1<<BPF):
             _warn("Underlying random() generator does not supply \n"
                 "enough bits to choose from a population range this large.\n"
                 "To remove the range limitation, add a getrandbits() method.")
-            return int(random() * n)
+            return _floor(random() * n)
         if n == 0:
             return 0
         rem = maxsize % n
         limit = (maxsize - rem) / maxsize   # int(limit * maxsize) % n == 0
         r = random()
         while r >= limit:
             r = random()
-        return int(r * maxsize) % n
+        return _floor(r * maxsize) % n
 
     _randbelow = _randbelow_with_getrandbits
 
@@ -346,10 +347,10 @@ def shuffle(self, x, random=None):
                   'since Python 3.9 and will be removed in a subsequent '
                   'version.',
                   DeprecationWarning, 2)
-            _int = int
+            floor = _floor
             for i in reversed(range(1, len(x))):
                 # pick an element in x[:i+1] with which to exchange x[i]
-                j = _int(random() * (i + 1))
+                j = floor(random() * (i + 1))
                 x[i], x[j] = x[j], x[i]
 
     def sample(self, population, k, *, counts=None):
@@ -462,9 +463,9 @@ def choices(self, population, weights=None, *, cum_weights=None, k=1):
         n = len(population)
         if cum_weights is None:
             if weights is None:
-                _int = int
+                floor = _floor
                 n += 0.0    # convert to float for a small speed improvement
-                return [population[_int(random() * n)] for i in _repeat(None, k)]
+                return [population[floor(random() * n)] for i in _repeat(None, k)]
             cum_weights = list(_accumulate(weights))
         elif weights is not None:
             raise TypeError('Cannot specify both weights and cumulative weights')
@@ -814,24 +815,20 @@ def _notimplemented(self, *args, **kwds):
 ## -------------------- test program --------------------
 
 def _test_generator(n, func, args):
-    import time
-    print(n, 'times', func.__name__)
-    total = 0.0
-    sqsum = 0.0
-    smallest = 1e10
-    largest = -1e10
-    t0 = time.perf_counter()
-    for i in range(n):
-        x = func(*args)
-        total += x
-        sqsum = sqsum + x*x
-        smallest = min(x, smallest)
-        largest = max(x, largest)
-    t1 = time.perf_counter()
-    print(round(t1 - t0, 3), 'sec,', end=' ')
-    avg = total / n
-    stddev = _sqrt(sqsum / n - avg * avg)
-    print('avg %g, stddev %g, min %g, max %g\n' % (avg, stddev, smallest, largest))
+    from statistics import stdev, fmean as mean
+    from time import perf_counter
+
+    t0 = perf_counter()
+    data = [func(*args) for i in range(n)]
+    t1 = perf_counter()
+
+    xbar = mean(data)
+    sigma = stdev(data, xbar)
+    low = min(data)
+    high = max(data)
+
+    print(f'{t1 - t0:.3f} sec, {n} times {func.__name__}')
+    print('avg %g, stddev %g, min %g, max %g\n' % (xbar, sigma, low, high))
 
 
 def _test(N=2000):

Lib/test/test_marshal.py

@@ -158,13 +158,6 @@ def test_sets(self):
         for constructor in (set, frozenset):
             self.helper(constructor(self.d.keys()))
 
-    @support.cpython_only
-    def test_empty_frozenset_singleton(self):
-        # marshal.loads() must reuse the empty frozenset singleton
-        obj = frozenset()
-        obj2 = marshal.loads(marshal.dumps(obj))
-        self.assertIs(obj2, obj)
-
 
 class BufferTestCase(unittest.TestCase, HelperMixin):
 

Lib/test/test_set.py

@@ -661,15 +661,6 @@ def test_init(self):
         s.__init__(self.otherword)
         self.assertEqual(s, set(self.word))
 
-    def test_singleton_empty_frozenset(self):
-        f = frozenset()
-        efs = [frozenset(), frozenset([]), frozenset(()), frozenset(''),
-               frozenset(), frozenset([]), frozenset(()), frozenset(''),
-               frozenset(range(0)), frozenset(frozenset()),
-               frozenset(f), f]
-        # All of the empty frozensets should have just one id()
-        self.assertEqual(len(set(map(id, efs))), 1)
-
     def test_constructor_identity(self):
         s = self.thetype(range(3))
         t = self.thetype(s)

Lib/zoneinfo/_common.py

@@ -136,8 +136,7 @@ class _TZifHeader:
     ]
 
     def __init__(self, *args):
-        assert len(self.__slots__) == len(args)
-        for attr, val in zip(self.__slots__, args):
+        for attr, val in zip(self.__slots__, args, strict=True):
             setattr(self, attr, val)
 
     @classmethod

Makefile.pre.in

@@ -1110,6 +1110,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_import.h \
 		$(srcdir)/Include/internal/pycore_initconfig.h \
 		$(srcdir)/Include/internal/pycore_interp.h \
+		$(srcdir)/Include/internal/pycore_list.h \
 		$(srcdir)/Include/internal/pycore_object.h \
 		$(srcdir)/Include/internal/pycore_pathconfig.h \
 		$(srcdir)/Include/internal/pycore_pyerrors.h \
@@ -1120,7 +1121,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_runtime.h \
 		$(srcdir)/Include/internal/pycore_sysmodule.h \
 		$(srcdir)/Include/internal/pycore_traceback.h \
-		$(srcdir)/Include/internal/pycore_tupleobject.h \
+		$(srcdir)/Include/internal/pycore_tuple.h \
 		$(srcdir)/Include/internal/pycore_warnings.h \
 		$(DTRACE_HEADERS)
 

Misc/NEWS.d/next/Core and Builtins/2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst

@@ -0,0 +1 @@
+Port :mod:`_lzma` to multiphase initialization.

Misc/NEWS.d/next/Core and Builtins/2020-05-20-01-17-34.bpo-40521.wvAehI.rst

@@ -1,4 +1,10 @@
-The tuple free lists, the empty tuple singleton, the list free list, the float
-free list, the slice cache, the frame free list, the asynchronous generator
-free lists, and the context free list are no longer shared by all interpreters:
-each interpreter now its has own free lists and caches.
+Each interpreter now its has own free lists, singletons and caches:
+
+* Free lists: float, tuple, list, dict, frame, context,
+  asynchronous generator.
+* Singletons: empty tuple, empty bytes string,
+  single byte character.
+* Slice cache.
+
+They are no longer shared by all interpreters.
+