Merge branch 3.8 into 3.8-slp

Author: Anselm Kruis

Doc/c-api/init_config.rst

@@ -194,25 +194,18 @@ PyPreConfig
    * Configure the LC_CTYPE locale
    * Set the UTF-8 mode
 
-   The :c:member:`struct_size` field must be explicitly initialized to
-   ``sizeof(PyPreConfig)``.
-
    Function to initialize a preconfiguration:
 
-   .. c:function:: PyStatus PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig)
+   .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig)
 
       Initialize the preconfiguration with :ref:`Python Configuration
       <init-python-config>`.
 
-   .. c:function:: PyStatus PyPreConfig_InitPythonConfig(PyPreConfig *preconfig)
+   .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig)
 
       Initialize the preconfiguration with :ref:`Isolated Configuration
       <init-isolated-conf>`.
 
-   The caller of these functions is responsible to handle exceptions (error or
-   exit) using :c:func:`PyStatus_Exception` and
-   :c:func:`Py_ExitStatusException`.
-
    Structure fields:
 
    .. c:member:: int allocator
@@ -274,13 +267,6 @@ PyPreConfig
       same way the regular Python parses command line arguments: see
       :ref:`Command Line Arguments <using-on-cmdline>`.
 
-   .. c:member:: size_t struct_size
-
-      Size of the structure in bytes: must be initialized to
-      ``sizeof(PyPreConfig)``.
-
-      Field used for API and ABI compatibility.
-
    .. c:member:: int use_environment
 
       See :c:member:`PyConfig.use_environment`.
@@ -332,12 +318,7 @@ Example using the preinitialization to enable the UTF-8 Mode::
 
     PyStatus status;
     PyPreConfig preconfig;
-    preconfig.struct_size = sizeof(PyPreConfig);
-
-    status = PyPreConfig_InitPythonConfig(&preconfig);
-    if (PyStatus_Exception(status)) {
-        Py_ExitStatusException(status);
-    }
+    PyPreConfig_InitPythonConfig(&preconfig);
 
     preconfig.utf8_mode = 1;
 
@@ -360,9 +341,6 @@ PyConfig
 
    Structure containing most parameters to configure Python.
 
-   The :c:member:`struct_size` field must be explicitly initialized to
-   ``sizeof(PyConfig)``.
-
    Structure methods:
 
    .. c:function:: PyStatus PyConfig_InitPythonConfig(PyConfig *config)
@@ -679,13 +657,6 @@ PyConfig
       Encoding and encoding errors of :data:`sys.stdin`, :data:`sys.stdout` and
       :data:`sys.stderr`.
 
-   .. c:member:: size_t struct_size
-
-      Size of the structure in bytes: must be initialized to
-      ``sizeof(PyConfig)``.
-
-      Field used for API and ABI compatibility.
-
    .. c:member:: int tracemalloc
 
       If non-zero, call :func:`tracemalloc.start` at startup.
@@ -754,7 +725,6 @@ Example setting the program name::
     {
         PyStatus status;
         PyConfig config;
-        config.struct_size = sizeof(PyConfig);
 
         status = PyConfig_InitPythonConfig(&config);
         if (PyStatus_Exception(status)) {
@@ -787,7 +757,6 @@ configuration, and then override some parameters::
     {
         PyStatus status;
         PyConfig config;
-        config.struct_size = sizeof(PyConfig);
 
         status = PyConfig_InitPythonConfig(&config);
         if (PyStatus_Exception(status)) {
@@ -875,7 +844,6 @@ Example of customized Python always running in isolated mode::
     {
         PyStatus status;
         PyConfig config;
-        config.struct_size = sizeof(PyConfig);
 
         status = PyConfig_InitPythonConfig(&config);
         if (PyStatus_Exception(status)) {
@@ -1067,7 +1035,6 @@ phases::
     {
         PyStatus status;
         PyConfig config;
-        config.struct_size = sizeof(PyConfig);
 
         status = PyConfig_InitPythonConfig(&config);
         if (PyStatus_Exception(status)) {

Doc/library/tarfile.rst

@@ -290,9 +290,10 @@ be finalized; only the internally used file object will be closed. See the
 
       *fileobj* is not closed, when :class:`TarFile` is closed.
 
-   *format* controls the archive format. It must be one of the constants
+   *format* controls the archive format for writing. It must be one of the constants
    :const:`USTAR_FORMAT`, :const:`GNU_FORMAT` or :const:`PAX_FORMAT` that are
-   defined at module level.
+   defined at module level. When reading, format will be automatically detected, even
+   if different formats are present in a single archive.
 
    The *tarinfo* argument can be used to replace the default :class:`TarInfo` class
    with a different one.

Include/cpython/initconfig.h

@@ -45,10 +45,6 @@ PyAPI_FUNC(PyStatus) PyWideStringList_Insert(PyWideStringList *list,
 /* --- PyPreConfig ----------------------------------------------- */
 
 typedef struct {
-    /* Size of the structure in bytes: must be initialized to
-       sizeof(PyPreConfig). Field used for API and ABI compatibility. */
-    size_t struct_size;
-
     int _config_init;     /* _PyConfigInitEnum value */
 
     /* Parse Py_PreInitializeFromBytesArgs() arguments?
@@ -124,17 +120,13 @@ typedef struct {
     int allocator;
 } PyPreConfig;
 
-PyAPI_FUNC(PyStatus) PyPreConfig_InitPythonConfig(PyPreConfig *config);
-PyAPI_FUNC(PyStatus) PyPreConfig_InitIsolatedConfig(PyPreConfig *config);
+PyAPI_FUNC(void) PyPreConfig_InitPythonConfig(PyPreConfig *config);
+PyAPI_FUNC(void) PyPreConfig_InitIsolatedConfig(PyPreConfig *config);
 
 
 /* --- PyConfig ---------------------------------------------- */
 
 typedef struct {
-    /* Size of the structure in bytes: must be initialized to
-       sizeof(PyConfig). Field used for API and ABI compatibility. */
-    size_t struct_size;
-
     int _config_init;     /* _PyConfigInitEnum value */
 
     int isolated;         /* Isolated mode? see PyPreConfig.isolated */

Include/internal/pycore_initconfig.h

@@ -120,8 +120,8 @@ extern PyStatus _PyPreCmdline_Read(_PyPreCmdline *cmdline,
 
 /* --- PyPreConfig ----------------------------------------------- */
 
-PyAPI_FUNC(PyStatus) _PyPreConfig_InitCompatConfig(PyPreConfig *preconfig);
-extern PyStatus _PyPreConfig_InitFromConfig(
+PyAPI_FUNC(void) _PyPreConfig_InitCompatConfig(PyPreConfig *preconfig);
+extern void _PyPreConfig_InitFromConfig(
     PyPreConfig *preconfig,
     const PyConfig *config);
 extern PyStatus _PyPreConfig_InitFromPreConfig(

Lib/test/support/__init__.py

@@ -74,6 +74,11 @@
 except ImportError:
     resource = None
 
+try:
+    import _hashlib
+except ImportError:
+    _hashlib = None
+
 __all__ = [
     # globals
     "PIPE_MAX_SIZE", "verbose", "max_memuse", "use_resources", "failfast",
@@ -91,8 +96,8 @@
     "create_empty_file", "can_symlink", "fs_is_case_insensitive",
     # unittest
     "is_resource_enabled", "requires", "requires_freebsd_version",
-    "requires_linux_version", "requires_mac_ver", "check_syntax_error",
-    "check_syntax_warning",
+    "requires_linux_version", "requires_mac_ver", "requires_hashdigest",
+    "check_syntax_error", "check_syntax_warning",
     "TransientResource", "time_out", "socket_peer_reset", "ioerror_peer_reset",
     "transient_internet", "BasicTestRunner", "run_unittest", "run_doctest",
     "skip_unless_symlink", "requires_gzip", "requires_bz2", "requires_lzma",
@@ -652,20 +657,27 @@ def wrapper(*args, **kw):
     return decorator
 
 
-def requires_hashdigest(digestname):
+def requires_hashdigest(digestname, openssl=None):
     """Decorator raising SkipTest if a hashing algorithm is not available
 
     The hashing algorithm could be missing or blocked by a strict crypto
     policy.
 
+    If 'openssl' is True, then the decorator checks that OpenSSL provides
+    the algorithm. Otherwise the check falls back to built-in
+    implementations.
+
     ValueError: [digital envelope routines: EVP_DigestInit_ex] disabled for FIPS
     ValueError: unsupported hash type md4
     """
     def decorator(func):
         @functools.wraps(func)
         def wrapper(*args, **kwargs):
             try:
-                hashlib.new(digestname)
+                if openssl and _hashlib is not None:
+                    _hashlib.new(digestname)
+                else:
+                    hashlib.new(digestname)
             except ValueError:
                 raise unittest.SkipTest(
                     f"hash digest '{digestname}' is not available."

Lib/test/test_hashlib.py

@@ -8,6 +8,7 @@
 
 import array
 from binascii import unhexlify
+import functools
 import hashlib
 import importlib
 import itertools
@@ -18,6 +19,7 @@
 import warnings
 from test import support
 from test.support import _4G, bigmemtest, import_fresh_module
+from test.support import requires_hashdigest
 from http.client import HTTPException
 
 # Were we compiled --with-pydebug or with #define Py_DEBUG?
@@ -119,6 +121,7 @@ def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs):
             constructors.add(_test_algorithm_via_hashlib_new)
 
         _hashlib = self._conditional_import_module('_hashlib')
+        self._hashlib = _hashlib
         if _hashlib:
             # These two algorithms should always be present when this module
             # is compiled.  If not, something was compiled wrong.
@@ -127,7 +130,13 @@ def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs):
             for algorithm, constructors in self.constructors_to_test.items():
                 constructor = getattr(_hashlib, 'openssl_'+algorithm, None)
                 if constructor:
-                    constructors.add(constructor)
+                    try:
+                        constructor()
+                    except ValueError:
+                        # default constructor blocked by crypto policy
+                        pass
+                    else:
+                        constructors.add(constructor)
 
         def add_builtin_constructor(name):
             constructor = getattr(hashlib, "__get_builtin_constructor")(name)

Lib/test/test_hmac.py

@@ -21,7 +21,7 @@ def wrapper(*args, **kwargs):
 
 class TestVectorsTestCase(unittest.TestCase):
 
-    @requires_hashdigest('md5')
+    @requires_hashdigest('md5', openssl=True)
     def test_md5_vectors(self):
         # Test the HMAC module against test vectors from the RFC.
 
@@ -79,7 +79,7 @@ def md5test(key, data, digest):
                  b"and Larger Than One Block-Size Data"),
                 "6f630fad67cda0ee1fb1f562db3aa53e")
 
-    @requires_hashdigest('sha1')
+    @requires_hashdigest('sha1', openssl=True)
     def test_sha_vectors(self):
         def shatest(key, data, digest):
             h = hmac.HMAC(key, data, digestmod=hashlib.sha1)
@@ -272,19 +272,19 @@ def hmactest(key, data, hexdigests):
                                    '134676fb6de0446065c97440fa8c6a58',
                  })
 
-    @requires_hashdigest('sha224')
+    @requires_hashdigest('sha224', openssl=True)
     def test_sha224_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha224, 'sha224', 28, 64)
 
-    @requires_hashdigest('sha256')
+    @requires_hashdigest('sha256', openssl=True)
     def test_sha256_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha256, 'sha256', 32, 64)
 
-    @requires_hashdigest('sha384')
+    @requires_hashdigest('sha384', openssl=True)
     def test_sha384_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha384, 'sha384', 48, 128)
 
-    @requires_hashdigest('sha512')
+    @requires_hashdigest('sha512', openssl=True)
     def test_sha512_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha512, 'sha512', 64, 128)
 

Misc/NEWS.d/next/C API/2019-09-28-03-43-27.bpo-38304.RqHAwd.rst

@@ -0,0 +1,3 @@
+Fix a bug due to the interaction of weakrefs and the cyclic garbage
+collector. We must clear any weakrefs in garbage in order to prevent their
+callbacks from executing and causing a crash.

Misc/NEWS.d/next/Core and Builtins/2019-09-30-09-33-21.bpo-38006.UYlJum.rst

@@ -556,7 +556,7 @@ EVPnew(const EVP_MD *digest,
     if ((self = newEVPobject()) == NULL)
         return NULL;
 
-    if (!EVP_DigestInit(self->ctx, digest)) {
+    if (!EVP_DigestInit_ex(self->ctx, digest, NULL)) {
         _setException(PyExc_ValueError);
         Py_DECREF(self);
         return NULL;

Modules/_hashopenssl.c

@@ -678,6 +678,21 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old)
         op = FROM_GC(gc);
         next = GC_NEXT(gc);
 
+        if (PyWeakref_Check(op)) {
+            /* A weakref inside the unreachable set must be cleared.  If we
+             * allow its callback to execute inside delete_garbage(), it
+             * could expose objects that have tp_clear already called on
+             * them.  Or, it could resurrect unreachable objects.  One way
+             * this can happen is if some container objects do not implement
+             * tp_traverse.  Then, wr_object can be outside the unreachable
+             * set but can be deallocated as a result of breaking the
+             * reference cycle.  If we don't clear the weakref, the callback
+             * will run and potentially cause a crash.  See bpo-38006 for
+             * one example.
+             */
+            _PyWeakref_ClearRef((PyWeakReference *)op);
+        }
+
         if (! PyType_SUPPORTS_WEAKREFS(Py_TYPE(op)))
             continue;
 
@@ -733,6 +748,8 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old)
              * is moved to wrcb_to_call in this case.
              */
             if (gc_is_collecting(AS_GC(wr))) {
+                /* it should already have been cleared above */
+                assert(wr->wr_object == Py_None);
                 continue;
             }
 

Modules/gcmodule.c

@@ -53,20 +53,15 @@ pymain_init(const _PyArgv *args)
 #endif
 
     PyPreConfig preconfig;
-    preconfig.struct_size = sizeof(PyPreConfig);
-
-    status = PyPreConfig_InitPythonConfig(&preconfig);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
+    PyPreConfig_InitPythonConfig(&preconfig);
 
     status = _Py_PreInitializeFromPyArgv(&preconfig, args);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
     PyConfig config;
-    config.struct_size = sizeof(PyConfig);
+
     status = PyConfig_InitPythonConfig(&config);
     if (_PyStatus_EXCEPTION(status)) {
         goto done;

Modules/main.c

@@ -315,15 +315,13 @@ canonicalize(wchar_t *buffer, const wchar_t *path)
    'prefix' is null terminated in bounds.  join() ensures
    'landmark' can not overflow prefix if too long. */
 static int
-gotlandmark(wchar_t *prefix, const wchar_t *landmark)
+gotlandmark(const wchar_t *prefix, const wchar_t *landmark)
 {
-    int ok;
-    Py_ssize_t n = wcsnlen_s(prefix, MAXPATHLEN);
-
-    join(prefix, landmark);
-    ok = ismodule(prefix, FALSE);
-    prefix[n] = '\0';
-    return ok;
+    wchar_t filename[MAXPATHLEN+1];
+    memset(filename, 0, sizeof(filename));
+    wcscpy_s(filename, Py_ARRAY_LENGTH(filename), prefix);
+    join(filename, landmark);
+    return ismodule(filename, FALSE);
 }