00361: openssl-3-compatibility

Backported from Python 3.8

Based on https://github.com/stratakis/cpython/tree/fedora-3.6_openssl3_compat

Some fixes

Author: miss-islington

Doc/library/hashlib.rst

@@ -109,10 +109,10 @@ More condensed:
 
 Using :func:`new` with an algorithm provided by OpenSSL:
 
-   >>> h = hashlib.new('ripemd160')
+   >>> h = hashlib.new('sha512_256')
    >>> h.update(b"Nobody inspects the spammish repetition")
    >>> h.hexdigest()
-   'cc4a5ce1b3df48aec5d22d1f16b894a0b894eccc'
+   '19197dc4d03829df858011c6c87600f994a858103bbc19005f20987aa19a97e2'
 
 Hashlib provides the following constant attributes:
 

Doc/library/ssl.rst

@@ -844,6 +844,14 @@ Constants
 
    .. versionadded:: 3.6
 
+.. data:: OP_IGNORE_UNEXPECTED_EOF
+
+   Ignore unexpected shutdown of TLS connections.
+
+   This option is only available with OpenSSL 3.0.0 and later.
+
+   .. versionadded:: 3..6.15-27
+
 .. data:: HAS_ALPN
 
    Whether the OpenSSL library has built-in support for the *Application-Layer
@@ -1143,6 +1151,9 @@ SSL sockets also have the following additional methods and attributes:
       The returned dictionary includes additional X509v3 extension items
         such as ``crlDistributionPoints``, ``caIssuers`` and ``OCSP`` URIs.
 
+   .. versionchanged:: 3.6.15-27
+      IPv6 address strings no longer have a trailing new line.
+
 .. method:: SSLSocket.cipher()
 
    Returns a three-value tuple containing the name of the cipher being used, the

Lib/test/test_ssl.py

@@ -0,0 +1,2 @@
+Make Python compatible with OpenSSL 3.0.0. :func:`ssl.SSLSocket.getpeercert`
+no longer returns IPv6 addresses with a trailing new line.

Misc/NEWS.d/next/Library/2019-11-16-16-09-07.bpo-38820.ivhUSV.rst

@@ -0,0 +1,4 @@
+The header files for :mod:`ssl` error codes are now OpenSSL
+version-specific. Exceptions will now show correct reason and library
+codes. The ``make_ssl_data.py`` script has been rewritten to use OpenSSL's
+text file with error codes.

Misc/NEWS.d/next/Library/2021-04-09-12-08-01.bpo-43788.YsvInM.rst

@@ -0,0 +1,2 @@
+OpenSSL 3.0.0: Don't call the password callback function a second time when
+first call has signaled an error condition.

Misc/NEWS.d/next/Library/2021-04-09-14-08-03.bpo-43789.eaHlAm.rst

@@ -0,0 +1 @@
+Add :data:`ssl.OP_IGNORE_UNEXPECTED_EOF` constants (OpenSSL 3.0.0)

Misc/NEWS.d/next/Library/2021-04-09-16-14-22.bpo-43794.-1XPDH.rst

@@ -0,0 +1,2 @@
+OpenSSL 3.0.0: :meth:`~ssl.SSLContext.load_verify_locations` now returns a
+consistent error message when cadata contains no valid certificate.

Misc/NEWS.d/next/Library/2021-04-23-11-54-38.bpo-43920.cJMQ2D.rst

@@ -0,0 +1,2 @@
+OpenSSL 3.0.0: Disable testing of legacy protocols TLS 1.0 and 1.1. Tests
+are failing with TLSV1_ALERT_INTERNAL_ERROR.

Misc/NEWS.d/next/Tests/2021-04-09-15-10-38.bpo-43791.4KxiXK.rst

@@ -0,0 +1 @@
+Fix test case for OpenSSL 3.0.1 version. OpenSSL 3.0 uses ``0xMNN00PP0L``.

Misc/NEWS.d/next/Tests/2021-12-17-14-46-19.bpo-46114.9iyZ_9.rst

@@ -364,7 +364,7 @@ EVP_tp_init(EVPobject *self, PyObject *args, PyObject *kwds)
         return -1;
     }
 
-    digest = EVP_get_digestbyname(nameStr);
+    digest = EVP_MD_fetch(NULL, nameStr, NULL);
     if (!digest) {
         PyErr_SetString(PyExc_ValueError, "unknown hash function");
         if (data_obj)
@@ -529,7 +529,7 @@ EVP_new(PyObject *self, PyObject *args, PyObject *kwdict)
     if (data_obj)
         GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view);
 
-    digest = EVP_get_digestbyname(name);
+    digest = EVP_MD_fetch(NULL, name, NULL);
 
     ret_obj = EVPnew(name_obj, digest, NULL, (unsigned char*)view.buf, view.len);
 
@@ -654,7 +654,7 @@ pbkdf2_hmac(PyObject *self, PyObject *args, PyObject *kwdict)
         return NULL;
     }
 
-    digest = EVP_get_digestbyname(name);
+    digest = EVP_MD_fetch(NULL, name, NULL);
     if (digest == NULL) {
         PyErr_SetString(PyExc_ValueError, "unsupported hash type");
         goto end;
@@ -941,8 +941,8 @@ generate_hash_name_list(void)
      \
         if (CONST_new_ ## NAME ## _ctx_p == NULL) { \
             EVP_MD_CTX *ctx_p = EVP_MD_CTX_new(); \
-            if (!EVP_get_digestbyname(#NAME) || \
-                !EVP_DigestInit(ctx_p, EVP_get_digestbyname(#NAME))) { \
+            if (!EVP_MD_fetch(NULL, #NAME, NULL) || \
+                !EVP_DigestInit(ctx_p, EVP_MD_fetch(NULL, #NAME, NULL))) { \
                 _setException(PyExc_ValueError); \
                 EVP_MD_CTX_free(ctx_p); \
                 return NULL; \

Modules/_hashopenssl.c

@@ -103,7 +103,13 @@ struct py_ssl_library_code {
 };
 
 /* Include generated data (error codes) */
+#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
+#include "_ssl_data_300.h"
+#elif (OPENSSL_VERSION_NUMBER >= 0x10101000L) && !defined(LIBRESSL_VERSION_NUMBER)
+#include "_ssl_data_111.h"
+#else
 #include "_ssl_data.h"
+#endif
 
 #if (OPENSSL_VERSION_NUMBER >= 0x10100000L) && !defined(LIBRESSL_VERSION_NUMBER)
 #  define OPENSSL_VERSION_1_1 1
@@ -1204,14 +1210,61 @@ _get_peer_alt_names (X509 *certificate) {
                 PyTuple_SET_ITEM(t, 1, v);
                 break;
 
+            case GEN_IPADD:
+                /* OpenSSL < 3.0.0 adds a trailing \n to IPv6. 3.0.0 removed
+                 * the trailing newline. Remove it in all versions
+                 */
+                t = PyTuple_New(2);
+                if (t == NULL)
+                    goto fail;
+
+                v = PyUnicode_FromString("IP Address");
+                if (v == NULL) {
+                    Py_DECREF(t);
+                    goto fail;
+                }
+                PyTuple_SET_ITEM(t, 0, v);
+
+                if (name->d.ip->length == 4) {
+                    unsigned char *p = name->d.ip->data;
+                    v = PyUnicode_FromFormat(
+                        "%d.%d.%d.%d",
+                        p[0], p[1], p[2], p[3]
+                    );
+                } else if (name->d.ip->length == 16) {
+                    /* PyUnicode_FromFormat() does not support %X */
+                    unsigned char *p = name->d.ip->data;
+                    len = sprintf(
+                        buf,
+                        "%X:%X:%X:%X:%X:%X:%X:%X",
+                        p[0] << 8 | p[1],
+                        p[2] << 8 | p[3],
+                        p[4] << 8 | p[5],
+                        p[6] << 8 | p[7],
+                        p[8] << 8 | p[9],
+                        p[10] << 8 | p[11],
+                        p[12] << 8 | p[13],
+                        p[14] << 8 | p[15]
+                    );
+                    v = PyUnicode_FromStringAndSize(buf, len);
+                } else {
+                    v = PyUnicode_FromString("<invalid>");
+                }
+
+                if (v == NULL) {
+                    Py_DECREF(t);
+                    goto fail;
+                }
+                PyTuple_SET_ITEM(t, 1, v);
+                break;
+
             default:
                 /* for everything else, we use the OpenSSL print form */
                 switch (gntype) {
                     /* check for new general name type */
                     case GEN_OTHERNAME:
                     case GEN_X400:
                     case GEN_EDIPARTY:
-                    case GEN_IPADD:
                     case GEN_RID:
                         break;
                     default:
@@ -2880,6 +2933,10 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version)
 #endif
 #ifdef SSL_OP_SINGLE_ECDH_USE
     options |= SSL_OP_SINGLE_ECDH_USE;
+#endif
+#ifdef SSL_OP_IGNORE_UNEXPECTED_EOF
+    /* Make OpenSSL 3.0.0 behave like 1.1.1 */
+    options |= SSL_OP_IGNORE_UNEXPECTED_EOF;
 #endif
     SSL_CTX_set_options(self->ctx, options);
 
@@ -3441,6 +3498,13 @@ _password_callback(char *buf, int size, int rwflag, void *userdata)
 
     PySSL_END_ALLOW_THREADS_S(pw_info->thread_state);
 
+    if (pw_info->error) {
+        /* already failed previously. OpenSSL 3.0.0-alpha14 invokes the
+         * callback multiple times which can lead to fatal Python error in
+         * exception check. */
+        goto error;
+    }
+
     if (pw_info->callable) {
         fn_ret = PyObject_CallFunctionObjArgs(pw_info->callable, NULL);
         if (!fn_ret) {
@@ -3584,7 +3648,7 @@ _add_ca_certs(PySSLContext *self, void *data, Py_ssize_t len,
 {
     BIO *biobuf = NULL;
     X509_STORE *store;
-    int retval = 0, err, loaded = 0;
+    int retval = -1, err, loaded = 0;
 
     assert(filetype == SSL_FILETYPE_ASN1 || filetype == SSL_FILETYPE_PEM);
 
@@ -3638,23 +3702,32 @@ _add_ca_certs(PySSLContext *self, void *data, Py_ssize_t len,
     }
 
     err = ERR_peek_last_error();
-    if ((filetype == SSL_FILETYPE_ASN1) &&
-            (loaded > 0) &&
-            (ERR_GET_LIB(err) == ERR_LIB_ASN1) &&
-            (ERR_GET_REASON(err) == ASN1_R_HEADER_TOO_LONG)) {
+    if (loaded == 0) {
+        const char *msg = NULL;
+        if (filetype == SSL_FILETYPE_PEM) {
+            msg = "no start line: cadata does not contain a certificate";
+        } else {
+            msg = "not enough data: cadata does not contain a certificate";
+        }
+        _setSSLError(msg, 0, __FILE__, __LINE__);
+        retval = -1;
+    } else if ((filetype == SSL_FILETYPE_ASN1) &&
+                    (ERR_GET_LIB(err) == ERR_LIB_ASN1) &&
+                    (ERR_GET_REASON(err) == ASN1_R_HEADER_TOO_LONG)) {
         /* EOF ASN1 file, not an error */
         ERR_clear_error();
         retval = 0;
     } else if ((filetype == SSL_FILETYPE_PEM) &&
-                   (loaded > 0) &&
                    (ERR_GET_LIB(err) == ERR_LIB_PEM) &&
                    (ERR_GET_REASON(err) == PEM_R_NO_START_LINE)) {
         /* EOF PEM file, not an error */
         ERR_clear_error();
         retval = 0;
-    } else {
+    } else if (err != 0) {
         _setSSLError(NULL, 0, __FILE__, __LINE__);
         retval = -1;
+    } else {
+        retval = 0;
     }
 
     BIO_free(biobuf);
@@ -5637,6 +5710,10 @@ PyInit__ssl(void)
     PyModule_AddIntConstant(m, "OP_ENABLE_MIDDLEBOX_COMPAT",
                             SSL_OP_ENABLE_MIDDLEBOX_COMPAT);
 #endif
+#ifdef SSL_OP_IGNORE_UNEXPECTED_EOF
+    PyModule_AddIntConstant(m, "OP_IGNORE_UNEXPECTED_EOF",
+                            SSL_OP_IGNORE_UNEXPECTED_EOF);
+#endif
 
 #if HAVE_SNI
     r = Py_True;