SNOW-201003 vendoring requests and urllib3 (snowflakedb#557)

Author: sfc-gh-mkeller

Diff for: .pre-commit-config.yaml

@@ -1,4 +1,4 @@
-exclude: '^(.*egg.info.*|.*/parameters.py).*$'
+exclude: '^(.*egg.info.*|.*/parameters.py|src/snowflake/connector/vendored/.*/).*$'
 repos:
 -   repo: https://github.com/asottile/seed-isort-config
     rev: v1.9.1

Diff for: MANIFEST.in

@@ -1,5 +1,6 @@
 include *.rst
 recursive-include src/snowflake/connector *.py *.pyx
 include LICENSE.txt
+include NOTICE
 recursive-include src/snowflake/connector/cpp *.cpp *.hpp
 prune test

Diff for: NOTICE

@@ -0,0 +1,8 @@
+Snowflake Python Connector
+Copyright 2020 Snowflake Inc.
+
+This software includes software derived from urllib3, licensed under the MIT license (https://urllib3.readthedocs.io).
+Copyright (c) 2008-2020 Andrey Petrov and contributors
+
+This software includes software derived from Requests: HTTP For Humans, licensed under the Apache license, developed by the Python Software Foundation (https://requests.readthedocs.io/)
+Requests Copyright 2019 Kenneth Reitz

Diff for: setup.py

@@ -195,8 +195,8 @@ def _get_arrow_lib_as_linker_input(self):
         'azure-common<2.0.0',
         'azure-storage-blob>=12.0.0,<13.0.0',
         'boto3>=1.4.4,<2.0.0',
-        'requests<2.24.0',
-        'urllib3>=1.20,<1.26.0',
+        'requests<3.0.0',
+        'urllib3<2.0.0',
         'certifi<2021.0.0',
         'pytz<2021.0',
         'pycryptodomex>=3.2,!=3.5.0,<4.0.0',
@@ -210,16 +210,29 @@ def _get_arrow_lib_as_linker_input(self):
         # required. Python 3.6 was released at the end of 2016. setuptools 34.0.0 was released
         # in early 2017, so we pick this version as a reasonably modern base.
         'setuptools>34.0.0',
+        # requests requirements
+        'chardet>=3.0.2,<4',
+        'idna>=2.5,<3',
+        'certifi>=2017.4.17',
     ],
 
     namespace_packages=['snowflake'],
     packages=[
         'snowflake.connector',
         'snowflake.connector.tool',
+
+        'snowflake.connector.vendored',
+        'snowflake.connector.vendored.requests',
+        'snowflake.connector.vendored.urllib3',
+        'snowflake.connector.vendored.urllib3.packages',
+        'snowflake.connector.vendored.urllib3.packages.ssl_match_hostname',
+        'snowflake.connector.vendored.urllib3.packages.backports',
+        'snowflake.connector.vendored.urllib3.contrib',
+        'snowflake.connector.vendored.urllib3.contrib._securetransport',
+        'snowflake.connector.vendored.urllib3.util',
     ],
     package_dir={
-        'snowflake.connector': os.path.join('src', 'snowflake', 'connector'),
-        'snowflake.connector.tool': os.path.join('src', 'snowflake', 'connector', 'tool'),
+        '': 'src',
     },
     package_data={
         'snowflake.connector': ['*.pem', '*.json', '*.rst', 'LICENSE.txt'],

Diff for: src/snowflake/connector/gcs_util.py

@@ -10,11 +10,10 @@
 from logging import getLogger
 from typing import Any, Dict
 
-import requests
-
 from .compat import quote
 from .constants import HTTP_HEADER_CONTENT_ENCODING, SHA256_DIGEST, FileHeader, ResultStatus
 from .encryption_util import EncryptionMetadata
+from .vendored import requests
 
 logger = getLogger(__name__)
 

Diff for: src/snowflake/connector/gzip_decoder.py

@@ -11,7 +11,7 @@
 from typing import IO
 
 CHUNK_SIZE = 16384
-MAGIC_NUMBER = 16  # magic number from requests/packages/urllib3/response.py
+MAGIC_NUMBER = 16  # magic number from .vendored.requests/packages/urllib3/response.py
 
 logger = getLogger(__name__)
 

Diff for: src/snowflake/connector/network.py

@@ -17,13 +17,6 @@
 from threading import Lock
 
 import OpenSSL.SSL
-import requests
-from requests.adapters import HTTPAdapter
-from requests.auth import AuthBase
-from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout, SSLError
-from requests.packages.urllib3.exceptions import ProtocolError, ReadTimeoutError
-
-from snowflake.connector.time_util import get_time_millis
 
 from . import ssl_wrap_socket
 from .compat import (
@@ -81,8 +74,13 @@
     SQLSTATE_IO_ERROR,
 )
 from .telemetry_oob import TelemetryService
-from .time_util import DEFAULT_MASTER_VALIDITY_IN_SECONDS, DecorrelateJitterBackoff
+from .time_util import DEFAULT_MASTER_VALIDITY_IN_SECONDS, DecorrelateJitterBackoff, get_time_millis
 from .tool.probe_connection import probe_connection
+from .vendored import requests
+from .vendored.requests.adapters import HTTPAdapter
+from .vendored.requests.auth import AuthBase
+from .vendored.requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout, SSLError
+from .vendored.urllib3.exceptions import ProtocolError, ReadTimeoutError
 
 logger = logging.getLogger(__name__)
 

Diff for: src/snowflake/connector/ssl_wrap_socket.py

@@ -16,16 +16,17 @@
 from functools import wraps
 from inspect import getfullargspec as get_args
 from socket import socket
+from typing import Optional
 
 import certifi
 import OpenSSL.SSL
-import requests.packages.urllib3.connection as connection_
-import requests.packages.urllib3.util.ssl_ as ssl_
 from urllib3.contrib.pyopenssl import PyOpenSSLContext
 
 from .constants import OCSPMode
 from .errorcode import ER_OCSP_RESPONSE_CERT_STATUS_REVOKED
 from .errors import OperationalError
+from .vendored.urllib3 import connection as connection_
+from .vendored.urllib3.util import ssl_ as ssl_
 
 FEATURE_OCSP_MODE = OCSPMode.FAIL_OPEN
 
@@ -98,7 +99,10 @@ def ssl_wrap_socket_with_ocsp(*args, **kwargs):
     return ret
 
 
-def _openssl_connect(hostname, port=443, max_retry=20):
+def _openssl_connect(hostname: str,
+                     port: int = 443,
+                     max_retry: int = 20,
+                     timeout: Optional[int] = None) -> OpenSSL.SSL.Connection:
     """The OpenSSL connection without validating certificates.
 
     This is used to diagnose SSL issues.
@@ -108,10 +112,11 @@ def _openssl_connect(hostname, port=443, max_retry=20):
     for _ in range(max_retry):
         try:
             client = socket()
-            # client.settimeout(5)
             client.connect((hostname, port))
-            client_ssl = OpenSSL.SSL.Connection(
-                OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD), client)
+            context = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
+            if timeout is not None:
+                context.set_timeout(timeout)
+            client_ssl = OpenSSL.SSL.Connection(context, client)
             client_ssl.set_connect_state()
             client_ssl.set_tlsext_host_name(hostname.encode('utf-8'))
             client_ssl.do_handshake()

Diff for: src/snowflake/connector/telemetry_oob.py

@@ -12,12 +12,11 @@
 from queue import Queue
 from typing import Optional
 
-import requests
-
 from .compat import OK
 from .description import CLIENT_NAME, SNOWFLAKE_CONNECTOR_VERSION
 from .secret_detector import SecretDetector
 from .test_util import ENABLE_TELEMETRY_LOG, rt_plain_logger
+from .vendored import requests
 
 logger = logging.getLogger(__name__)
 

Diff for: src/snowflake/connector/vendored/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2012-2020 Snowflake Computing Inc. All right reserved.
+#

Diff for: src/snowflake/connector/vendored/requests/__init__.py

@@ -0,0 +1,130 @@
+# -*- coding: utf-8 -*-
+
+#   __
+#  /__)  _  _     _   _ _/   _
+# / (   (- (/ (/ (- _)  /  _)
+#          /
+
+"""
+Requests HTTP Library
+~~~~~~~~~~~~~~~~~~~~~
+
+Requests is an HTTP library, written in Python, for human beings.
+Basic GET usage:
+
+   >>> import requests
+   >>> r = requests.get('https://www.python.org')
+   >>> r.status_code
+   200
+   >>> b'Python is a programming language' in r.content
+   True
+
+... or POST:
+
+   >>> payload = dict(key1='value1', key2='value2')
+   >>> r = requests.post('https://httpbin.org/post', data=payload)
+   >>> print(r.text)
+   {
+     ...
+     "form": {
+       "key1": "value1",
+       "key2": "value2"
+     },
+     ...
+   }
+
+The other HTTP methods are supported - see `requests.api`. Full documentation
+is at <https://requests.readthedocs.io>.
+
+:copyright: (c) 2017 by Kenneth Reitz.
+:license: Apache 2.0, see LICENSE for more details.
+"""
+
+from .. import urllib3
+import chardet
+import warnings
+from .exceptions import RequestsDependencyWarning
+
+
+def check_compatibility(urllib3_version, chardet_version):
+    urllib3_version = urllib3_version.split('.')
+    assert urllib3_version != ['dev']  # Verify urllib3 isn't installed from git.
+
+    # Sometimes, urllib3 only reports its version as 16.1.
+    if len(urllib3_version) == 2:
+        urllib3_version.append('0')
+
+    # Check urllib3 for compatibility.
+    major, minor, patch = urllib3_version  # noqa: F811
+    major, minor, patch = int(major), int(minor), int(patch)
+    # urllib3 >= 1.21.1, <= 1.25
+    assert major == 1
+    assert minor >= 21
+    assert minor <= 25
+
+    # Check chardet for compatibility.
+    major, minor, patch = chardet_version.split('.')[:3]
+    major, minor, patch = int(major), int(minor), int(patch)
+    # chardet >= 3.0.2, < 3.1.0
+    assert major == 3
+    assert minor < 1
+    assert patch >= 2
+
+
+def _check_cryptography(cryptography_version):
+    # cryptography < 1.3.4
+    try:
+        cryptography_version = list(map(int, cryptography_version.split('.')))
+    except ValueError:
+        return
+
+    if cryptography_version < [1, 3, 4]:
+        warning = 'Old version of cryptography ({}) may cause slowdown.'.format(cryptography_version)
+        warnings.warn(warning, RequestsDependencyWarning)
+
+# Check imported dependencies for compatibility.
+try:
+    check_compatibility(urllib3.__version__, chardet.__version__)
+except (AssertionError, ValueError):
+    warnings.warn("urllib3 ({}) or chardet ({}) doesn't match a supported "
+                  "version!".format(urllib3.__version__, chardet.__version__),
+                  RequestsDependencyWarning)
+
+# Attempt to enable urllib3's SNI support, if possible
+try:
+    from ..urllib3.contrib import pyopenssl
+    pyopenssl.inject_into_urllib3()
+
+    # Check cryptography version
+    from cryptography import __version__ as cryptography_version
+    _check_cryptography(cryptography_version)
+except ImportError:
+    pass
+
+# urllib3's DependencyWarnings should be silenced.
+from ..urllib3.exceptions import DependencyWarning
+warnings.simplefilter('ignore', DependencyWarning)
+
+from .__version__ import __title__, __description__, __url__, __version__
+from .__version__ import __build__, __author__, __author_email__, __license__
+from .__version__ import __copyright__, __cake__
+
+from . import utils
+from .models import Request, Response, PreparedRequest
+from .api import request, get, head, post, patch, put, delete, options
+from .sessions import session, Session
+from .status_codes import codes
+from .exceptions import (
+    RequestException, Timeout, URLRequired,
+    TooManyRedirects, HTTPError, ConnectionError,
+    FileModeWarning, ConnectTimeout, ReadTimeout
+)
+
+# Set default logging handler to avoid "No handler found" warnings.
+import logging
+from logging import NullHandler
+
+logging.getLogger(__name__).addHandler(NullHandler())
+
+# FileModeWarnings go off per the default.
+warnings.simplefilter('default', FileModeWarning, append=True)

Diff for: src/snowflake/connector/vendored/requests/__version__.py

@@ -0,0 +1,14 @@
+# .-. .-. .-. . . .-. .-. .-. .-.
+# |(  |-  |.| | | |-  `-.  |  `-.
+# ' ' `-' `-`.`-' `-' `-'  '  `-'
+
+__title__ = 'requests'
+__description__ = 'Python HTTP for Humans.'
+__url__ = 'https://requests.readthedocs.io'
+__version__ = '2.23.0'
+__build__ = 0x022300
+__author__ = 'Kenneth Reitz'
+__author_email__ = '[email protected]'
+__license__ = 'Apache 2.0'
+__copyright__ = 'Copyright 2020 Kenneth Reitz'
+__cake__ = u'\u2728 \U0001f370 \u2728'

Diff for: src/snowflake/connector/vendored/requests/_internal_utils.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+
+"""
+requests._internal_utils
+~~~~~~~~~~~~~~
+
+Provides utility functions that are consumed internally by Requests
+which depend on extremely few external helpers (such as compat)
+"""
+
+from .compat import is_py2, builtin_str, str
+
+
+def to_native_string(string, encoding='ascii'):
+    """Given a string object, regardless of type, returns a representation of
+    that string in the native string type, encoding and decoding where
+    necessary. This assumes ASCII unless told otherwise.
+    """
+    if isinstance(string, builtin_str):
+        out = string
+    else:
+        if is_py2:
+            out = string.encode(encoding)
+        else:
+            out = string.decode(encoding)
+
+    return out
+
+
+def unicode_is_ascii(u_string):
+    """Determine if unicode string only contains ASCII characters.
+
+    :param str u_string: unicode string to check. Must be unicode
+        and not Python 2 `str`.
+    :rtype: bool
+    """
+    assert isinstance(u_string, str)
+    try:
+        u_string.encode('ascii')
+        return True
+    except UnicodeEncodeError:
+        return False