Creating client for pipeline

sdk/Table/azure/azure_table/__init__.py

@@ -0,0 +1,5 @@
+__all__ = [
+    'generate_account_sas',
+]
+
+from azure.table import generate_account_sas

sdk/Table/azure/azure_table/_generated/version.py

@@ -0,0 +1,13 @@
+# coding=utf-8
+# --------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+#
+# Code generated by Microsoft (R) AutoRest Code Generator.
+# Changes may cause incorrect behavior and will be lost if the code is
+# regenerated.
+# --------------------------------------------------------------------------
+
+VERSION = "2019-07-07"
+

sdk/Table/azure/storage/tables/_shared/_common_conversion.py → sdk/Table/azure/azure_table/_shared/_common_conversion.py

@@ -11,6 +11,7 @@
 from io import (SEEK_SET)
 
 from dateutil.tz import tzutc
+from pyparsing import unicode
 
 from ._error import (
     _ERROR_VALUE_SHOULD_BE_BYTES_OR_STREAM,

sdk/Table/azure/azure_table/_shared/_constants.py

@@ -0,0 +1,51 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+import platform
+import sys
+
+__author__ = 'Microsoft Corp. <[email protected]>'
+__version__ = '1.4.2'
+
+# UserAgent string sample: 'Azure-Storage/0.37.0-0.38.0 (Python CPython 3.4.2; Windows 8)'
+# First version(0.37.0) is the common package, and the second version(0.38.0) is the service package
+USER_AGENT_STRING_PREFIX = 'Azure-Storage/{}-'.format(__version__)
+USER_AGENT_STRING_SUFFIX = '(Python {} {}; {} {})'.format(platform.python_implementation(),
+                                                          platform.python_version(), platform.system(),
+                                                          platform.release())
+
+# default values for common package, in case it is used directly
+DEFAULT_X_MS_VERSION = '2018-03-28'
+DEFAULT_USER_AGENT_STRING = '{}None {}'.format(USER_AGENT_STRING_PREFIX, USER_AGENT_STRING_SUFFIX)
+
+# Live ServiceClient URLs
+SERVICE_HOST_BASE = 'core.windows.net'
+DEFAULT_PROTOCOL = 'https'
+
+# Development ServiceClient URLs
+DEV_BLOB_HOST = '127.0.0.1:10000'
+DEV_QUEUE_HOST = '127.0.0.1:10001'
+
+# Default credentials for Development Storage Service
+DEV_ACCOUNT_NAME = 'devstoreaccount1'
+DEV_ACCOUNT_SECONDARY_NAME = 'devstoreaccount1-secondary'
+DEV_ACCOUNT_KEY = 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=='
+
+# Socket timeout in seconds
+DEFAULT_SOCKET_TIMEOUT = 20
+
+# for python 3.5+, there was a change to the definition of the socket timeout (as far as socket.sendall is concerned)
+# The socket timeout is now the maximum total duration to send all data.
+if sys.version_info >= (3, 5):
+    # the timeout to connect is 20 seconds, and the read timeout is 2000 seconds
+    # the 2000 seconds was calculated with: 100MB (max block size)/ 50KB/s (an arbitrarily chosen minimum upload speed)
+    DEFAULT_SOCKET_TIMEOUT = (20, 2000)
+
+# Encryption constants
+_ENCRYPTION_PROTOCOL_V1 = '1.0'
+
+_AUTHORIZATION_HEADER_NAME = 'Authorization'
+_COPY_SOURCE_HEADER_NAME = 'x-ms-copy-source'
+_REDACTED_VALUE = 'REDACTED'

sdk/Table/azure/storage/tables/_shared/_error.py → sdk/Table/azure/azure_table/_shared/_error.py

@@ -5,6 +5,8 @@
 # --------------------------------------------------------------------------
 from sys import version_info
 
+from pyparsing import unicode
+
 if version_info < (3,):
     def _str(value):
         if isinstance(value, unicode):

sdk/Table/azure/storage/tables/_shared/authentication.py → sdk/Table/azure/azure_table/_shared/authentication.py

@@ -8,10 +8,10 @@
 import sys
 
 try:
-    from urllib.parse import urlparse, unquote
+    from urllib.parse import urlparse, unquote, parse_qsl
 except ImportError:
-    from urlparse import urlparse # type: ignore
-    from urllib2 import unquote # type: ignore
+    from urlparse import urlparse  # type: ignore
+    from urllib2 import unquote  # type: ignore
 
 try:
     from yarl import URL
@@ -26,11 +26,21 @@
 from azure.core.exceptions import ClientAuthenticationError
 from azure.core.pipeline.policies import SansIOHTTPPolicy
 
-from . import sign_string
+from ._common_conversion import (
+    _sign_string,
+)
 
+from azure.table import (
+    DEV_ACCOUNT_NAME,
+    DEV_ACCOUNT_SECONDARY_NAME
+)
 
-logger = logging.getLogger(__name__)
+from ._error import (
+    AzureSigningError,
+    _wrap_exception,
+)
 
+logger = logging.getLogger(__name__)
 
 
 # wraps a given exception with the desired exception type
@@ -59,35 +69,36 @@ class AzureSigningError(ClientAuthenticationError):
 # pylint: disable=no-self-use
 class SharedKeyCredentialPolicy(SansIOHTTPPolicy):
 
-    def __init__(self, account_name, account_key):
+    def __init__(self, account_name, account_key, is_emulated=False):
         self.account_name = account_name
         self.account_key = account_key
-        super(SharedKeyCredentialPolicy, self).__init__()
+        self.is_emulated = is_emulated
 
     def _get_headers(self, request, headers_to_sign):
-        headers = dict((name.lower(), value) for name, value in request.http_request.headers.items() if value)
+        headers = dict((name.lower(), value) for name, value in request.headers.items() if value)
         if 'content-length' in headers and headers['content-length'] == '0':
             del headers['content-length']
         return '\n'.join(headers.get(x, '') for x in headers_to_sign) + '\n'
 
     def _get_verb(self, request):
-        return request.http_request.method + '\n'
+        return request.method + '\n'
 
     def _get_canonicalized_resource(self, request):
-        uri_path = urlparse(request.http_request.url).path
-        try:
-            if isinstance(request.context.transport, AioHttpTransport) or \
-                isinstance(getattr(request.context.transport, "_transport", None), AioHttpTransport):
-                uri_path = URL(uri_path)
-                return '/' + self.account_name + str(uri_path)
-        except TypeError:
-            pass
+        #uri_path = request.path.split('?')[0]
+        uri_path = urlparse(request.url).path
+
+        # for emulator, use the DEV_ACCOUNT_NAME instead of DEV_ACCOUNT_SECONDARY_NAME
+        # as this is how the emulator works
+        if self.is_emulated and uri_path.find(DEV_ACCOUNT_SECONDARY_NAME) == 1:
+            # only replace the first instance
+            uri_path = uri_path.replace(DEV_ACCOUNT_SECONDARY_NAME, DEV_ACCOUNT_NAME, 1)
+
         return '/' + self.account_name + uri_path
 
     def _get_canonicalized_headers(self, request):
         string_to_sign = ''
         x_ms_headers = []
-        for name, value in request.http_request.headers.items():
+        for name, value in request.headers.items():
             if name.startswith('x-ms-'):
                 x_ms_headers.append((name.lower(), value))
         x_ms_headers.sort()
@@ -96,41 +107,39 @@ def _get_canonicalized_headers(self, request):
                 string_to_sign += ''.join([name, ':', value, '\n'])
         return string_to_sign
 
-    def _get_canonicalized_resource_query(self, request):
-        sorted_queries = [(name, value) for name, value in request.http_request.query.items()]
-        sorted_queries.sort()
-
-        string_to_sign = ''
-        for name, value in sorted_queries:
-            if value is not None:
-                string_to_sign += '\n' + name.lower() + ':' + unquote(value)
-
-        return string_to_sign
-
     def _add_authorization_header(self, request, string_to_sign):
         try:
-            signature = sign_string(self.account_key, string_to_sign)
+            signature = _sign_string(self.account_key, string_to_sign)
             auth_string = 'SharedKey ' + self.account_name + ':' + signature
-            request.http_request.headers['Authorization'] = auth_string
+            request.headers['Authorization'] = auth_string
         except Exception as ex:
             # Wrap any error that occurred as signing error
             # Doing so will clarify/locate the source of problem
             raise _wrap_exception(ex, AzureSigningError)
 
-    def on_request(self, request):
+    def on_request(self, request):  # type: (PipelineRequest) -> Union[None, Awaitable[None]]
+        self.sign_request(request.http_request)
+
+    def sign_request(self, request):
         string_to_sign = \
             self._get_verb(request) + \
             self._get_headers(
                 request,
-                [
-                    'content-encoding', 'content-language', 'content-length',
-                    'content-md5', 'content-type', 'date', 'if-modified-since',
-                    'if-match', 'if-none-match', 'if-unmodified-since', 'byte_range'
-                ]
+                ['content-md5', 'content-type', 'x-ms-date'],
             ) + \
-            self._get_canonicalized_headers(request) + \
             self._get_canonicalized_resource(request) + \
             self._get_canonicalized_resource_query(request)
 
         self._add_authorization_header(request, string_to_sign)
-        #logger.debug("String_to_sign=%s", string_to_sign)
+        logger.debug("String_to_sign=%s", string_to_sign)
+
+    def _get_canonicalized_resource_query(self, request):
+        sorted_queries = [(name, value) for name, value in request.query.items()]
+        sorted_queries.sort()
+
+        string_to_sign = ''
+        for name, value in sorted_queries:
+            if value is not None:
+                string_to_sign += '\n' + name.lower() + ':' + value
+
+        return string_to_sign

sdk/Table/azure/storage/tables/_shared/base_client.py → sdk/Table/azure/azure_table/_shared/base_client.py

@@ -50,11 +50,10 @@
     StorageResponseHook,
     StorageLoggingPolicy,
     StorageHosts,
-    QueueMessagePolicy,
     ExponentialRetry,
 )
 from .._version import VERSION
-from .._generated.models import StorageErrorException
+# from .._generated.models import StorageErrorException
 from .response_handlers import process_storage_error, PartialBatchErrorException
 
 
@@ -63,6 +62,7 @@
     "blob": {"primary": "BlobEndpoint", "secondary": "BlobSecondaryEndpoint"},
     "queue": {"primary": "QueueEndpoint", "secondary": "QueueSecondaryEndpoint"},
     "file": {"primary": "FileEndpoint", "secondary": "FileSecondaryEndpoint"},
+    "table": {"primary": "TableEndpoint", "secondary": "TableSecondaryEndpoint"},
     "dfs": {"primary": "BlobEndpoint", "secondary": "BlobEndpoint"},
 }
 
@@ -80,7 +80,7 @@ def __init__(
         self._hosts = kwargs.get("_hosts")
         self.scheme = parsed_url.scheme
 
-        if service not in ["blob", "queue", "file-share", "dfs"]:
+        if service not in ["blob", "queue", "file-share", "dfs", "table"]:
             raise ValueError("Invalid service: {}".format(service))
         service_name = service.split('-')[0]
         account = parsed_url.netloc.split(".{}.core.".format(service_name))
@@ -230,19 +230,17 @@ def _create_pipeline(self, credential, **kwargs):
         if not config.transport:
             config.transport = RequestsTransport(**kwargs)
         policies = [
-            QueueMessagePolicy(),
             config.headers_policy,
             config.proxy_policy,
             config.user_agent_policy,
-            StorageContentValidation(),
-            StorageRequestHook(**kwargs),
+            # StorageRequestHook(**kwargs),
             self._credential_policy,
             ContentDecodePolicy(response_encoding="utf-8"),
             RedirectPolicy(**kwargs),
-            StorageHosts(hosts=self._hosts, **kwargs),
+            # StorageHosts(hosts=self._hosts, **kwargs),
             config.retry_policy,
             config.logging_policy,
-            StorageResponseHook(**kwargs),
+            # StorageResponseHook(**kwargs),
             DistributedTracingPolicy(**kwargs),
             HttpLoggingPolicy(**kwargs)
         ]
@@ -291,7 +289,7 @@ def _batch_send(
                     raise error
                 return iter(parts)
             return parts
-        except StorageErrorException as error:
+        except HttpResponseError as error:
             process_storage_error(error)
 
 class TransportWrapper(HttpTransport):
@@ -328,7 +326,9 @@ def format_shared_key_credential(account, credential):
             raise ValueError("Shared key credential missing 'account_name")
         if "account_key" not in credential:
             raise ValueError("Shared key credential missing 'account_key")
+        print('SharedKey ', credential)
         return SharedKeyCredentialPolicy(**credential)
+    print(credential)
     return credential
 
 
@@ -386,6 +386,7 @@ def create_configuration(**kwargs):
     config.logging_policy = StorageLoggingPolicy(**kwargs)
     config.proxy_policy = ProxyPolicy(**kwargs)
 
+# all can be ignored
     # Storage settings
     config.max_single_put_size = kwargs.get("max_single_put_size", 64 * 1024 * 1024)
     config.copy_polling_interval = 15

sdk/Table/azure/azure_table/_shared/constants.py

@@ -0,0 +1,26 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+import sys
+from .._generated.version import VERSION
+
+
+X_MS_VERSION = VERSION
+
+# Socket timeout in seconds
+CONNECTION_TIMEOUT = 20
+READ_TIMEOUT = 20
+
+# for python 3.5+, there was a change to the definition of the socket timeout (as far as socket.sendall is concerned)
+# The socket timeout is now the maximum total duration to send all data.
+if sys.version_info >= (3, 5):
+    # the timeout to connect is 20 seconds, and the read timeout is 2000 seconds
+    # the 2000 seconds was calculated with: 100MB (max block size)/ 50KB/s (an arbitrarily chosen minimum upload speed)
+    READ_TIMEOUT = 2000
+
+STORAGE_OAUTH_SCOPE = "https://storage.azure.com/.default"
+
+SERVICE_HOST_BASE = 'core.windows.net'

sdk/Table/azure/storage/tables/_shared/models.py → sdk/Table/azure/azure_table/_shared/models.py

@@ -3,9 +3,20 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
-
+import sys
 from enum import Enum
 
+from pyparsing import unicode
+
+if sys.version_info < (3,):
+    from collections import Iterable
+
+    _unicode_type = unicode
+else:
+    from collections.abc import Iterable
+
+    _unicode_type = str
+
 
 def get_enum_value(value):
     if value is None or value in ["None", ""]: