[Perfstress][Storage] Added Blobs perf tests

sdk/storage/azure-storage-blob/tests/perfstress_tests/T1_legacy_tests/_test_base_legacy.py

@@ -0,0 +1,51 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import os
+import uuid
+
+from azure_devtools.perfstress_tests import PerfStressTest
+
+from azure.storage.blob import BlockBlobService
+
+class _LegacyServiceTest(PerfStressTest):
+    service_client = None
+    async_service_client = None
+
+    def __init__(self, arguments):
+        super().__init__(arguments)
+        connection_string = self.get_from_env("AZURE_STORAGE_CONNECTION_STRING")
+        if not _LegacyServiceTest.service_client or self.args.no_client_share:
+            _LegacyServiceTest.service_client = BlockBlobService(connection_string=connection_string)
+            _LegacyServiceTest.service_client.MAX_SINGLE_PUT_SIZE = self.args.max_put_size
+            _LegacyServiceTest.service_client.MAX_BLOCK_SIZE = self.args.max_block_size
+            _LegacyServiceTest.service_client.MIN_LARGE_BLOCK_UPLOAD_THRESHOLD = self.args.buffer_threshold
+        self.async_service_client = None
+        self.service_client = _LegacyServiceTest.service_client
+
+    @staticmethod
+    def add_arguments(parser):
+        super(_LegacyServiceTest, _LegacyServiceTest).add_arguments(parser)
+        parser.add_argument('--max-put-size', nargs='?', type=int, help='Maximum size of data uploading in single HTTP PUT. Defaults to 64*1024*1024', default=64*1024*1024)
+        parser.add_argument('--max-block-size', nargs='?', type=int, help='Maximum size of data in a block within a blob. Defaults to 4*1024*1024', default=4*1024*1024)
+        parser.add_argument('--buffer-threshold', nargs='?', type=int, help='Minimum block size to prevent full block buffering. Defaults to 4*1024*1024+1', default=4*1024*1024+1)
+        parser.add_argument('-c', '--max-concurrency', nargs='?', type=int, help='Maximum number of concurrent threads used for data transfer. Defaults to 1', default=1)
+        parser.add_argument('-s', '--size', nargs='?', type=int, help='Size of data to transfer.  Default is 10240.', default=10240)
+        parser.add_argument('--no-client-share', action='store_true', help='Create one ServiceClient per test instance.  Default is to share a single ServiceClient.', default=False)
+
+
+class _LegacyContainerTest(_LegacyServiceTest):
+    container_name = "perfstress-legacy-" + str(uuid.uuid4())
+
+    def __init__(self, arguments):
+        super().__init__(arguments)
+
+    async def global_setup(self):
+        await super().global_setup()
+        self.service_client.create_container(self.container_name)
+
+    async def global_cleanup(self):
+        self.service_client.delete_container(self.container_name)
+        await super().global_cleanup()

sdk/storage/azure-storage-blob/tests/perfstress_tests/T1_legacy_tests/download.py

@@ -0,0 +1,36 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+from azure_devtools.perfstress_tests import get_random_bytes, WriteStream
+
+from ._test_base_legacy import _LegacyContainerTest
+
+
+class LegacyDownloadTest(_LegacyContainerTest):
+    def __init__(self, arguments):
+        super().__init__(arguments)
+        self.blob_name = "downloadtest"
+        self.download_stream = WriteStream()
+
+    async def global_setup(self):
+        await super().global_setup()
+        data = get_random_bytes(self.args.size)
+        self.service_client.create_blob_from_bytes(
+            container_name=self.container_name,
+            blob_name=self.blob_name,
+            blob=data)
+
+    def reset_sync(self):
+        self.download_stream = WriteStream()
+
+    def run_sync(self):
+        self.service_client.get_blob_to_stream(
+            container_name=self.container_name,
+            blob_name=self.blob_name,
+            stream=self.download_stream,
+            max_connections=self.args.max_concurrency)
+
+    async def run_async(self):
+        raise NotImplementedError("Async not supported for legacy T1 tests.")

sdk/storage/azure-storage-blob/tests/perfstress_tests/T1_legacy_tests/list_blobs.py

@@ -0,0 +1,29 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+from ._test_base_legacy import _LegacyContainerTest
+
+
+class LegacyListBlobsTest(_LegacyContainerTest):
+
+    async def global_setup(self):
+        await super().global_setup()
+        for i in range(self.args.num_blobs):
+            self.service_client.create_blob_from_bytes(
+                container_name=self.container_name,
+                blob_name="listtest" + str(i),
+                blob=b"")
+
+    def run_sync(self):
+        for _ in self.service_client.list_blobs(container_name=self.container_name):
+            pass
+
+    async def run_async(self):
+        raise NotImplementedError("Async not supported for legacy T1 tests.")
+
+    @staticmethod
+    def add_arguments(parser):
+        super(LegacyListBlobsTest, LegacyListBlobsTest).add_arguments(parser)
+        parser.add_argument('--num-blobs', nargs='?', type=int, help='Number of blobs to list. Defaults to 100', default=100)

sdk/storage/azure-storage-blob/tests/perfstress_tests/T1_legacy_tests/t1_test_requirements.txt

@@ -0,0 +1 @@
+azure-storage-blob==2.1.0

sdk/storage/azure-storage-blob/tests/perfstress_tests/T1_legacy_tests/upload.py

@@ -0,0 +1,30 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import uuid
+
+from azure_devtools.perfstress_tests import RandomStream
+
+from ._test_base_legacy import _LegacyContainerTest
+
+
+class LegacyUploadTest(_LegacyContainerTest):
+    def __init__(self, arguments):
+        super().__init__(arguments)
+        self.blob_name = "blobtest-" + str(uuid.uuid4())
+        self.upload_stream = RandomStream(self.args.size)
+
+    def reset_sync(self):
+        self.upload_stream = RandomStream(self.args.size)
+
+    def run_sync(self):
+        self.service_client.create_blob_from_stream(
+            container_name=self.container_name,
+            blob_name=self.blob_name,
+            stream=self.upload_stream,
+            max_connections=self.args.max_concurrency)
+
+    async def run_async(self):
+        raise NotImplementedError("Async not supported for legacy T1 tests.")

sdk/storage/azure-storage-blob/tests/perfstress_tests/T1_legacy_tests/upload_block.py

@@ -0,0 +1,28 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import uuid
+
+from azure_devtools.perfstress_tests import get_random_bytes
+
+from ._test_base_legacy import _LegacyContainerTest
+
+
+class LegacyUploadBlockTest(_LegacyContainerTest):
+    def __init__(self, arguments):
+        super().__init__(arguments)
+        self.blob_name = "blobblocktest-" + str(uuid.uuid4())
+        self.block_id = str(uuid.uuid4())
+        self.data = get_random_bytes(self.args.size)
+
+    def run_sync(self):
+        self.service_client.put_block(
+            container_name=self.container_name,
+            blob_name=self.blob_name,
+            block=self.data,
+            block_id=self.block_id)
+
+    async def run_async(self):
+        raise NotImplementedError("Async not supported for legacy T1 tests.")

sdk/storage/azure-storage-blob/tests/perfstress_tests/T1_legacy_tests/upload_from_file.py

@@ -0,0 +1,41 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import os
+import tempfile
+import uuid
+
+from azure_devtools.perfstress_tests import get_random_bytes
+
+from ._test_base_legacy import _LegacyContainerTest
+
+
+class LegacyUploadFromFileTest(_LegacyContainerTest):
+    temp_file = None
+
+    def __init__(self, arguments):
+        super().__init__(arguments)
+        self.blob_name = "containertest-" + str(uuid.uuid4())
+
+    async def global_setup(self):
+        await super().global_setup()
+        data = get_random_bytes(self.args.size)
+        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+            LegacyUploadFromFileTest.temp_file = temp_file.name
+            temp_file.write(data)
+
+    async def global_cleanup(self):
+        os.remove(LegacyUploadFromFileTest.temp_file)
+        await super().global_cleanup()
+
+    def run_sync(self):
+        self.service_client.create_blob_from_path(
+            container_name=self.container_name,
+            blob_name=self.blob_name,
+            file_path=LegacyUploadFromFileTest.temp_file,
+            max_connections=self.args.max_concurrency)
+
+    async def run_async(self):
+        raise NotImplementedError("Async not supported for legacy T1 tests.")

sdk/storage/azure-storage-blob/tests/perfstress_tests/_test_base.py

@@ -0,0 +1,78 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import os
+import uuid
+
+from azure_devtools.perfstress_tests import PerfStressTest
+
+from azure.core.exceptions import ResourceNotFoundError
+from azure.storage.blob import BlobServiceClient as SyncBlobServiceClient
+from azure.storage.blob.aio import BlobServiceClient as AsyncBlobServiceClient
+
+
+class _ServiceTest(PerfStressTest):
+    service_client = None
+    async_service_client = None
+
+    def __init__(self, arguments):
+        super().__init__(arguments)
+        connection_string = self.get_from_env("AZURE_STORAGE_CONNECTION_STRING")
+        kwargs = {}
+        kwargs['max_single_put_size'] = self.args.max_put_size
+        kwargs['max_block_size'] = self.args.max_block_size
+        kwargs['min_large_block_upload_threshold'] = self.args.buffer_threshold
+        if not _ServiceTest.service_client or self.args.no_client_share:
+            _ServiceTest.service_client = SyncBlobServiceClient.from_connection_string(conn_str=connection_string, **kwargs)
+            _ServiceTest.async_service_client = AsyncBlobServiceClient.from_connection_string(conn_str=connection_string, **kwargs)
+        self.service_client = _ServiceTest.service_client
+        self.async_service_client =_ServiceTest.async_service_client
+
+    async def close(self):
+        await self.async_service_client.close()
+        await super().close()
+
+    @staticmethod
+    def add_arguments(parser):
+        super(_ServiceTest, _ServiceTest).add_arguments(parser)
+        parser.add_argument('--max-put-size', nargs='?', type=int, help='Maximum size of data uploading in single HTTP PUT. Defaults to 64*1024*1024', default=64*1024*1024)
+        parser.add_argument('--max-block-size', nargs='?', type=int, help='Maximum size of data in a block within a blob. Defaults to 4*1024*1024', default=4*1024*1024)
+        parser.add_argument('--buffer-threshold', nargs='?', type=int, help='Minimum block size to prevent full block buffering. Defaults to 4*1024*1024+1', default=4*1024*1024+1)
+        parser.add_argument('-c', '--max-concurrency', nargs='?', type=int, help='Maximum number of concurrent threads used for data transfer. Defaults to 1', default=1)
+        parser.add_argument('-s', '--size', nargs='?', type=int, help='Size of data to transfer.  Default is 10240.', default=10240)
+        parser.add_argument('--no-client-share', action='store_true', help='Create one ServiceClient per test instance.  Default is to share a single ServiceClient.', default=False)
+
+
+class _ContainerTest(_ServiceTest):
+    container_name = "perfstress-" + str(uuid.uuid4())
+
+    def __init__(self, arguments):
+        super().__init__(arguments)
+        self.container_client = self.service_client.get_container_client(self.container_name)
+        self.async_container_client = self.async_service_client.get_container_client(self.container_name)
+
+    async def global_setup(self):
+        await super().global_setup()
+        await self.async_container_client.create_container()
+
+    async def global_cleanup(self):
+        await self.async_container_client.delete_container()
+        await super().global_cleanup()
+
+    async def close(self):
+        await self.async_container_client.close()
+        await super().close()
+
+
+class _BlobTest(_ContainerTest):
+    def __init__(self, arguments):
+        super().__init__(arguments)
+        blob_name = "blobtest-" + str(uuid.uuid4())
+        self.blob_client = self.container_client.get_blob_client(blob_name)
+        self.async_blob_client = self.async_container_client.get_blob_client(blob_name)
+
+    async def close(self):
+        await self.async_blob_client.close()
+        await super().close()

sdk/storage/azure-storage-blob/tests/perfstress_tests/download.py

@@ -0,0 +1,40 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+from azure_devtools.perfstress_tests import get_random_bytes, WriteStream
+
+from ._test_base import _ContainerTest
+
+
+class DownloadTest(_ContainerTest):
+    def __init__(self, arguments):
+        super().__init__(arguments)
+        blob_name = "downloadtest"
+        self.blob_client = self.container_client.get_blob_client(blob_name)
+        self.async_blob_client = self.async_container_client.get_blob_client(blob_name)
+        self.download_stream = WriteStream()
+
+    async def global_setup(self):
+        await super().global_setup()
+        data = get_random_bytes(self.args.size)
+        await self.async_blob_client.upload_blob(data)
+
+    def reset_sync(self):
+        self.download_stream = WriteStream()
+
+    async def reset_async(self):
+        self.download_stream = WriteStream()
+
+    def run_sync(self):
+        stream = self.blob_client.download_blob(max_concurrency=self.args.max_concurrency)
+        stream.readinto(self.download_stream)
+
+    async def run_async(self):
+        stream = await self.async_blob_client.download_blob(max_concurrency=self.args.max_concurrency)
+        await stream.readinto(self.download_stream)
+
+    async def close(self):
+        await self.async_blob_client.close()
+        await super().close()

sdk/storage/azure-storage-blob/tests/perfstress_tests/list_blobs.py

@@ -0,0 +1,39 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+import itertools
+import asyncio
+
+from ._test_base import _ContainerTest
+
+
+class ListBlobsTest(_ContainerTest):
+
+    async def global_setup(self):
+        await super().global_setup()
+        pending = (asyncio.ensure_future(self.async_container_client.upload_blob("listtest" + str(i), data=b"")) for i in range(self.args.num_blobs))
+        running = list(itertools.islice(pending, 16))
+        while True:
+            # Wait for some upload to finish before adding a new one
+            done, running = await asyncio.wait(running, return_when=asyncio.FIRST_COMPLETED)
+            try:
+                for _ in range(0, len(done)):
+                    next_upload = next(pending)
+                    running.add(next_upload)
+            except StopIteration:
+                await asyncio.wait(running, return_when=asyncio.ALL_COMPLETED)
+                break
+
+    def run_sync(self):
+        for _ in self.container_client.list_blobs():
+            pass
+
+    async def run_async(self):
+        async for _ in self.async_container_client.list_blobs():
+            pass
+
+    @staticmethod
+    def add_arguments(parser):
+        super(ListBlobsTest, ListBlobsTest).add_arguments(parser)
+        parser.add_argument('--num-blobs', nargs='?', type=int, help='Number of blobs to list. Defaults to 100', default=100)