From 56968cd4a6d1c575590d261b1435275342a99ab7 Mon Sep 17 00:00:00 2001 From: antisch Date: Thu, 3 Dec 2020 15:59:24 +1300 Subject: [PATCH 01/15] Fileshare perf tests --- .../T1_legacy_tests/__init__.py | 0 .../T1_legacy_tests/_test_base.py | 48 ++++++++++ .../T1_legacy_tests/download.py | 31 +++++++ .../T1_legacy_tests/download_to_file.py | 44 +++++++++ .../T1_legacy_tests/t1_test_requirements.txt | 1 + .../T1_legacy_tests/upload.py | 42 +++++++++ .../T1_legacy_tests/upload_from_file.py | 40 +++++++++ .../tests/perfstress_tests/__init__.py | 0 .../tests/perfstress_tests/_test_base.py | 89 +++++++++++++++++++ .../tests/perfstress_tests/download.py | 33 +++++++ .../perfstress_tests/download_to_file.py | 37 ++++++++ .../tests/perfstress_tests/upload.py | 34 +++++++ .../perfstress_tests/upload_from_file.py | 36 ++++++++ 13 files changed, 435 insertions(+) create mode 100644 sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/__init__.py create mode 100644 sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/_test_base.py create mode 100644 sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download.py create mode 100644 sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download_to_file.py create mode 100644 sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/t1_test_requirements.txt create mode 100644 sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py create mode 100644 sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload_from_file.py create mode 100644 sdk/storage/azure-storage-file-share/tests/perfstress_tests/__init__.py create mode 100644 sdk/storage/azure-storage-file-share/tests/perfstress_tests/_test_base.py create mode 100644 sdk/storage/azure-storage-file-share/tests/perfstress_tests/download.py create mode 100644 sdk/storage/azure-storage-file-share/tests/perfstress_tests/download_to_file.py create mode 100644 sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload.py create mode 100644 sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload_from_file.py diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/__init__.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/_test_base.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/_test_base.py new file mode 100644 index 000000000000..7ff978632e49 --- /dev/null +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/_test_base.py @@ -0,0 +1,48 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import uuid + +from azure_devtools.perfstress_tests import PerfStressTest + +from azure.storage.file import FileService + +class _LegacyServiceTest(PerfStressTest): + service_client = None + async_service_client = None + + def __init__(self, arguments): + super().__init__(arguments) + connection_string = self.get_from_env("AZURE_STORAGE_CONNECTION_STRING") + if not _LegacyServiceTest.service_client or self.args.no_client_share: + _LegacyServiceTest.service_client = FileService(connection_string=connection_string) + if self.args.max_range_size: + _LegacyServiceTest.service_client.MAX_RANGE_SIZE = self.args.max_range_size + self.async_service_client = None + self.service_client = _LegacyServiceTest.service_client + + @staticmethod + def add_arguments(parser): + super(_LegacyServiceTest, _LegacyServiceTest).add_arguments(parser) + parser.add_argument('-r', '--max-range-size', nargs='?', type=int, help='Maximum size of data uploading in single HTTP PUT. Defaults to 4*1024*1024', default=4*1024*1024) + parser.add_argument('-c', '--max-concurrency', nargs='?', type=int, help='Maximum number of concurrent threads used for data transfer. Defaults to 1', default=1) + parser.add_argument('-s', '--size', nargs='?', type=int, help='Size of data to transfer. Default is 10240.', default=10240) + parser.add_argument('--no-client-share', action='store_true', help='Create one ServiceClient per test instance. Default is to share a single ServiceClient.', default=False) + + +class _LegacyShareTest(_LegacyServiceTest): + share_name = "perfstress-legacy-" + str(uuid.uuid4()) + + def __init__(self, arguments): + super().__init__(arguments) + + async def global_setup(self): + await super().global_setup() + self.service_client.create_share(self.share_name) + + async def global_cleanup(self): + self.service_client.delete_share(self.share_name) + await super().global_cleanup() diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download.py new file mode 100644 index 000000000000..6a717d08e308 --- /dev/null +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download.py @@ -0,0 +1,31 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from azure_devtools.perfstress_tests import get_random_bytes + +from ._test_base_legacy import _LegacyShareTest + + +class LegacyDownloadTest(_LegacyShareTest): + file_name = "downloadtest" + + async def global_setup(self): + await super().global_setup() + data = get_random_bytes(self.args.size) + self.service_client.create_file_from_bytes( + share_name=self.share_name, + directory_name=None, + file_name=self.file_name, + file=data) + + def run_sync(self): + self.service_client.get_file_to_bytes( + share_name=self.share_name, + directory_name=None, + file_name=self.file_name, + max_connections=self.args.max_concurrency) + + async def run_async(self): + raise NotImplementedError("Async not supported for legacy T1 tests.") diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download_to_file.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download_to_file.py new file mode 100644 index 000000000000..8433c885f6a6 --- /dev/null +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download_to_file.py @@ -0,0 +1,44 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import tempfile +import uuid + +from azure_devtools.perfstress_tests import get_random_bytes + +from ._test_base_legacy import _LegacyShareTest + + +class LegacyDownloadToFileTest(_LegacyShareTest): + file_name = "downloadtest" + + async def global_setup(self): + await super().global_setup() + data = get_random_bytes(self.args.size) + self.service_client.create_file_from_bytes( + share_name=self.share_name, + directory_name=None, + file_name=self.file_name, + file=data) + + async def setup(self): + await super().setup() + self.temp_file = os.path.join(tempfile.gettempdir(), str(uuid.uuid4())) + + async def cleanup(self): + os.remove(self.temp_file) + await super().cleanup() + + def run_sync(self): + self.service_client.get_file_to_path( + share_name=self.share_name, + directory_name=None, + file_name=self.file_name, + file_path=self.temp_file, + max_connections=self.args.max_concurrency) + + async def run_async(self): + raise NotImplementedError("Async not supported for legacy T1 tests.") diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/t1_test_requirements.txt b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/t1_test_requirements.txt new file mode 100644 index 000000000000..f162f7a0e8df --- /dev/null +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/t1_test_requirements.txt @@ -0,0 +1 @@ +azure-storage-file==2.1.0 diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py new file mode 100644 index 000000000000..e0f3cbbb88a3 --- /dev/null +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py @@ -0,0 +1,42 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import uuid + +from azure_devtools.perfstress_tests import RandomStream, get_random_bytes + +from ._test_base_legacy import _LegacyShareTest + + +class LegacyUploadTest(_LegacyShareTest): + def __init__(self, arguments): + super().__init__(arguments) + self.file_name = "sharefiletest-" + str(uuid.uuid4()) + self.data = get_random_bytes(self.args.size) + + def run_sync(self): + if self.args.stream: + data = RandomStream(self.args.size) + self.service_client.create_file_from_stream( + share_name=self.share_name, + directory_name=None, + file_name=self.file_name, + stream=data, + max_connections=self.args.max_concurrency) + else: + self.service_client.create_file_from_bytes( + share_name=self.share_name, + directory_name=None, + file_name=self.file_name, + file=self.data, + max_connections=self.args.max_concurrency) + + async def run_async(self): + raise NotImplementedError("Async not supported for legacy T1 tests.") + + @staticmethod + def add_arguments(parser): + super(LegacyUploadTest, LegacyUploadTest).add_arguments(parser) + parser.add_argument('--stream', action='store_true', help='Upload stream instead of byte array.', default=False) diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload_from_file.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload_from_file.py new file mode 100644 index 000000000000..094579977772 --- /dev/null +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload_from_file.py @@ -0,0 +1,40 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import tempfile +import uuid + +from azure_devtools.perfstress_tests import get_random_bytes + +from ._test_base_legacy import _LegacyShareTest + + +class LegacyUploadFromFileTest(_LegacyShareTest): + def __init__(self, arguments): + super().__init__(arguments) + self.file_name = "sharefiletest-" + str(uuid.uuid4()) + self.data = get_random_bytes(self.args.size) + + async def global_setup(self): + await super().global_setup() + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + self.temp_file = temp_file.name + temp_file.write(self.data) + + async def global_cleanup(self): + os.remove(self.temp_file) + await super().global_cleanup() + + def run_sync(self): + self.service_client.create_file_from_path( + share_name=self.share_name, + directory_name=None, + file_name=self.file_name, + local_file_path=self.temp_file, + max_connections=self.args.max_concurrency) + + async def run_async(self): + raise NotImplementedError("Async not supported for legacy T1 tests.") diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/__init__.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/_test_base.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/_test_base.py new file mode 100644 index 000000000000..1d90517f38b8 --- /dev/null +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/_test_base.py @@ -0,0 +1,89 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import uuid + +from azure_devtools.perfstress_tests import PerfStressTest + +from azure.core.exceptions import ResourceNotFoundError +from azure.storage.fileshare import ShareServiceClient as SyncShareServiceClient +from azure.storage.fileshare.aio import ShareServiceClient as AsyncShareServiceClient + + +class _ServiceTest(PerfStressTest): + service_client = None + async_service_client = None + + def __init__(self, arguments): + super().__init__(arguments) + connection_string = self.get_from_env("AZURE_STORAGE_CONNECTION_STRING") + kwargs = {} + if self.args.max_range_size: + kwargs['max_range_size'] = self.args.max_range_size + if not _ServiceTest.service_client or self.args.no_client_share: + _ServiceTest.service_client = SyncShareServiceClient.from_connection_string(conn_str=connection_string, **kwargs) + _ServiceTest.async_service_client = AsyncShareServiceClient.from_connection_string(conn_str=connection_string, **kwargs) + self.service_client = _ServiceTest.service_client + self.async_service_client =_ServiceTest.async_service_client + + async def close(self): + await self.async_service_client.close() + await super().close() + + @staticmethod + def add_arguments(parser): + super(_ServiceTest, _ServiceTest).add_arguments(parser) + parser.add_argument('-r', '--max-range-size', nargs='?', type=int, help='Maximum size of data uploading in single HTTP PUT. Defaults to 4*1024*1024', default=4*1024*1024) + parser.add_argument('-c', '--max-concurrency', nargs='?', type=int, help='Maximum number of concurrent threads used for data transfer. Defaults to 1', default=1) + parser.add_argument('-s', '--size', nargs='?', type=int, help='Size of data to transfer. Default is 10240.', default=10240) + parser.add_argument('--no-client-share', action='store_true', help='Create one ServiceClient per test instance. Default is to share a single ServiceClient.', default=False) + + +class _ShareTest(_ServiceTest): + share_name = "perfstress-" + str(uuid.uuid4()) + + def __init__(self, arguments): + super().__init__(arguments) + self.share_client = self.service_client.get_share_client(self.share_name) + self.async_share_client = self.async_service_client.get_share_client(self.share_name) + + async def global_setup(self): + await super().global_setup() + self.share_client.create_share() + + async def global_cleanup(self): + self.share_client.delete_share() + await super().global_cleanup() + + async def close(self): + await self.async_share_client.close() + await super().close() + + +class _FileTest(_ShareTest): + def __init__(self, arguments): + super().__init__(arguments) + file_name = "sharefiletest-" + str(uuid.uuid4()) + self.sharefile_client = self.share_client.get_file_client(file_name) + self.async_sharefile_client = self.async_share_client.get_file_client(file_name) + + async def global_setup(self): + await super().global_setup() + try: + self.sharefile_client.delete_file() + except ResourceNotFoundError: + pass + + async def global_cleanup(self): + try: + self.sharefile_client.delete_file() + except ResourceNotFoundError: + pass + await super().global_cleanup() + + async def close(self): + await self.async_sharefile_client.close() + await super().close() diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/download.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/download.py new file mode 100644 index 000000000000..0f568e577138 --- /dev/null +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/download.py @@ -0,0 +1,33 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from azure_devtools.perfstress_tests import get_random_bytes + +from ._test_base import _ShareTest + + +class DownloadTest(_ShareTest): + def __init__(self, arguments): + super().__init__(arguments) + file_name = "downloadtest" + self.sharefile_client = self.share_client.get_file_client(file_name) + self.async_sharefile_client = self.async_share_client.get_file_client(file_name) + + async def global_setup(self): + await super().global_setup() + data = get_random_bytes(self.args.size) + await self.async_sharefile_client.upload_file(data) + + def run_sync(self): + stream = self.sharefile_client.download_file(max_concurrency=self.args.max_concurrency) + stream.readall() + + async def run_async(self): + stream = await self.async_sharefile_client.download_file(max_concurrency=self.args.max_concurrency) + await stream.readall() + + async def close(self): + await self.async_sharefile_client.close() + await super().close() diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/download_to_file.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/download_to_file.py new file mode 100644 index 000000000000..807fffe69c9f --- /dev/null +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/download_to_file.py @@ -0,0 +1,37 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import tempfile + +from azure_devtools.perfstress_tests import get_random_bytes + +from ._test_base import _ShareTest + + +class DownloadToFileTest(_ShareTest): + def __init__(self, arguments): + super().__init__(arguments) + file_name = "downloadtest" + self.sharefile_client = self.share_client.get_file_client(file_name) + self.async_sharefile_client = self.async_share_client.get_file_client(file_name) + + async def global_setup(self): + await super().global_setup() + data = get_random_bytes(self.args.size) + await self.async_sharefile_client.upload_file(data) + + def run_sync(self): + with tempfile.TemporaryFile() as fp: + stream = self.sharefile_client.download_file(max_concurrency=self.args.max_concurrency) + stream.readinto(fp) + + async def run_async(self): + with tempfile.TemporaryFile() as fp: + stream = await self.async_sharefile_client.download_file(max_concurrency=self.args.max_concurrency) + await stream.readinto(fp) + + async def close(self): + await self.async_sharefile_client.close() + await super().close() diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload.py new file mode 100644 index 000000000000..664b98009e90 --- /dev/null +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload.py @@ -0,0 +1,34 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from ._test_base import _FileTest + +from azure_devtools.perfstress_tests import RandomStream, get_random_bytes +from azure_devtools.perfstress_tests import AsyncRandomStream + + +class UploadTest(_FileTest): + def __init__(self, arguments): + super().__init__(arguments) + self.data = get_random_bytes(self.args.size) + + def run_sync(self): + data = RandomStream(self.args.size) if self.args.stream else self.data + self.sharefile_client.upload_file( + data, + length=self.args.size, + max_concurrency=self.args.max_concurrency) + + async def run_async(self): + data = AsyncRandomStream(self.args.size) if self.args.stream else self.data + await self.async_sharefile_client.upload_file( + data, + length=self.args.size, + max_concurrency=self.args.max_concurrency) + + @staticmethod + def add_arguments(parser): + super(UploadTest, UploadTest).add_arguments(parser) + parser.add_argument('--stream', action='store_true', help='Upload stream instead of byte array.', default=False) diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload_from_file.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload_from_file.py new file mode 100644 index 000000000000..fbc0c4b3d8dc --- /dev/null +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload_from_file.py @@ -0,0 +1,36 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import tempfile + +from azure_devtools.perfstress_tests import get_random_bytes + +from ._test_base import _FileTest + + +class UploadFromFileTest(_FileTest): + def __init__(self, arguments): + super().__init__(arguments) + self.temp_file = None + self.data = get_random_bytes(self.args.size) + + async def global_setup(self): + await super().global_setup() + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + self.temp_file = temp_file.name + temp_file.write(self.data) + + async def global_cleanup(self): + os.remove(self.temp_file) + await super().global_cleanup() + + def run_sync(self): + with open(self.temp_file) as fp: + self.sharefile_client.upload_file(fp, max_concurrency=self.args.max_concurrency) + + async def run_async(self): + with open(self.temp_file) as fp: + await self.async_sharefile_client.upload_file(fp, max_concurrency=self.args.max_concurrency) From 2b397065832a0487ccd3ce9cab2fac1353372906 Mon Sep 17 00:00:00 2001 From: antisch Date: Fri, 11 Dec 2020 08:35:23 +1300 Subject: [PATCH 02/15] Memory management --- .../tests/perfstress_tests/download.py | 10 ++++++---- .../tests/perfstress_tests/upload.py | 12 +++--------- .../tests/perfstress_tests/upload_from_file.py | 17 ++++++++--------- 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/download.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/download.py index 0f568e577138..beee28bfb80d 100644 --- a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/download.py +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/download.py @@ -21,12 +21,14 @@ async def global_setup(self): await self.async_sharefile_client.upload_file(data) def run_sync(self): - stream = self.sharefile_client.download_file(max_concurrency=self.args.max_concurrency) - stream.readall() + stream = self.sharefile_client.download_file() + for _ in stream.chunks(): + pass async def run_async(self): - stream = await self.async_sharefile_client.download_file(max_concurrency=self.args.max_concurrency) - await stream.readall() + stream = await self.async_sharefile_client.download_file() + async for _ in stream.chunks(): + pass async def close(self): await self.async_sharefile_client.close() diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload.py index 664b98009e90..0b1b0fbbfd03 100644 --- a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload.py +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload.py @@ -12,23 +12,17 @@ class UploadTest(_FileTest): def __init__(self, arguments): super().__init__(arguments) - self.data = get_random_bytes(self.args.size) def run_sync(self): - data = RandomStream(self.args.size) if self.args.stream else self.data + data = RandomStream(self.args.size) self.sharefile_client.upload_file( data, length=self.args.size, max_concurrency=self.args.max_concurrency) async def run_async(self): - data = AsyncRandomStream(self.args.size) if self.args.stream else self.data + data = AsyncRandomStream(self.args.size) await self.async_sharefile_client.upload_file( data, length=self.args.size, - max_concurrency=self.args.max_concurrency) - - @staticmethod - def add_arguments(parser): - super(UploadTest, UploadTest).add_arguments(parser) - parser.add_argument('--stream', action='store_true', help='Upload stream instead of byte array.', default=False) + max_concurrency=self.args.max_concurrency) \ No newline at end of file diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload_from_file.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload_from_file.py index fbc0c4b3d8dc..6530666f3248 100644 --- a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload_from_file.py +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload_from_file.py @@ -12,25 +12,24 @@ class UploadFromFileTest(_FileTest): - def __init__(self, arguments): - super().__init__(arguments) - self.temp_file = None - self.data = get_random_bytes(self.args.size) + temp_file = None async def global_setup(self): await super().global_setup() + data = get_random_bytes(self.args.size) with tempfile.NamedTemporaryFile(delete=False) as temp_file: - self.temp_file = temp_file.name - temp_file.write(self.data) + UploadFromFileTest.temp_file = temp_file.name + temp_file.write(data) async def global_cleanup(self): - os.remove(self.temp_file) + if self.temp_file: + os.remove(UploadFromFileTest.temp_file) await super().global_cleanup() def run_sync(self): - with open(self.temp_file) as fp: + with open(UploadFromFileTest.temp_file, 'rb') as fp: self.sharefile_client.upload_file(fp, max_concurrency=self.args.max_concurrency) async def run_async(self): - with open(self.temp_file) as fp: + with open(UploadFromFileTest.temp_file, 'rb') as fp: await self.async_sharefile_client.upload_file(fp, max_concurrency=self.args.max_concurrency) From 8db909bfc0a14244ec1f45cd830526c78fab6f5d Mon Sep 17 00:00:00 2001 From: antisch Date: Mon, 14 Dec 2020 12:47:56 +1300 Subject: [PATCH 03/15] Fix legacy test memory --- .../T1_legacy_tests/upload.py | 28 +++++-------------- .../T1_legacy_tests/upload_from_file.py | 4 +-- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py index e0f3cbbb88a3..07de5de4e7c8 100644 --- a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py @@ -14,29 +14,15 @@ class LegacyUploadTest(_LegacyShareTest): def __init__(self, arguments): super().__init__(arguments) self.file_name = "sharefiletest-" + str(uuid.uuid4()) - self.data = get_random_bytes(self.args.size) def run_sync(self): - if self.args.stream: - data = RandomStream(self.args.size) - self.service_client.create_file_from_stream( - share_name=self.share_name, - directory_name=None, - file_name=self.file_name, - stream=data, - max_connections=self.args.max_concurrency) - else: - self.service_client.create_file_from_bytes( - share_name=self.share_name, - directory_name=None, - file_name=self.file_name, - file=self.data, - max_connections=self.args.max_concurrency) + data = RandomStream(self.args.size) + self.service_client.create_file_from_stream( + share_name=self.share_name, + directory_name=None, + file_name=self.file_name, + stream=data, + max_connections=self.args.max_concurrency) async def run_async(self): raise NotImplementedError("Async not supported for legacy T1 tests.") - - @staticmethod - def add_arguments(parser): - super(LegacyUploadTest, LegacyUploadTest).add_arguments(parser) - parser.add_argument('--stream', action='store_true', help='Upload stream instead of byte array.', default=False) diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload_from_file.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload_from_file.py index 094579977772..e406fc33aa06 100644 --- a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload_from_file.py +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload_from_file.py @@ -16,13 +16,13 @@ class LegacyUploadFromFileTest(_LegacyShareTest): def __init__(self, arguments): super().__init__(arguments) self.file_name = "sharefiletest-" + str(uuid.uuid4()) - self.data = get_random_bytes(self.args.size) async def global_setup(self): await super().global_setup() + data = get_random_bytes(self.args.size) with tempfile.NamedTemporaryFile(delete=False) as temp_file: self.temp_file = temp_file.name - temp_file.write(self.data) + temp_file.write(data) async def global_cleanup(self): os.remove(self.temp_file) From 84d11bb4acad8df5f9e3a59c7d92587d67e0e604 Mon Sep 17 00:00:00 2001 From: antisch Date: Mon, 14 Dec 2020 12:51:47 +1300 Subject: [PATCH 04/15] Fix upload test --- .../tests/perfstress_tests/T1_legacy_tests/upload.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py index 07de5de4e7c8..f1f345a3dea3 100644 --- a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py @@ -22,6 +22,7 @@ def run_sync(self): directory_name=None, file_name=self.file_name, stream=data, + count=self.args.size, max_connections=self.args.max_concurrency) async def run_async(self): From a2289a8014128cbc32faaf2d7547f61ceb7b04a0 Mon Sep 17 00:00:00 2001 From: antisch Date: Mon, 14 Dec 2020 14:38:23 +1300 Subject: [PATCH 05/15] Fix file upload test --- .../tests/perfstress_tests/T1_legacy_tests/download.py | 2 +- .../T1_legacy_tests/download_to_file.py | 2 +- .../tests/perfstress_tests/T1_legacy_tests/upload.py | 2 +- .../T1_legacy_tests/upload_from_file.py | 10 ++++++---- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download.py index 6a717d08e308..35fc89a0cd66 100644 --- a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download.py +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download.py @@ -5,7 +5,7 @@ from azure_devtools.perfstress_tests import get_random_bytes -from ._test_base_legacy import _LegacyShareTest +from ._test_base import _LegacyShareTest class LegacyDownloadTest(_LegacyShareTest): diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download_to_file.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download_to_file.py index 8433c885f6a6..ff5153c1268c 100644 --- a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download_to_file.py +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download_to_file.py @@ -9,7 +9,7 @@ from azure_devtools.perfstress_tests import get_random_bytes -from ._test_base_legacy import _LegacyShareTest +from ._test_base import _LegacyShareTest class LegacyDownloadToFileTest(_LegacyShareTest): diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py index f1f345a3dea3..f45efa3ab797 100644 --- a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py @@ -7,7 +7,7 @@ from azure_devtools.perfstress_tests import RandomStream, get_random_bytes -from ._test_base_legacy import _LegacyShareTest +from ._test_base import _LegacyShareTest class LegacyUploadTest(_LegacyShareTest): diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload_from_file.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload_from_file.py index e406fc33aa06..d762c2cf4f4a 100644 --- a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload_from_file.py +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload_from_file.py @@ -9,10 +9,12 @@ from azure_devtools.perfstress_tests import get_random_bytes -from ._test_base_legacy import _LegacyShareTest +from ._test_base import _LegacyShareTest class LegacyUploadFromFileTest(_LegacyShareTest): + temp_file = None + def __init__(self, arguments): super().__init__(arguments) self.file_name = "sharefiletest-" + str(uuid.uuid4()) @@ -21,11 +23,11 @@ async def global_setup(self): await super().global_setup() data = get_random_bytes(self.args.size) with tempfile.NamedTemporaryFile(delete=False) as temp_file: - self.temp_file = temp_file.name + LegacyUploadFromFileTest.temp_file = temp_file.name temp_file.write(data) async def global_cleanup(self): - os.remove(self.temp_file) + os.remove(LegacyUploadFromFileTest.temp_file) await super().global_cleanup() def run_sync(self): @@ -33,7 +35,7 @@ def run_sync(self): share_name=self.share_name, directory_name=None, file_name=self.file_name, - local_file_path=self.temp_file, + local_file_path=LegacyUploadFromFileTest.temp_file, max_connections=self.args.max_concurrency) async def run_async(self): From 74b529e6b081f279697015e8c8fbde7d58b46e5b Mon Sep 17 00:00:00 2001 From: antisch Date: Tue, 15 Dec 2020 09:12:11 +1300 Subject: [PATCH 06/15] Fix perf stream --- .../src/azure_devtools/perfstress_tests/random_stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py b/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py index 0ab5283c288f..91208c8d12a7 100644 --- a/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py +++ b/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py @@ -18,7 +18,7 @@ def __init__(self, length, initial_buffer_length=1024*1024): def read(self, size=None): if self._remaining == 0: - return None + return b"" if size is None: e = self._base_data_length From 5fbbdc517102bccf193e939c4f2d860d14801c32 Mon Sep 17 00:00:00 2001 From: antisch Date: Mon, 11 Jan 2021 10:10:15 +1300 Subject: [PATCH 07/15] Removed global cleanup --- .../tests/perfstress_tests/_test_base.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/_test_base.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/_test_base.py index 1d90517f38b8..e409571dfc85 100644 --- a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/_test_base.py +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/_test_base.py @@ -77,13 +77,6 @@ async def global_setup(self): except ResourceNotFoundError: pass - async def global_cleanup(self): - try: - self.sharefile_client.delete_file() - except ResourceNotFoundError: - pass - await super().global_cleanup() - async def close(self): await self.async_sharefile_client.close() await super().close() From b16b86aae775e17f66fb85b78b07ea5deb253036 Mon Sep 17 00:00:00 2001 From: antisch Date: Tue, 12 Jan 2021 14:08:13 +1300 Subject: [PATCH 08/15] Update stream support --- .../T1_legacy_tests/download.py | 11 +++- .../T1_legacy_tests/upload.py | 5 +- .../tests/perfstress_tests/download.py | 15 ++--- .../tests/perfstress_tests/upload.py | 10 +-- .../perfstress_tests/__init__.py | 3 +- .../perfstress_tests/async_random_stream.py | 30 +++++++-- .../perfstress_tests/random_stream.py | 61 ++++++++++++++++--- 7 files changed, 105 insertions(+), 30 deletions(-) diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download.py index 35fc89a0cd66..662d21f92cf3 100644 --- a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download.py +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/download.py @@ -3,13 +3,16 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -from azure_devtools.perfstress_tests import get_random_bytes +from azure_devtools.perfstress_tests import get_random_bytes, WriteStream from ._test_base import _LegacyShareTest class LegacyDownloadTest(_LegacyShareTest): - file_name = "downloadtest" + def __init__(self, arguments): + super().__init__(arguments) + self.file_name = "downloadtest" + self.download_stream = WriteStream() async def global_setup(self): await super().global_setup() @@ -21,10 +24,12 @@ async def global_setup(self): file=data) def run_sync(self): - self.service_client.get_file_to_bytes( + self.download_stream.reset() + self.service_client.get_file_to_stream( share_name=self.share_name, directory_name=None, file_name=self.file_name, + stream=self.download_stream, max_connections=self.args.max_concurrency) async def run_async(self): diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py index f45efa3ab797..8e59e3caeb2e 100644 --- a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/T1_legacy_tests/upload.py @@ -14,14 +14,15 @@ class LegacyUploadTest(_LegacyShareTest): def __init__(self, arguments): super().__init__(arguments) self.file_name = "sharefiletest-" + str(uuid.uuid4()) + self.upload_stream = RandomStream(self.args.size) def run_sync(self): - data = RandomStream(self.args.size) + self.upload_stream.reset() self.service_client.create_file_from_stream( share_name=self.share_name, directory_name=None, file_name=self.file_name, - stream=data, + stream=self.upload_stream, count=self.args.size, max_connections=self.args.max_concurrency) diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/download.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/download.py index beee28bfb80d..0ff3b6188d81 100644 --- a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/download.py +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/download.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -from azure_devtools.perfstress_tests import get_random_bytes +from azure_devtools.perfstress_tests import get_random_bytes, WriteStream from ._test_base import _ShareTest @@ -14,6 +14,7 @@ def __init__(self, arguments): file_name = "downloadtest" self.sharefile_client = self.share_client.get_file_client(file_name) self.async_sharefile_client = self.async_share_client.get_file_client(file_name) + self.download_stream = WriteStream() async def global_setup(self): await super().global_setup() @@ -21,14 +22,14 @@ async def global_setup(self): await self.async_sharefile_client.upload_file(data) def run_sync(self): - stream = self.sharefile_client.download_file() - for _ in stream.chunks(): - pass + self.download_stream.reset() + stream = self.sharefile_client.download_file(max_concurrency=self.args.max_concurrency) + stream.readinto(self.download_stream) async def run_async(self): - stream = await self.async_sharefile_client.download_file() - async for _ in stream.chunks(): - pass + self.download_stream.reset() + stream = await self.async_sharefile_client.download_file(max_concurrency=self.args.max_concurrency) + await stream.readinto(self.download_stream) async def close(self): await self.async_sharefile_client.close() diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload.py index 0b1b0fbbfd03..a3f018701978 100644 --- a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload.py +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload.py @@ -12,17 +12,19 @@ class UploadTest(_FileTest): def __init__(self, arguments): super().__init__(arguments) + self.upload_stream = RandomStream(self.args.size) + self.upload_stream_async = AsyncRandomStream(self.args.size) def run_sync(self): - data = RandomStream(self.args.size) + self.upload_stream.reset() self.sharefile_client.upload_file( - data, + self.upload_stream, length=self.args.size, max_concurrency=self.args.max_concurrency) async def run_async(self): - data = AsyncRandomStream(self.args.size) + self.upload_stream_async.reset() await self.async_sharefile_client.upload_file( - data, + self.upload_stream_async, length=self.args.size, max_concurrency=self.args.max_concurrency) \ No newline at end of file diff --git a/tools/azure-devtools/src/azure_devtools/perfstress_tests/__init__.py b/tools/azure-devtools/src/azure_devtools/perfstress_tests/__init__.py index e3c21008314b..0716333ea00a 100644 --- a/tools/azure-devtools/src/azure_devtools/perfstress_tests/__init__.py +++ b/tools/azure-devtools/src/azure_devtools/perfstress_tests/__init__.py @@ -8,13 +8,14 @@ from .perf_stress_runner import PerfStressRunner from .perf_stress_test import PerfStressTest -from .random_stream import RandomStream, get_random_bytes +from .random_stream import RandomStream, WriteStream, get_random_bytes from .async_random_stream import AsyncRandomStream __all__ = [ "PerfStressRunner", "PerfStressTest", "RandomStream", + "WriteStream", "AsyncRandomStream", "get_random_bytes" ] diff --git a/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py b/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py index 11f0d663c416..473b0ac348df 100644 --- a/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py +++ b/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py @@ -5,33 +5,51 @@ from io import BytesIO -from .random_stream import get_random_bytes +from .random_stream import get_random_bytes, _DEFAULT_LENGTH class AsyncRandomStream(BytesIO): - def __init__(self, length, initial_buffer_length=1024 * 1024): + def __init__(self, length, initial_buffer_length=_DEFAULT_LENGTH): super().__init__() self._base_data = get_random_bytes(initial_buffer_length) - self._base_data_length = initial_buffer_length + self._data_length = length + self._base_buffer_length = initial_buffer_length self._position = 0 self._remaining = length self._closed = False + + def reset(self): + self._position = 0 + self._remaining = self._data_length + self._closed = False def read(self, size=None): if self._remaining == 0: return b"" if size is None: - e = self._base_data_length + e = self._base_buffer_length else: e = size e = min(e, self._remaining) - if e > self._base_data_length: + if e > self._base_buffer_length: self._base_data = get_random_bytes(e) - self._base_data_length = e + self._base_buffer_length = e self._remaining = self._remaining - e + self._position += e return self._base_data[:e] + def seek(self, index, whence=0): + if whence == 0: + self._position = index + elif whence == 1: + self._position = self._position + index + elif whence == 2: + self._position = self._length - 1 + index + + def tell(self): + return self._position + def remaining(self): return self._remaining diff --git a/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py b/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py index 91208c8d12a7..324a3eb67553 100644 --- a/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py +++ b/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py @@ -5,31 +5,78 @@ import os +_DEFAULT_LENGTH = 1024*1024 +_BYTE_BUFFER = [_DEFAULT_LENGTH, os.urandom(_DEFAULT_LENGTH)] + + def get_random_bytes(buffer_length): - return os.urandom(buffer_length) + if buffer_length > _BYTE_BUFFER[0]: + _BYTE_BUFFER[0] = buffer_length + _BYTE_BUFFER[1] = os.urandom(buffer_length) + return _BYTE_BUFFER[1][:buffer_length] class RandomStream: - def __init__(self, length, initial_buffer_length=1024*1024): + def __init__(self, length, initial_buffer_length=_DEFAULT_LENGTH): self._base_data = get_random_bytes(initial_buffer_length) - self._base_data_length = initial_buffer_length + self._data_length = length + self._base_buffer_length = initial_buffer_length self._position = 0 self._remaining = length + + def reset(self): + self._position = 0 + self._remaining = self._data_length def read(self, size=None): if self._remaining == 0: return b"" if size is None: - e = self._base_data_length + e = self._base_buffer_length else: e = size e = min(e, self._remaining) - if e > self._base_data_length: + if e > self._base_buffer_length: self._base_data = get_random_bytes(e) - self._base_data_length = e + self._base_buffer_length = e self._remaining = self._remaining - e + self._position += e return self._base_data[:e] + def tell(self): + return self._position + + def seek(self, index, whence=0): + if whence == 0: + self._position = index + elif whence == 1: + self._position = self._position + index + elif whence == 2: + self._position = self._length - 1 + index + def remaining(self): - return self._remaining \ No newline at end of file + return self._remaining + + +class WriteStream: + + def __init__(self): + self._position = 0 + + def reset(self): + self._position = 0 + + def write(self, content): + length = len(content) + self._position += length + return length + + def seek(self, index): + self._position = index + + def seekable(self): + return True + + def tell(self): + return self._position From 75bfaade2346c8502cb34c42e870b9622418794d Mon Sep 17 00:00:00 2001 From: antisch Date: Wed, 13 Jan 2021 09:23:16 +1300 Subject: [PATCH 09/15] Remove useless setup --- .../tests/perfstress_tests/_test_base.py | 8 -------- .../tests/perfstress_tests/upload.py | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/_test_base.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/_test_base.py index e409571dfc85..970e7dcf5ff6 100644 --- a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/_test_base.py +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/_test_base.py @@ -8,7 +8,6 @@ from azure_devtools.perfstress_tests import PerfStressTest -from azure.core.exceptions import ResourceNotFoundError from azure.storage.fileshare import ShareServiceClient as SyncShareServiceClient from azure.storage.fileshare.aio import ShareServiceClient as AsyncShareServiceClient @@ -70,13 +69,6 @@ def __init__(self, arguments): self.sharefile_client = self.share_client.get_file_client(file_name) self.async_sharefile_client = self.async_share_client.get_file_client(file_name) - async def global_setup(self): - await super().global_setup() - try: - self.sharefile_client.delete_file() - except ResourceNotFoundError: - pass - async def close(self): await self.async_sharefile_client.close() await super().close() diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload.py b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload.py index a3f018701978..65f06655bffd 100644 --- a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload.py +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/upload.py @@ -27,4 +27,4 @@ async def run_async(self): await self.async_sharefile_client.upload_file( self.upload_stream_async, length=self.args.size, - max_concurrency=self.args.max_concurrency) \ No newline at end of file + max_concurrency=self.args.max_concurrency) From 83e7797f908f08c9be441e726c5d755655e0e8ed Mon Sep 17 00:00:00 2001 From: antisch Date: Fri, 15 Jan 2021 10:13:11 +1300 Subject: [PATCH 10/15] Added readme --- .../tests/perfstress_tests/README.md | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 sdk/storage/azure-storage-file-share/tests/perfstress_tests/README.md diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/README.md b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/README.md new file mode 100644 index 000000000000..132a3e6950f0 --- /dev/null +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/README.md @@ -0,0 +1,64 @@ +# FileShare Performance Tests + +In order to run the performance tests, the `azure-devtools` package must be installed. This is done as part of the `dev_requirements`. +Start be creating a new virtual environment for your perf tests. This will need to be a Python 3 environment, preferably >=3.7. +Note that tests for T1 and T2 SDKs cannot be run from the same environment, and will need to be setup separately. + +### Setup for T2 perf test runs + +```cmd +(env) ~/azure-storage-file-share> pip install -r dev_requirements.txt +(env) ~/azure-storage-file-share> pip install -e . +``` + +### Setup for T1 perf test runs + +```cmd +(env) ~/azure-storage-file-share> pip install -r dev_requirements.txt +(env) ~/azure-storage-file-share> pip install tests/perfstress_tests/T1_legacy_tests/t1_test_requirements.txt +``` + +## Test commands + +When `azure-devtools` is installed, you will have access to the `perfstress` command line tool, which will scan the current module for runable perf tests. Only a specific test can be run at a time (i.e. there is no "run all" feature). + +```cmd +(env) ~/azure-storage-file-share> cd tests +(env) ~/azure-storage-file-share/tests> perfstress +``` +Using the `perfstress` command alone will list the available perf tests found. Note that the available tests discovered will vary depending on whether your environment is configured for the T1 or T2 SDK. + +### Common perf command line options +These options are available for all perf tests: +- `--duration=10` Number of seconds to run as many operations (the "run" function) as possible. Default is 10. +- `--iterations=1` Number of test iterations to run. Default is 1. +- `--parallel=1` Number of tests to run in parallel. Default is 1. +- `--no-client-share` Whether each parallel test instance should share a single client, or use their own. Default is False (sharing). +- `--warm-up=5` Number of seconds to spend warming up the connection before measuing begins. Default is 5. +- `--sync` Whether to run the tests in sync or async. Default is False (async). This flag must be used for Storage legacy tests, which do not support async. +- `--no-cleanup` Whether to keep newly created resources after test run. Default is False (resources will be deleted). + +### Common FileShare command line options +The options are available for all SB perf tests: +- `--size=100` Size in bytes of data to be transferred in upload or download tests. Default is 10240. +- `--max-concurrency=1` Number of threads to concurrently upload/download a single operation using the SDK API parameter. Default is 1. +- `--max-range-size`Maximum size of data uploading in single HTTP PUT. Default is 4*1024*1024. + +### T2 Tests +The tests currently written for the T2 SDK: +- `UploadTest` Uploads a stream of `size` bytes to a new File. +- `UploadFromFileTest` Uploads a local file of `size` bytes to a new File. +- `DownloadTest` Download a stream of `size` bytes. +- `DownloadToFileTest` Download `size` bytes into a local file. + +### T1 Tests +The tests currently written for the T2 SDK: +- `LegacyUploadTest` Uploads a stream of `size` bytes to a new File. +- `LegacyUploadFromFileTest` Uploads a local file of `size` bytes to a new File. +- `LegacyDownloadTest` Download a stream of `size` bytes. +- `LegacyDownloadToFileTest` Download `size` bytes into a local file. + +## Example command +```cmd +(env) ~/azure-storage-file-share/tests> perfstress UploadTest --parallel=2 --size=10240 +``` \ No newline at end of file From 69151e1f9e123d599d75cbc09b21a276c9ce7e3f Mon Sep 17 00:00:00 2001 From: antisch Date: Fri, 15 Jan 2021 10:36:27 +1300 Subject: [PATCH 11/15] Ignore readme --- eng/.docsettings.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/.docsettings.yml b/eng/.docsettings.yml index 07f073a72430..23b1fe73b38c 100644 --- a/eng/.docsettings.yml +++ b/eng/.docsettings.yml @@ -8,6 +8,7 @@ omitted_paths: - doc/* - sdk/**/samples/* - sdk/identity/azure-identity/tests/* + - sdk/**/tests/perfstress_tests/* language: python root_check_enabled: True From d784aac6be90bc952f09df3bb73394d5a624389a Mon Sep 17 00:00:00 2001 From: antisch Date: Thu, 11 Feb 2021 09:55:05 +1300 Subject: [PATCH 12/15] Fix stream merge --- .../perfstress_tests/async_random_stream.py | 8 +------- .../src/azure_devtools/perfstress_tests/random_stream.py | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py b/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py index 098a019fa36e..e483bc016afe 100644 --- a/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py +++ b/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py @@ -38,12 +38,6 @@ def read(self, size=None): self._remaining = self._remaining - e self._position += e return self._base_data[:e] - - def seek(self, index): - self._position = index - - def tell(self): - return self._position def seek(self, index, whence=0): if whence == 0: @@ -51,7 +45,7 @@ def seek(self, index, whence=0): elif whence == 1: self._position = self._position + index elif whence == 2: - self._position = self._length - 1 + index + self._position = self._data_length - 1 + index def tell(self): return self._position diff --git a/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py b/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py index 71a873e5dd56..9937e1a0c091 100644 --- a/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py +++ b/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py @@ -59,7 +59,7 @@ def seek(self, index, whence=0): elif whence == 1: self._position = self._position + index elif whence == 2: - self._position = self._length - 1 + index + self._position = self._data_length - 1 + index def remaining(self): return self._remaining From 8d5344a001e9f13fd96ea2cbfff46abff46fe5ec Mon Sep 17 00:00:00 2001 From: antisch Date: Thu, 11 Feb 2021 09:55:19 +1300 Subject: [PATCH 13/15] Fix readme --- .../tests/perfstress_tests/README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/README.md b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/README.md index 132a3e6950f0..16a0730bfaaa 100644 --- a/sdk/storage/azure-storage-file-share/tests/perfstress_tests/README.md +++ b/sdk/storage/azure-storage-file-share/tests/perfstress_tests/README.md @@ -4,6 +4,13 @@ In order to run the performance tests, the `azure-devtools` package must be inst Start be creating a new virtual environment for your perf tests. This will need to be a Python 3 environment, preferably >=3.7. Note that tests for T1 and T2 SDKs cannot be run from the same environment, and will need to be setup separately. +### Setup for test resources + +These tests will run against a pre-configured Storage account. The following environment variable will need to be set for the tests to access the live resources: +``` +AZURE_STORAGE_CONNECTION_STRING= +``` + ### Setup for T2 perf test runs ```cmd @@ -34,7 +41,7 @@ These options are available for all perf tests: - `--iterations=1` Number of test iterations to run. Default is 1. - `--parallel=1` Number of tests to run in parallel. Default is 1. - `--no-client-share` Whether each parallel test instance should share a single client, or use their own. Default is False (sharing). -- `--warm-up=5` Number of seconds to spend warming up the connection before measuing begins. Default is 5. +- `--warm-up=5` Number of seconds to spend warming up the connection before measuring begins. Default is 5. - `--sync` Whether to run the tests in sync or async. Default is False (async). This flag must be used for Storage legacy tests, which do not support async. - `--no-cleanup` Whether to keep newly created resources after test run. Default is False (resources will be deleted). From f7824e81175e015e7a66231e28b613703d9e5999 Mon Sep 17 00:00:00 2001 From: antisch Date: Thu, 11 Feb 2021 09:59:26 +1300 Subject: [PATCH 14/15] Fix stream merge --- .../src/azure_devtools/perfstress_tests/random_stream.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py b/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py index 9937e1a0c091..b68be9bfd806 100644 --- a/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py +++ b/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py @@ -43,12 +43,6 @@ def read(self, size=None): self._remaining = self._remaining - e self._position += e return self._base_data[:e] - - def tell(self): - return self._position - - def seek(self, index): - self._position = index def tell(self): return self._position From d09452b06d987e10f8685df5c4de56200d0be45a Mon Sep 17 00:00:00 2001 From: antisch Date: Thu, 4 Mar 2021 08:43:13 +1300 Subject: [PATCH 15/15] Fix async stream --- .../perfstress_tests/async_random_stream.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py b/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py index f2544340fcb2..e483bc016afe 100644 --- a/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py +++ b/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py @@ -50,17 +50,6 @@ def seek(self, index, whence=0): def tell(self): return self._position - def seek(self, index, whence=0): - if whence == 0: - self._position = index - elif whence == 1: - self._position = self._position + index - elif whence == 2: - self._position = self._length - 1 + index - - def tell(self): - return self._position - def remaining(self): return self._remaining