Skip to content

[Storage] Use tempfile for blob package #27895

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Jan 4, 2023
Merged

[Storage] Use tempfile for blob package #27895

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
b5a56aa
Test append blob done
vincenttran-msft Dec 9, 2022
4bff23b
Finished test_blob_encryption
vincenttran-msft Dec 9, 2022
001e15c
Finished test_block_blob
vincenttran-msft Dec 9, 2022
5555a88
Finished test_common_blob
vincenttran-msft Dec 9, 2022
7025c55
Finished test_get_blob
vincenttran-msft Dec 9, 2022
8505220
Finished test_large_block_blob
vincenttran-msft Dec 10, 2022
41047bc
Finished test_largest_block_blob
vincenttran-msft Dec 10, 2022
0d8efa7
Finished test_page_blob
vincenttran-msft Dec 12, 2022
8e26e90
PR feedback
vincenttran-msft Dec 14, 2022
2609842
Add os.unlink to NamedTempFile w/ delete=false
vincenttran-msft Dec 20, 2022
6021abf
Merge branch 'main' into vincenttran/tempfile_blobs
vincenttran-msft Jan 3, 2023
90ac580
Merge branch 'main' into vincenttran/tempfile_blobs
vincenttran-msft Jan 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 37 additions & 59 deletions sdk/storage/azure-storage-blob/tests/test_append_blob.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
# license information.
# --------------------------------------------------------------------------

import tempfile
import uuid
from datetime import datetime, timedelta
from os import path, remove
Expand Down Expand Up @@ -47,13 +48,6 @@ def _setup(self, bsc):
except:
pass

def _teardown(self, file_name):
if path.isfile(file_name):
try:
remove(file_name)
except:
pass

def _get_blob_reference(self, prefix=TEST_BLOB_PREFIX):
return self.get_resource_name(prefix)

Expand Down Expand Up @@ -1121,21 +1115,19 @@ def test_append_blob_from_path_chunked_upload(self, **kwargs):
self._setup(bsc)
blob = self._create_blob(bsc)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
FILE_PATH = 'from_path_chunked_upload.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)

# Act
with open(FILE_PATH, 'rb') as stream:
append_resp = blob.upload_blob(stream, blob_type=BlobType.AppendBlob)
with tempfile.TemporaryFile() as temp_file:
temp_file.write(data)
temp_file.seek(0)
append_resp = blob.upload_blob(temp_file, blob_type=BlobType.AppendBlob)

blob_properties = blob.get_blob_properties()

# Assert
self.assertBlobEqual(blob, data)
assert blob_properties.etag == append_resp.get('etag')
assert blob_properties.last_modified == append_resp.get('last_modified')
self._teardown(FILE_PATH)

@BlobPreparer()
@recorded_by_proxy
Expand All @@ -1147,9 +1139,6 @@ def test_append_blob_from_path_with_progress_chunked_upload(self, **kwargs):
self._setup(bsc)
blob = self._create_blob(bsc)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
FILE_PATH = 'progress_chunked_upload.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)

# Act
progress = []
Expand All @@ -1166,14 +1155,15 @@ def progress_gen(upload):
yield chunk
current += len(chunk)

with open(FILE_PATH, 'rb') as stream:
upload_data = progress_gen(stream)
with tempfile.TemporaryFile() as temp_file:
temp_file.write(data)
temp_file.seek(0)
upload_data = progress_gen(temp_file)
blob.upload_blob(upload_data, blob_type=BlobType.AppendBlob)

# Assert
self.assertBlobEqual(blob, data)
self.assert_upload_progress(len(data), self.config.max_block_size, progress)
self._teardown(FILE_PATH)

@BlobPreparer()
@recorded_by_proxy
Expand All @@ -1185,20 +1175,18 @@ def test_append_blob_from_stream_chunked_upload(self, **kwargs):
self._setup(bsc)
blob = self._create_blob(bsc)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
FILE_PATH = 'stream_chunked_upload.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)

# Act
with open(FILE_PATH, 'rb') as stream:
append_resp = blob.upload_blob(stream, blob_type=BlobType.AppendBlob)
with tempfile.TemporaryFile() as temp_file:
temp_file.write(data)
temp_file.seek(0)
append_resp = blob.upload_blob(temp_file, blob_type=BlobType.AppendBlob)
blob_properties = blob.get_blob_properties()

# Assert
self.assertBlobEqual(blob, data)
assert blob_properties.etag == append_resp.get('etag')
assert blob_properties.last_modified == append_resp.get('last_modified')
self._teardown(FILE_PATH)

@BlobPreparer()
@recorded_by_proxy
Expand All @@ -1210,19 +1198,17 @@ def test_app_blob_from_stream_nonseekable_chnked_upload_known_size(self, **kwarg
self._setup(bsc)
blob = self._create_blob(bsc)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
FILE_PATH = 'upload_known_size.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
blob_size = len(data) - 66

# Act
with open(FILE_PATH, 'rb') as stream:
non_seekable_file = NonSeekableStream(stream)
with tempfile.TemporaryFile() as temp_file:
temp_file.write(data)
temp_file.seek(0)
non_seekable_file = NonSeekableStream(temp_file)
blob.upload_blob(non_seekable_file, length=blob_size, blob_type=BlobType.AppendBlob)

# Assert
self.assertBlobEqual(blob, data[:blob_size])
self._teardown(FILE_PATH)

@BlobPreparer()
@recorded_by_proxy
Expand All @@ -1234,18 +1220,16 @@ def test_app_blob_from_stream_nonseekable_chnked_upload_unk_size(self, **kwargs)
self._setup(bsc)
blob = self._create_blob(bsc)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
FILE_PATH = 'upload_unk_size.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)

# Act
with open(FILE_PATH, 'rb') as stream:
non_seekable_file = NonSeekableStream(stream)
with tempfile.TemporaryFile() as temp_file:
temp_file.write(data)
temp_file.seek(0)
non_seekable_file = NonSeekableStream(temp_file)
blob.upload_blob(non_seekable_file, blob_type=BlobType.AppendBlob)

# Assert
self.assertBlobEqual(blob, data)
self._teardown(FILE_PATH)

@BlobPreparer()
@recorded_by_proxy
Expand All @@ -1257,22 +1241,20 @@ def test_append_blob_from_stream_with_multiple_appends(self, **kwargs):
self._setup(bsc)
blob = self._create_blob(bsc)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
FILE_PATH = 'multiple_appends.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream1:
stream1.write(data)
with open(FILE_PATH, 'wb') as stream2:
stream2.write(data)

# Act
with open(FILE_PATH, 'rb') as stream1:
blob.upload_blob(stream1, blob_type=BlobType.AppendBlob)
with open(FILE_PATH, 'rb') as stream2:
blob.upload_blob(stream2, blob_type=BlobType.AppendBlob)
with tempfile.TemporaryFile() as temp_file:
temp_file.write(data)
temp_file.seek(0)
with tempfile.TemporaryFile() as temp_file2:
temp_file2.write(data)
temp_file2.seek(0)
blob.upload_blob(temp_file, blob_type=BlobType.AppendBlob)
blob.upload_blob(temp_file2, blob_type=BlobType.AppendBlob)

# Assert
data = data * 2
self.assertBlobEqual(blob, data)
self._teardown(FILE_PATH)

@BlobPreparer()
@recorded_by_proxy
Expand All @@ -1284,18 +1266,16 @@ def test_append_blob_from_stream_chunked_upload_with_count(self, **kwargs):
self._setup(bsc)
blob = self._create_blob(bsc)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
FILE_PATH = 'upload_with_count.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)

# Act
blob_size = len(data) - 301
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream, length=blob_size, blob_type=BlobType.AppendBlob)
with tempfile.TemporaryFile() as temp_file:
temp_file.write(data)
temp_file.seek(0)
blob.upload_blob(temp_file, length=blob_size, blob_type=BlobType.AppendBlob)

# Assert
self.assertBlobEqual(blob, data[:blob_size])
self._teardown(FILE_PATH)

@pytest.mark.live_test_only
@BlobPreparer()
Expand All @@ -1308,21 +1288,19 @@ def test_append_blob_from_stream_chunked_upload_with_count_parallel(self, **kwar
self._setup(bsc)
blob = self._create_blob(bsc)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
FILE_PATH = 'upload_with_count_parallel.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)

# Act
blob_size = len(data) - 301
with open(FILE_PATH, 'rb') as stream:
append_resp = blob.upload_blob(stream, length=blob_size, blob_type=BlobType.AppendBlob)
with tempfile.TemporaryFile() as temp_file:
temp_file.write(data)
temp_file.seek(0)
append_resp = blob.upload_blob(temp_file, length=blob_size, blob_type=BlobType.AppendBlob)
blob_properties = blob.get_blob_properties()

# Assert
self.assertBlobEqual(blob, data[:blob_size])
assert blob_properties.etag == append_resp.get('etag')
assert blob_properties.last_modified == append_resp.get('last_modified')
self._teardown(FILE_PATH)

@BlobPreparer()
@recorded_by_proxy
Expand Down
Loading