Skip to content

[Cleanup] storage/s3-sdk: Adds docstrings, return value instead of print #9948

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
May 17, 2023
25 changes: 17 additions & 8 deletions storage/s3-sdk/list_gcs_buckets.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,19 @@


def list_gcs_buckets(google_access_key_id, google_access_key_secret):
"""Lists all GCS buckets using boto3 SDK"""
# Create a new client and do the following:
# 1. Change the endpoint URL to use the
# Google Cloud Storage XML API endpoint.
# 2. Use Cloud Storage HMAC Credentials.
"""Lists all Cloud Storage buckets using AWS SDK for Python (boto3)
Positional arguments:
google_access_key_id -- hash-based message authentication code (HMAC) access ID
google_access_key_secret -- HMAC access secret

Returned value is a list of strings, one for each bucket name.

To use this sample:
1. Create a Cloud Storage HMAC key: https://cloud.google.com/storage/docs/authentication/managing-hmackeys#create
2. Change endpoint_url to a Google Cloud Storage XML API endpoint.

To learn more about HMAC: https://cloud.google.com/storage/docs/authentication/hmackeys#overview
"""
client = boto3.client(
"s3",
region_name="auto",
Expand All @@ -37,10 +45,11 @@ def list_gcs_buckets(google_access_key_id, google_access_key_secret):
# Call GCS to list current buckets
response = client.list_buckets()

# Print bucket names
print("Buckets:")
# Return list of bucket names
results = []
for bucket in response["Buckets"]:
print(bucket["Name"])
results.append(bucket["Name"])
return results


# [END storage_s3_sdk_list_buckets]
Expand Down
25 changes: 17 additions & 8 deletions storage/s3-sdk/list_gcs_objects.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,20 @@


def list_gcs_objects(google_access_key_id, google_access_key_secret, bucket_name):
"""Lists GCS objects using boto3 SDK"""
# Create a new client and do the following:
# 1. Change the endpoint URL to use the
# Google Cloud Storage XML API endpoint.
# 2. Use Cloud Storage HMAC Credentials.
"""Lists all Cloud Storage objects using AWS SDK for Python (boto3)
Positional arguments:
google_access_key_id -- hash-based message authentication code (HMAC) access ID
google_access_key_secret -- HMAC access secret
bucket_name -- name of Cloud Storage bucket

Returned value is a list of strings, one for each object in the bucket.

To use this sample:
1. Create a Cloud Storage HMAC key: https://cloud.google.com/storage/docs/authentication/managing-hmackeys#create
2. Change endpoint_url to a Google Cloud Storage XML API endpoint.

To learn more about HMAC: https://cloud.google.com/storage/docs/authentication/hmackeys#overview
"""
client = boto3.client(
"s3",
region_name="auto",
Expand All @@ -38,10 +46,11 @@ def list_gcs_objects(google_access_key_id, google_access_key_secret, bucket_name
# Call GCS to list objects in bucket_name
response = client.list_objects(Bucket=bucket_name)

# Print object names
print("Objects:")
# Return list of object names in bucket
results = []
for blob in response["Contents"]:
print(blob["Key"])
results.append(blob["Key"])
return results


# [END storage_s3_sdk_list_objects]
Expand Down
6 changes: 3 additions & 3 deletions storage/s3-sdk/noxfile_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,13 @@ def get_service_account_email():
TEST_CONFIG_OVERRIDE = {
# A dictionary you want to inject into your test. Don't put any
# secrets here. These values will override predefined values.
'envs': {
'HMAC_KEY_TEST_SERVICE_ACCOUNT': get_service_account_email(),
"envs": {
"HMAC_KEY_TEST_SERVICE_ACCOUNT": get_service_account_email(),
# Some tests can not use multiple projects because of several reasons:
# 1. The new projects is enforced to have the
# 'constraints/iam.disableServiceAccountKeyCreation' policy.
# 2. The new projects buckets need to have universal permission model.
# For those tests, we'll use the original project.
'MAIN_GOOGLE_CLOUD_PROJECT': 'python-docs-samples-tests'
"MAIN_GOOGLE_CLOUD_PROJECT": "python-docs-samples-tests",
},
}
12 changes: 4 additions & 8 deletions storage/s3-sdk/s3_sdk_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,24 +69,20 @@ def test_blob(test_bucket):
# to 15s.
@backoff.on_exception(backoff.constant, ClientError, interval=1, max_time=15)
def test_list_buckets(capsys, hmac_fixture, test_bucket):
list_gcs_buckets.list_gcs_buckets(
results = list_gcs_buckets.list_gcs_buckets(
google_access_key_id=hmac_fixture[0].access_id,
google_access_key_secret=hmac_fixture[1],
)
out, _ = capsys.readouterr()
assert "Buckets:" in out
assert test_bucket.name in out
assert test_bucket.name in results


# Retry request because the created key may not be fully propagated for up
# to 15s.
@backoff.on_exception(backoff.constant, ClientError, interval=1, max_time=15)
def test_list_blobs(capsys, hmac_fixture, test_bucket, test_blob):
list_gcs_objects.list_gcs_objects(
result = list_gcs_objects.list_gcs_objects(
google_access_key_id=hmac_fixture[0].access_id,
google_access_key_secret=hmac_fixture[1],
bucket_name=test_bucket.name,
)
out, _ = capsys.readouterr()
assert "Objects:" in out
assert test_blob.name in out
assert test_blob.name in result