From 36a2098d7aa5337af41fc2c11a4c5dbc97f00124 Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Fri, 20 Sep 2024 16:06:44 -0700 Subject: [PATCH 1/4] feat: add integration test for universe domain --- .kokoro/build.sh | 6 +++++ .kokoro/presubmit/system-3.8.cfg | 6 +++++ owlbot.py | 11 +++++++- tests/system/_helpers.py | 4 +++ tests/system/conftest.py | 44 ++++++++++++++++++++++++++++++++ tests/system/test_client.py | 27 ++++++++++++++++++++ 6 files changed, 97 insertions(+), 1 deletion(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 5ac9f8a51..79346053a 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -34,6 +34,12 @@ export API_VERSION_OVERRIDE export DUAL_REGION_LOC_1 export DUAL_REGION_LOC_2 +# Setup universe domain testing needed environment variables. +export TEST_UNIVERSE_DOMAIN_CREDENTIAL=$(realpath ${KOKORO_GFILE_DIR}/secret_manager/client-library-test-universe-domain-credential) +export TEST_UNIVERSE_DOMAIN=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-domain) +export TEST_UNIVERSE_PROJECT_ID=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-project-id) +export TEST_UNIVERSE_LOCATION=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-storage-location) + # Debug: show build environment env | grep KOKORO diff --git a/.kokoro/presubmit/system-3.8.cfg b/.kokoro/presubmit/system-3.8.cfg index f4bcee3db..6d3603eed 100644 --- a/.kokoro/presubmit/system-3.8.cfg +++ b/.kokoro/presubmit/system-3.8.cfg @@ -4,4 +4,10 @@ env_vars: { key: "NOX_SESSION" value: "system-3.8" +} + +# Credentials needed to test universe domain. +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "client-library-test-universe-domain-credential" } \ No newline at end of file diff --git a/owlbot.py b/owlbot.py index a06ae8cc4..61871e3e4 100644 --- a/owlbot.py +++ b/owlbot.py @@ -46,6 +46,7 @@ "noxfile.py", "CONTRIBUTING.rst", "README.rst", + ".kokoro/presubmit/system-3.8.cfg", ".kokoro/samples/python3.6", # remove python 3.6 support ".github/blunderbuss.yml", # blunderbuss assignment to python squad ".github/workflows", # exclude gh actions as credentials are needed for tests @@ -66,7 +67,15 @@ # Export dual region locations export DUAL_REGION_LOC_1 -export DUAL_REGION_LOC_2""") +export DUAL_REGION_LOC_2 + +# Setup universe domain testing needed environment variables. +export TEST_UNIVERSE_DOMAIN_CREDENTIAL=$(realpath ${KOKORO_GFILE_DIR}/secret_manager/client-library-test-universe-domain-credential) +export TEST_UNIVERSE_DOMAIN=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-domain) +export TEST_UNIVERSE_PROJECT_ID=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-project-id) +export TEST_UNIVERSE_LOCATION=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-storage-location) + +""") s.replace( ".coveragerc", diff --git a/tests/system/_helpers.py b/tests/system/_helpers.py index a044c4ca8..7274610a8 100644 --- a/tests/system/_helpers.py +++ b/tests/system/_helpers.py @@ -31,6 +31,10 @@ user_project = os.environ.get("GOOGLE_CLOUD_TESTS_USER_PROJECT") testing_mtls = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE") == "true" +test_universe_domain = os.getenv("TEST_UNIVERSE_DOMAIN") +test_universe_project_id = os.getenv("TEST_UNIVERSE_PROJECT_ID") +test_universe_location = os.getenv("TEST_UNIVERSE_LOCATION") +test_universe_domain_credential = os.getenv("TEST_UNIVERSE_DOMAIN_CREDENTIAL") signing_blob_content = b"This time for sure, Rocky!" is_api_endpoint_override = ( _get_default_storage_base_url() != "https://storage.googleapis.com" diff --git a/tests/system/conftest.py b/tests/system/conftest.py index a97b98648..84d782372 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -331,3 +331,47 @@ def keyring(storage_client, kms_bucket, kms_client): except exceptions.NotFound: key = {"purpose": purpose} kms_client.create_crypto_key(keyring_path, key_name, key) + + +@pytest.fixture(scope="function") +def test_universe_domain(): + if _helpers.test_universe_domain is None: + pytest.skip("TEST_UNIVERSE_DOMAIN not set in environment.") + return _helpers.test_universe_domain + + +@pytest.fixture(scope="function") +def test_universe_project_id(): + if _helpers.test_universe_project_id is None: + pytest.skip("TEST_UNIVERSE_PROJECT_ID not set in environment.") + return _helpers.test_universe_domain_project + + +@pytest.fixture(scope="function") +def test_universe_location(): + if _helpers.test_universe_location is None: + pytest.skip("TEST_UNIVERSE_LOCATION not set in environment.") + return _helpers.test_universe_location + + +@pytest.fixture(scope="function") +def test_universe_domain_credential(): + if _helpers.test_universe_domain_credential is None: + pytest.skip("TEST_UNIVERSE_DOMAIN_CREDENTIAL not set in environment.") + return _helpers.test_universe_domain_credential + + +@pytest.fixture(scope="function") +def universe_domain_client( + test_universe_domain, test_universe_project_id, test_universe_domain_credential +): + from google.cloud.storage import Client + + client_options = {"universe_domain": test_universe_domain} + ud_storage_client = Client( + project=test_universe_project_id, + credentials=test_universe_domain_credential, + client_options=client_options, + ) + with contextlib.closing(ud_storage_client): + yield ud_storage_client diff --git a/tests/system/test_client.py b/tests/system/test_client.py index 70f341851..baf4556b7 100644 --- a/tests/system/test_client.py +++ b/tests/system/test_client.py @@ -184,3 +184,30 @@ def test_download_blob_to_file_w_etag( if_etag_match=blob.etag, ) assert buffer.getvalue() == payload + + +def test_client_universe_domain( + universe_domain_client, + test_universe_location, + buckets_to_delete, + blobs_to_delete, +): + bucket_name = _helpers.unique_name("gcp-systest-ud") + ud_bucket = universe_domain_client.create_bucket( + bucket_name, location=test_universe_location + ) + buckets_to_delete.append(ud_bucket) + + blob_name = _helpers.unique_name("gcp-systest-ud") + blob = ud_bucket.blob(blob_name) + payload = b"The quick brown fox jumps over the lazy dog" + blob.upload_from_string(payload) + blobs_to_delete.append(blob) + + with tempfile.NamedTemporaryFile() as temp_f: + with open(temp_f.name, "wb") as file_obj: + universe_domain_client.download_blob_to_file(blob, file_obj) + with open(temp_f.name, "rb") as file_obj: + stored_contents = file_obj.read() + + assert stored_contents == payload From a553b689de9af02a2fd6caa833275efbbc530c1d Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Fri, 20 Sep 2024 16:52:01 -0700 Subject: [PATCH 2/4] update fixture --- tests/system/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system/conftest.py b/tests/system/conftest.py index 84d782372..f33c905f2 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -344,7 +344,7 @@ def test_universe_domain(): def test_universe_project_id(): if _helpers.test_universe_project_id is None: pytest.skip("TEST_UNIVERSE_PROJECT_ID not set in environment.") - return _helpers.test_universe_domain_project + return _helpers.test_universe_project_id @pytest.fixture(scope="function") From c331a23bd046e7331af97b7ad6ecf984563df184 Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Fri, 20 Sep 2024 17:32:48 -0700 Subject: [PATCH 3/4] add fixture for credentials from sa file --- tests/system/conftest.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/system/conftest.py b/tests/system/conftest.py index f33c905f2..4ec56176d 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -361,16 +361,25 @@ def test_universe_domain_credential(): return _helpers.test_universe_domain_credential +@pytest.fixture(scope="function") +def universe_domain_credential(test_universe_domain_credential): + from google.oauth2 import service_account + + return service_account.Credentials.from_service_account_file( + test_universe_domain_credential + ) + + @pytest.fixture(scope="function") def universe_domain_client( - test_universe_domain, test_universe_project_id, test_universe_domain_credential + test_universe_domain, test_universe_project_id, universe_domain_credential ): from google.cloud.storage import Client client_options = {"universe_domain": test_universe_domain} ud_storage_client = Client( project=test_universe_project_id, - credentials=test_universe_domain_credential, + credentials=universe_domain_credential, client_options=client_options, ) with contextlib.closing(ud_storage_client): From 5b1e821bad0762fa7b26d9775355ca306b57d9f4 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 24 Sep 2024 17:17:44 +0000 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .kokoro/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 79346053a..fdc6d0271 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -40,6 +40,8 @@ export TEST_UNIVERSE_DOMAIN=$(gcloud secrets versions access latest --project cl export TEST_UNIVERSE_PROJECT_ID=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-project-id) export TEST_UNIVERSE_LOCATION=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-storage-location) + + # Debug: show build environment env | grep KOKORO