From 25ea6a58b9436dd53f37780bfe5fbe8af94ebc01 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 8 Feb 2021 10:30:24 +1100 Subject: [PATCH 01/16] WIP - retry-based django e2e testing --- run/django/e2e_test.py | 406 -------------------------- run/django/test/e2e_test.py | 222 ++++++++++++++ run/django/test/e2e_test_cleanup.yaml | 51 ++++ run/django/test/e2e_test_setup.yaml | 171 +++++++++++ run/django/test/retry.sh | 67 +++++ 5 files changed, 511 insertions(+), 406 deletions(-) delete mode 100644 run/django/e2e_test.py create mode 100644 run/django/test/e2e_test.py create mode 100644 run/django/test/e2e_test_cleanup.yaml create mode 100644 run/django/test/e2e_test_setup.yaml create mode 100755 run/django/test/retry.sh diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py deleted file mode 100644 index 9a161d5a118..00000000000 --- a/run/django/e2e_test.py +++ /dev/null @@ -1,406 +0,0 @@ -# Copyright 2020 Google, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This test creates a Cloud SQL instance, a Cloud Storage bucket, associated -# secrets, and deploys a Django service - -import os -import subprocess -from typing import Iterator, List, Tuple -import uuid - -from google.cloud import secretmanager_v1 as sm -import pytest -import requests - -# Unique suffix to create distinct service names -SUFFIX = uuid.uuid4().hex[:10] - -PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] -REGION = "us-central1" -POSTGRES_INSTANCE = os.environ["POSTGRES_INSTANCE"] - -# Most commands in this test require the short instance form -if ":" in POSTGRES_INSTANCE: - POSTGRES_INSTANCE = POSTGRES_INSTANCE.split(":")[-1] - -CLOUD_STORAGE_BUCKET = f"{PROJECT}-media-{SUFFIX}" - -POSTGRES_DATABASE = f"polls-{SUFFIX}" -POSTGRES_USER = f"django-{SUFFIX}" -POSTGRES_PASSWORD = uuid.uuid4().hex[:26] - -ADMIN_NAME = "admin" -ADMIN_PASSWORD = uuid.uuid4().hex[:26] - -SECRET_SETTINGS_NAME = f"django_settings-{SUFFIX}" -SECRET_PASSWORD_NAME = f"superuser_password-{SUFFIX}" - - -@pytest.fixture -def project_number() -> Iterator[str]: - projectnum = ( - subprocess.run( - [ - "gcloud", - "projects", - "list", - "--filter", - f"name={PROJECT}", - "--format", - "value(projectNumber)", - ], - stdout=subprocess.PIPE, - check=True, - ) - .stdout.strip() - .decode() - ) - yield projectnum - - -@pytest.fixture -def postgres_host() -> Iterator[str]: - # Create database - subprocess.run( - [ - "gcloud", - "sql", - "databases", - "create", - POSTGRES_DATABASE, - "--instance", - POSTGRES_INSTANCE, - "--project", - PROJECT, - ], - check=True, - ) - # Create User - # NOTE Creating a user via the tutorial method is not automatable. - subprocess.run( - [ - "gcloud", - "sql", - "users", - "create", - POSTGRES_USER, - "--password", - POSTGRES_PASSWORD, - "--instance", - POSTGRES_INSTANCE, - "--project", - PROJECT, - ], - check=True, - ) - yield POSTGRES_INSTANCE - - subprocess.run( - [ - "gcloud", - "sql", - "databases", - "delete", - POSTGRES_DATABASE, - "--instance", - POSTGRES_INSTANCE, - "--project", - PROJECT, - "--quiet", - ], - check=True, - ) - - subprocess.run( - [ - "gcloud", - "sql", - "users", - "delete", - POSTGRES_USER, - "--instance", - POSTGRES_INSTANCE, - "--project", - PROJECT, - "--quiet", - ], - check=True, - ) - - -@pytest.fixture -def media_bucket() -> Iterator[str]: - # Create storage bucket - subprocess.run( - ["gsutil", "mb", "-l", REGION, "-p", PROJECT, f"gs://{CLOUD_STORAGE_BUCKET}"], - check=True, - ) - - yield CLOUD_STORAGE_BUCKET - - # Recursively delete assets and bucket (does not take a -p flag, apparently) - subprocess.run( - ["gsutil", "-m", "rm", "-r", f"gs://{CLOUD_STORAGE_BUCKET}"], - check=True, - ) - - -@pytest.fixture -def secrets(project_number: str) -> Iterator[str]: - # Create a number of secrets and allow Google Cloud services access to them - - def create_secret(name: str, value: str) -> None: - secret = client.create_secret( - request={ - "parent": f"projects/{PROJECT}", - "secret": {"replication": {"automatic": {}}}, - "secret_id": name, - } - ) - - client.add_secret_version( - request={"parent": secret.name, "payload": {"data": value.encode("UTF-8")}} - ) - - def allow_access(name: str, member: str) -> None: - subprocess.run( - [ - "gcloud", - "secrets", - "add-iam-policy-binding", - name, - "--member", - member, - "--role", - "roles/secretmanager.secretAccessor", - "--project", - PROJECT, - ], - check=True, - ) - - client = sm.SecretManagerServiceClient() - secret_key = uuid.uuid4().hex[:56] - settings = f""" -DATABASE_URL=postgres://{POSTGRES_USER}:{POSTGRES_PASSWORD}@//cloudsql/{PROJECT}:{REGION}:{POSTGRES_INSTANCE}/{POSTGRES_DATABASE} -GS_BUCKET_NAME={CLOUD_STORAGE_BUCKET} -SECRET_KEY={secret_key} -PASSWORD_NAME={SECRET_PASSWORD_NAME} - """ - - create_secret(SECRET_SETTINGS_NAME, settings) - allow_access( - SECRET_SETTINGS_NAME, - f"serviceAccount:{project_number}-compute@developer.gserviceaccount.com", - ) - allow_access( - SECRET_SETTINGS_NAME, - f"serviceAccount:{project_number}@cloudbuild.gserviceaccount.com", - ) - - create_secret(SECRET_PASSWORD_NAME, ADMIN_PASSWORD) - allow_access( - SECRET_PASSWORD_NAME, - f"serviceAccount:{project_number}@cloudbuild.gserviceaccount.com", - ) - - yield SECRET_SETTINGS_NAME - - # delete secrets - subprocess.run( - [ - "gcloud", - "secrets", - "delete", - SECRET_PASSWORD_NAME, - "--project", - PROJECT, - "--quiet", - ], - check=True, - ) - subprocess.run( - [ - "gcloud", - "secrets", - "delete", - SECRET_SETTINGS_NAME, - "--project", - PROJECT, - "--quiet", - ], - check=True, - ) - - -@pytest.fixture -def container_image(postgres_host: str, media_bucket: str, secrets: str) -> Iterator[str]: - # Build container image for Cloud Run deployment - image_name = f"gcr.io/{PROJECT}/polls-{SUFFIX}" - service_name = f"polls-{SUFFIX}" - cloudbuild_config = "cloudmigrate.yaml" - subprocess.run( - [ - "gcloud", - "builds", - "submit", - "--config", - cloudbuild_config, - "--substitutions", - ( - f"_INSTANCE_NAME={postgres_host}," - f"_REGION={REGION}," - f"_SERVICE_NAME={service_name}," - f"_SECRET_SETTINGS_NAME={SECRET_SETTINGS_NAME}" - ), - "--project", - PROJECT, - ], - check=True, - ) - yield image_name - - # Delete container image - subprocess.run( - [ - "gcloud", - "container", - "images", - "delete", - image_name, - "--quiet", - "--project", - PROJECT, - ], - check=True, - ) - - -@pytest.fixture -def deployed_service(container_image: str) -> Iterator[str]: - # Deploy image to Cloud Run - service_name = f"polls-{SUFFIX}" - subprocess.run( - [ - "gcloud", - "run", - "deploy", - service_name, - "--image", - container_image, - "--platform=managed", - "--no-allow-unauthenticated", - "--region", - REGION, - "--add-cloudsql-instances", - f"{PROJECT}:{REGION}:{POSTGRES_INSTANCE}", - "--set-env-vars", - f"SETTINGS_NAME={SECRET_SETTINGS_NAME}", - "--project", - PROJECT, - ], - check=True, - ) - yield service_name - - # Delete Cloud Run service - subprocess.run( - [ - "gcloud", - "run", - "services", - "delete", - service_name, - "--platform=managed", - "--region=us-central1", - "--quiet", - "--project", - PROJECT, - ], - check=True, - ) - - -@pytest.fixture -def service_url_auth_token(deployed_service: str) -> Iterator[Tuple[str, str]]: - # Get Cloud Run service URL and auth token - service_url = ( - subprocess.run( - [ - "gcloud", - "run", - "services", - "describe", - deployed_service, - "--platform", - "managed", - "--region", - REGION, - "--format", - "value(status.url)", - "--project", - PROJECT, - ], - stdout=subprocess.PIPE, - check=True, - ) - .stdout.strip() - .decode() - ) - auth_token = ( - subprocess.run( - ["gcloud", "auth", "print-identity-token", "--project", PROJECT], - stdout=subprocess.PIPE, - check=True, - ) - .stdout.strip() - .decode() - ) - - yield service_url, auth_token - - # no deletion needed - - -def test_end_to_end(service_url_auth_token: List[str]) -> None: - service_url, auth_token = service_url_auth_token - headers = {"Authorization": f"Bearer {auth_token}"} - login_slug = "/admin/login/?next=/admin/" - client = requests.session() - - # Check homepage - response = client.get(service_url, headers=headers) - body = response.text - - assert response.status_code == 200 - assert "Hello, world" in body - - # Load login page, collecting csrf token - client.get(service_url + login_slug, headers=headers) - csrftoken = client.cookies["csrftoken"] - - # Log into Django admin - payload = { - "username": ADMIN_NAME, - "password": ADMIN_PASSWORD, - "csrfmiddlewaretoken": csrftoken, - } - response = client.post(service_url + login_slug, data=payload, headers=headers) - body = response.text - - # Check Django admin landing page - assert response.status_code == 200 - assert "Site administration" in body - assert "Polls" in body diff --git a/run/django/test/e2e_test.py b/run/django/test/e2e_test.py new file mode 100644 index 00000000000..03e6f7a01c8 --- /dev/null +++ b/run/django/test/e2e_test.py @@ -0,0 +1,222 @@ +# Copyright 2020 Google, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This test creates a Cloud SQL instance, a Cloud Storage bucket, associated +# secrets, and deploys a Django service + +import os +import subprocess +from typing import Iterator, List, Tuple +import uuid + +import pytest +import requests + +# Unique suffix to create distinct service names +SUFFIX = uuid.uuid4().hex[:10] + +SAMPLE_VERSION = os.environ.get("SAMPLE_VERSION", None) +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +REGION = "us-central1" + +SERVICE_NAME = f"polls-{SUFFIX}" + +# Retreieve Cloud SQL test config +POSTGRES_INSTANCE = os.environ.get("POSTGRES_INSTANCE", None) +if not POSTGRES_INSTANCE: + raise Exception("'POSTGRES_INSTANCE' env var not found") + +# Presuming POSTGRES_INSTANCE comes in the form project:region:instance +# Require the short form in some cases. +# POSTGRES_INSTANCE_FULL: project:region:instance +# POSTGRES_INSTANCE_NAME: instance only +if ":" in POSTGRES_INSTANCE: + POSTGRES_INSTANCE_FULL = POSTGRES_INSTANCE + POSTGRES_INSTANCE_NAME = POSTGRES_INSTANCE.split(":")[-1] +else: + POSTGRES_INSTANCE_FULL = f"{GOOGLE_CLOUD_PROJECT}:{REGION}:{POSTGRES_INSTANCE}" + POSTGRES_INSTANCE_NAME = POSTGRES_INSTANCE + +POSTGRES_DATABASE = f"django-database-{SUFFIX}" + +CLOUD_STORAGE_BUCKET = f"{PROJECT}-media-{SUFFIX}" + +POSTGRES_DATABASE = f"polls-{SUFFIX}" +POSTGRES_USER = f"django-{SUFFIX}" +POSTGRES_PASSWORD = uuid.uuid4().hex[:26] + +ADMIN_NAME = "admin" +ADMIN_PASSWORD = uuid.uuid4().hex[:26] + +SECRET_SETTINGS_NAME = f"django_settings-{SUFFIX}" +SECRET_PASSWORD_NAME = f"superuser_password-{SUFFIX}" + +@pytest.fixture +def deployed_service() -> str: + substitutions = [ + f"_SERVICE={SERVICE_NAME}," + f"_PLATFORM={PLATFORM}," + f"_REGION={REGION}," + f"_STORAGE_BUCKET={CLOUD_STORAGE_BUCKET}," + f"_DB_NAME={POSTGRES_DATABASE}," + f"_DB_USER={POSTGRES_USER}," + f"_DB_PASS={POSTGRES_PASSWORD}," + f"_DB_INSTANCE={POSTGRES_INSTANCE_NAME}," + f"_SECRET_SETTINGS_NAME={SECRET_SETTINGS_NAME}," + f"_SECRET_PASSWORD_NAME={SECRET_PASSWORD_NAME}," + f"_SECRET_PASSWORD_VALUE={ADMIN_PASSWORD}," + f"_CLOUD_SQL_CONNECTION_NAME={POSTGRES_INSTANCE_FULL}," + ] + if SAMPLE_VERSION: + substitutions.append(f"_VERSION={SAMPLE_VERSION}") + + subprocess.run( + [ + "gcloud", + "builds", + "submit", + "--project", + GOOGLE_CLOUD_PROJECT, + "--config", + "./test/e2e_test_setup.yaml", + "--substitutions", + ] + + substitutions, + check=True, + ) + + service_url = ( + subprocess.run( + [ + "gcloud", + "run", + "services", + "describe", + SERVICE_NAME, + "--project", + GOOGLE_CLOUD_PROJECT, + "--platform", + PLATFORM, + "--region", + REGION, + "--format", + "value(status.url)", + ], + stdout=subprocess.PIPE, + check=True, + ) + .stdout.strip() + .decode() + ) + + yield service_url + + # Cleanup + + substitutions = [ + f"_SERVICE={SERVICE_NAME}," + f"_PLATFORM={PLATFORM}," + f"_REGION={REGION}," + f"_DB_NAME={POSTGRES_DATABASE}," + f"_DB_INSTANCE={POSTGRES_INSTANCE_NAME}," + ] + if SAMPLE_VERSION: + substitutions.append(f"_SAMPLE_VERSION={SAMPLE_VERSION}") + + subprocess.run( + [ + "gcloud", + "builds", + "submit", + "--project", + GOOGLE_CLOUD_PROJECT, + "--config", + "./test/e2e_test_cleanup.yaml", + "--substitutions", + ] + + substitutions, + check=True, + ) + +@pytest.fixture +def service_url_auth_token(deployed_service: str) -> Iterator[Tuple[str, str]]: + # Get Cloud Run service URL and auth token + service_url = ( + subprocess.run( + [ + "gcloud", + "run", + "services", + "describe", + deployed_service, + "--platform", + "managed", + "--region", + REGION, + "--format", + "value(status.url)", + "--project", + PROJECT, + ], + stdout=subprocess.PIPE, + check=True, + ) + .stdout.strip() + .decode() + ) + auth_token = ( + subprocess.run( + ["gcloud", "auth", "print-identity-token", "--project", PROJECT], + stdout=subprocess.PIPE, + check=True, + ) + .stdout.strip() + .decode() + ) + + yield service_url, auth_token + + # no deletion needed + + +def test_end_to_end(service_url_auth_token: List[str]) -> None: + service_url, auth_token = service_url_auth_token + headers = {"Authorization": f"Bearer {auth_token}"} + login_slug = "/admin/login/?next=/admin/" + client = requests.session() + + # Check homepage + response = client.get(service_url, headers=headers) + body = response.text + + assert response.status_code == 200 + assert "Hello, world" in body + + # Load login page, collecting csrf token + client.get(service_url + login_slug, headers=headers) + csrftoken = client.cookies["csrftoken"] + + # Log into Django admin + payload = { + "username": ADMIN_NAME, + "password": ADMIN_PASSWORD, + "csrfmiddlewaretoken": csrftoken, + } + response = client.post(service_url + login_slug, data=payload, headers=headers) + body = response.text + + # Check Django admin landing page + assert response.status_code == 200 + assert "Site administration" in body + assert "Polls" in body diff --git a/run/django/test/e2e_test_cleanup.yaml b/run/django/test/e2e_test_cleanup.yaml new file mode 100644 index 00000000000..1556abae541 --- /dev/null +++ b/run/django/test/e2e_test_cleanup.yaml @@ -0,0 +1,51 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + +- id: 'Delete resources' + name: 'gcr.io/cloud-builders/gcloud' + entrypoint: '/bin/bash' + args: + - '-c' + - | + ./test/retry.sh "gcloud secrets describe ${_SECRET_SETTINGS_NAME}" \ + "gcloud secrets delete ${_SECRET_SETTINGS_NAME} --quiet --project $PROJECT_ID" + + ./test/retry.sh "gcloud secrets describe ${_SECRET_PASSWORD_NAME}" \ + "gcloud secrets delete ${_SECRET_PASSWORD_NAME} --quiet --project $PROJECT_ID" + + ./test/retry.sh "gcloud container images describe gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" \ + "gcloud container images delete gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} --quiet" + + ./test/retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM}" \ + "gcloud run services delete ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM} --quiet" + + ./test/retry.sh "gcloud sql databases describe ${_DB_NAME} --instance ${_DB_INSTANCE} --project $PROJECT_ID" \ + "gcloud sql databases delete ${_DB_NAME} --instance ${_DB_INSTANCE} --quiet --project $PROJECT_ID" + + ./test/retry.sh "gcloud sql users describe ${_DB_USER} --instance ${_DB_INSTANCE} --project $PROJECT_ID" \ + "gcloud sql users delete ${_DB_USER} --instance ${_DB_INSTANCE} --quiet --project $PROJECT_ID" + + +substitutions: + _SERVICE: django + _VERSION: manual + _REGION: us-central1 + _PLATFORM: managed + _DB_NAME: django + _DB_USER: django + _DB_INSTANCE: django-instance + _SECRET_SETTINGS_NAME: django_settings + _SECRET_PASSWORD_NAME: admin_password diff --git a/run/django/test/e2e_test_setup.yaml b/run/django/test/e2e_test_setup.yaml new file mode 100644 index 00000000000..ddbdaca626e --- /dev/null +++ b/run/django/test/e2e_test_setup.yaml @@ -0,0 +1,171 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + - id: "Create a dedicated database" + name: "gcr.io/cloud-builders/gcloud" + entrypoint: "/bin/bash" + args: + - "-c" + - | + ./test/retry.sh "gcloud sql databases create ${_DB_NAME} \ + --instance ${_DB_INSTANCE} \ + --project ${PROJECT_ID}" \ + "gcloud sql databases describe ${_DB_NAME} --instance ${_DB_INSTANCE}" + + - id: "Create a dedicated database user" + name: "gcr.io/cloud-builders/gcloud" + entrypoint: "/bin/bash" + args: + - "-c" + - | + ./test/retry.sh "gcloud sql users create ${_DB_USER} \ + --instance ${_DB_INSTANCE} \ + --project ${PROJECT_ID}" \ + "gcloud sql users describe ${_DB_USER} --instance ${_DB_INSTANCE}" + + - id: "Create a dedicated storage bucket" + name: "gcr.io/cloud-builders/gcloud" + entrypoint: "/bin/bash" + args: + - "-c" + - | + ./test/retry.sh "gsutil mb \ + -l ${_REGION} \ + -p ${PROJECT_ID} \ + gs://${_STORAGE_BUCKET}" \ + "gsutil ls gs://${_STORAGE_BUCKET}" + + - id: "Add Django secrets to Secret Manager" + name: "gcr.io/cloud-builders/gcloud" + entrypoint: "/bin/bash" + args: + - "-c" + - | + echo " + DATABASE_URL=postgres://${_DB_USER}:${_DB_PASS}@//cloudsql/{PROJET_ID}:{_REGION}:{_DB_INSTANCE}/{POSTGRES_DATABASE} + GS_BUCKET_NAME=${_STORAGE_BUCKET} + SECRET_KEY=$(cat /dev/urandom | LC_ALL=C tr -dc '[:alpha:]' | fold -w 30 | head -n1)" > ${_SECRET_SETTINGS_NAME} + + ./test/retry.sh "gcloud secrets create ${_SECRET_SETTINGS_NAME} \ + --project $PROJECT_ID \ + --data-file=${_SECRET_SETTINGS_NAME}" \ + "gcloud secrets describe ${_SECRET_SETTINGS_NAME}" + + PROJECT_NUMBER=$(gcloud projects list --filter "name=${PROJET_ID}" --format "value(projectNumber)") + CLOUDBUILD_SA=serviceAccount:${PROJECT_NUMBER}@cloudbuild.gserviceaccount.com + + gcloud secrets add-iam-policy-binding ${_SECRET_SETTINGS_NAME} \ + --member ${CLOUDBUILD_SA} \ + --role roles/secretmanager.secretAccessor \ + --project ${PROJECT_ID} + + rm ${_SECRET_SETTINGS_NAME} + + cat ${_SECRET_PASSWORD_VALUE} > ${_SECRET_PASSWORD_NAME} + + gcloud secrets create ${_SECRET_PASSWORD_NAME} \ + --project ${PROJECT_ID} \ + --data-file=${_SECRET_PASSWORD_NAME} + + gcloud secrets add-iam-policy-binding ${_SECRET_PASSWORD_NAME} \ + --member $CLOUDBUILD_SA \ + --role roles/secretmanager.secretAccessor \ + --project ${PROJECT_ID} + + rm ${_SECRET_PASSWORD_NAME} + + - id: "Build Container Image" + name: "gcr.io/cloud-builders/docker" + entrypoint: "/bin/bash" + args: + - "-c" + - | + ./test/retry.sh "docker build -t gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} ." + + - id: "Push Container Image" + name: "gcr.io/cloud-builders/docker" + entrypoint: "/bin/bash" + args: + - "-c" + - | + ./test/retry.sh "docker push gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" + + - id: "Migrate database" + name: "gcr.io/google-appengine/exec-wrapper" + args: + [ + "-i", + "gcr.io/$PROJECT_ID/${_SERVICE_NAME}", + "-s", + "${PROJECT_ID}:${_REGION}:${_INSTANCE_NAME}", + "-e", + "SETTINGS_NAME=${_SECRET_SETTINGS_NAME}", + "--", + "python", + "manage.py", + "migrate", + ] + + - id: "Collect static" + name: "gcr.io/google-appengine/exec-wrapper" + args: + [ + "-i", + "gcr.io/$PROJECT_ID/${_SERVICE_NAME}", + "-s", + "${PROJECT_ID}:${_REGION}:${_INSTANCE_NAME}", + "-e", + "SETTINGS_NAME=${_SECRET_SETTINGS_NAME}", + "--", + "python", + "manage.py", + "collectstatic", + "--verbosity", + "2", + "--no-input", + ] + + - id: "Deploy to Cloud Run" + name: "gcr.io/cloud-builders/gcloud:latest" + entrypoint: /bin/bash + args: + - "-c" + - | + ./test/retry.sh "gcloud run deploy ${_SERVICE} \ + --project $PROJECT_ID \ + --image gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} \ + --no-allow-unauthenticated \ + --region ${_REGION} \ + --platform ${_PLATFORM} \ + --add-cloudsql-instances ${_CLOUD_SQL_CONNECTION_NAME} \ + --update-env-vars SETTINGS_NAME=${_SECRET_SETTINGS_NAME}" + +images: + - gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} + +substitutions: + _SERVICE: django + _VERSION: manual + _REGION: us-central1 + _PLATFORM: managed + _STORAGE_BUCKET: ${PROJECT_ID}-bucket + _DB_INSTANCE: django-instance + _CLOUD_SQL_CONNECTION_NAME: $PROJECT_ID:us-central1:django-instance + _DB_NAME: postgres + _DB_USER: postgres + _DB_PASS: password1234 + _SECRET_SETTINGS_NAME: django_settings + _SECRET_PASSWORD_NAME: admin_password + _SECRET_PASSWORD_VALUE: password diff --git a/run/django/test/retry.sh b/run/django/test/retry.sh new file mode 100755 index 00000000000..bca6fae257d --- /dev/null +++ b/run/django/test/retry.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## +# retry.sh +# Provides utility function commonly needed across Cloud Build pipelines to +# retry commands on failure. +# +# Usage: +# 1. Retry single command: +# +# ./retry.sh "CMD" +# +# 2. Retry with check: +# +# ./retry.sh "gcloud RESOURCE EXISTS?" "gcloud ACTION" +# +## + +# Usage: try "cmd1" "cmd2" +# If first cmd executes successfully then execute second cmd +runIfSuccessful() { + echo "running: $1" + $($1 > /dev/null) + if [ $? -eq 0 ]; then + echo "running: $2" + $($2 > /dev/null) + fi +} + +# Define max retries +max_attempts=3; +attempt_num=1; + +arg1="$1" +arg2="$2" + +if [ $# -eq 1 ] +then + cmd="$arg1" +else + cmd="runIfSuccessful \"$arg1\" \"$arg2\"" +fi + +until eval $cmd +do + if ((attempt_num==max_attempts)) + then + echo "Attempt $attempt_num / $max_attempts failed! No more retries left!" + exit + else + echo "Attempt $attempt_num / $max_attempts failed!" + sleep $((attempt_num++)) + fi +done From a155a0be7e6ddc1ff26e39567ce8cf811f30bfc8 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 8 Feb 2021 13:42:56 +1100 Subject: [PATCH 02/16] Fixes --- run/django/.gcloudignore | 1 + .../mysite/migrations/0001_createsuperuser.py | 34 ++++++------- run/django/mysite/settings.py | 50 +++++++++---------- run/django/test/e2e_test.py | 31 +++++++----- run/django/test/e2e_test_cleanup.yaml | 2 +- run/django/test/e2e_test_setup.yaml | 36 +++++++------ 6 files changed, 81 insertions(+), 73 deletions(-) diff --git a/run/django/.gcloudignore b/run/django/.gcloudignore index 4c49bd78f1d..2217a41e75b 100644 --- a/run/django/.gcloudignore +++ b/run/django/.gcloudignore @@ -1 +1,2 @@ .env +venv diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py index 149d21c19d1..8d58b666802 100644 --- a/run/django/mysite/migrations/0001_createsuperuser.py +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -15,11 +15,13 @@ # [START cloudrun_django_superuser] import os +from django.contrib.auth.models import User from django.db import migrations from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps + import google.auth -from google.cloud import secretmanager_v1 +from google.cloud import secretmanager def createsuperuser(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: @@ -27,32 +29,26 @@ def createsuperuser(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> Non Dynamically create an admin user as part of a migration Password is pulled from Secret Manger (previously created as part of tutorial) """ - if os.getenv("TRAMPOLINE_CI", None): - admin_password = "test" - else: - client = secretmanager_v1.SecretManagerServiceClient() - - # Get project value for identifying current context - _, project = google.auth.default() + # TODO(glasnt): removed trampoline CI for testing. + client = secretmanager.SecretManagerServiceClient() - # Retrieve the previously stored admin password - PASSWORD_NAME = os.environ.get("PASSWORD_NAME", "superuser_password") - name = f"projects/{project}/secrets/{PASSWORD_NAME}/versions/latest" - admin_password = client.access_secret_version(name=name).payload.data.decode( - "UTF-8" - ) + # Get project value for identifying current context + _, project = google.auth.default() - # Create a new user using acquired password - from django.contrib.auth.models import User + # Retrieve the previously stored admin password + PASSWORD_NAME = os.environ.get("PASSWORD_NAME", "superuser_password") + name = f"projects/{project}/secrets/{PASSWORD_NAME}/versions/latest" + admin_pass = client.access_secret_version(name=name).payload.data.decode("UTF-8") - User.objects.create_superuser("admin", password=admin_password) + # Create a new user using acquired password, stripping any accidentally stored newline characters + User.objects.create_superuser("admin", password=admin_pass.strip()) class Migration(migrations.Migration): initial = True - dependencies = [] - operations = [migrations.RunPython(createsuperuser)] + + # [END cloudrun_django_superuser] diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 3f160e827a0..dbd726a45eb 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -23,6 +23,7 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/3.0/ref/settings/ """ +import io import os import environ @@ -30,32 +31,31 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) env_file = os.path.join(BASE_DIR, ".env") -# If no .env has been provided, pull it from Secret Manager, storing it locally -if not os.path.isfile(".env"): - if os.getenv('TRAMPOLINE_CI', None): - payload = f"SECRET_KEY=a\nGS_BUCKET_NAME=none\nDATABASE_URL=sqlite://{os.path.join(BASE_DIR, 'db.sqlite3')}" - else: - # [START cloudrun_django_secretconfig] - import google.auth - from google.cloud import secretmanager_v1 - - _, project = google.auth.default() - - if project: - client = secretmanager_v1.SecretManagerServiceClient() - - SETTINGS_NAME = os.environ.get("SETTINGS_NAME", "django_settings") - name = f"projects/{project}/secrets/{SETTINGS_NAME}/versions/latest" - payload = client.access_secret_version(name=name).payload.data.decode( - "UTF-8" - ) - - with open(env_file, "w") as f: - f.write(payload) - -env = environ.Env() -env.read_env(env_file) +# If no .env has been provided, pull it from Secret Manager +if os.path.isfile(".env"): + env = environ.Env() + env.read_env(env_file) +else: + # TODO(glasnt) removed trampoline for testing + # [START cloudrun_django_secretconfig] + import google.auth + from google.cloud import secretmanager + + _, project = google.auth.default() + + if project: + client = secretmanager.SecretManagerServiceClient() + + SETTINGS_NAME = os.environ.get("SETTINGS_NAME", "django_settings") + name = f"projects/{project}/secrets/{SETTINGS_NAME}/versions/latest" + payload = client.access_secret_version(name=name).payload.data.decode( + "UTF-8" + ) + env = environ.Env() + env.read_env(io.StringIO(payload)) # [END cloudrun_django_secretconfig] + else: + raise Exception("No environment configuration found.") SECRET_KEY = env("SECRET_KEY") diff --git a/run/django/test/e2e_test.py b/run/django/test/e2e_test.py index 03e6f7a01c8..68b09d89043 100644 --- a/run/django/test/e2e_test.py +++ b/run/django/test/e2e_test.py @@ -1,4 +1,4 @@ -# Copyright 2020 Google, LLC. +# Copyright 2021 Google, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,10 +27,11 @@ SUFFIX = uuid.uuid4().hex[:10] SAMPLE_VERSION = os.environ.get("SAMPLE_VERSION", None) -PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +GOOGLE_CLOUD_PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] REGION = "us-central1" +PLATFORM = "managed" -SERVICE_NAME = f"polls-{SUFFIX}" +SERVICE = f"polls-{SUFFIX}" # Retreieve Cloud SQL test config POSTGRES_INSTANCE = os.environ.get("POSTGRES_INSTANCE", None) @@ -50,7 +51,7 @@ POSTGRES_DATABASE = f"django-database-{SUFFIX}" -CLOUD_STORAGE_BUCKET = f"{PROJECT}-media-{SUFFIX}" +CLOUD_STORAGE_BUCKET = f"{GOOGLE_CLOUD_PROJECT}-media-{SUFFIX}" POSTGRES_DATABASE = f"polls-{SUFFIX}" POSTGRES_USER = f"django-{SUFFIX}" @@ -64,8 +65,9 @@ @pytest.fixture def deployed_service() -> str: + substitutions = [ - f"_SERVICE={SERVICE_NAME}," + f"_SERVICE={SERVICE}," f"_PLATFORM={PLATFORM}," f"_REGION={REGION}," f"_STORAGE_BUCKET={CLOUD_STORAGE_BUCKET}," @@ -76,10 +78,10 @@ def deployed_service() -> str: f"_SECRET_SETTINGS_NAME={SECRET_SETTINGS_NAME}," f"_SECRET_PASSWORD_NAME={SECRET_PASSWORD_NAME}," f"_SECRET_PASSWORD_VALUE={ADMIN_PASSWORD}," - f"_CLOUD_SQL_CONNECTION_NAME={POSTGRES_INSTANCE_FULL}," + f"_CLOUD_SQL_CONNECTION_NAME={POSTGRES_INSTANCE_FULL}" ] if SAMPLE_VERSION: - substitutions.append(f"_VERSION={SAMPLE_VERSION}") + substitutions.append(f",_VERSION={SAMPLE_VERSION}") subprocess.run( [ @@ -103,7 +105,7 @@ def deployed_service() -> str: "run", "services", "describe", - SERVICE_NAME, + SERVICE, "--project", GOOGLE_CLOUD_PROJECT, "--platform", @@ -120,16 +122,19 @@ def deployed_service() -> str: .decode() ) - yield service_url + yield SERVICE # Cleanup substitutions = [ - f"_SERVICE={SERVICE_NAME}," + f"_SERVICE={SERVICE}," f"_PLATFORM={PLATFORM}," f"_REGION={REGION}," f"_DB_NAME={POSTGRES_DATABASE}," + f"_DB_USER={POSTGRES_USER}," f"_DB_INSTANCE={POSTGRES_INSTANCE_NAME}," + f"_SECRET_SETTINGS_NAME={SECRET_SETTINGS_NAME}," + f"_SECRET_PASSWORD_NAME={SECRET_PASSWORD_NAME}," ] if SAMPLE_VERSION: substitutions.append(f"_SAMPLE_VERSION={SAMPLE_VERSION}") @@ -148,6 +153,7 @@ def deployed_service() -> str: + substitutions, check=True, ) + @pytest.fixture def service_url_auth_token(deployed_service: str) -> Iterator[Tuple[str, str]]: @@ -167,7 +173,7 @@ def service_url_auth_token(deployed_service: str) -> Iterator[Tuple[str, str]]: "--format", "value(status.url)", "--project", - PROJECT, + GOOGLE_CLOUD_PROJECT, ], stdout=subprocess.PIPE, check=True, @@ -177,7 +183,7 @@ def service_url_auth_token(deployed_service: str) -> Iterator[Tuple[str, str]]: ) auth_token = ( subprocess.run( - ["gcloud", "auth", "print-identity-token", "--project", PROJECT], + ["gcloud", "auth", "print-identity-token", "--project", GOOGLE_CLOUD_PROJECT], stdout=subprocess.PIPE, check=True, ) @@ -218,5 +224,6 @@ def test_end_to_end(service_url_auth_token: List[str]) -> None: # Check Django admin landing page assert response.status_code == 200 + assert "Please enter the correct username and password" not in body assert "Site administration" in body assert "Polls" in body diff --git a/run/django/test/e2e_test_cleanup.yaml b/run/django/test/e2e_test_cleanup.yaml index 1556abae541..858a35a4303 100644 --- a/run/django/test/e2e_test_cleanup.yaml +++ b/run/django/test/e2e_test_cleanup.yaml @@ -35,7 +35,7 @@ steps: ./test/retry.sh "gcloud sql databases describe ${_DB_NAME} --instance ${_DB_INSTANCE} --project $PROJECT_ID" \ "gcloud sql databases delete ${_DB_NAME} --instance ${_DB_INSTANCE} --quiet --project $PROJECT_ID" - ./test/retry.sh "gcloud sql users describe ${_DB_USER} --instance ${_DB_INSTANCE} --project $PROJECT_ID" \ + ./test/retry.sh "gcloud sql users list --filter \"name=${_DB_USER}\" --instance ${_DB_INSTANCE}" \ "gcloud sql users delete ${_DB_USER} --instance ${_DB_INSTANCE} --quiet --project $PROJECT_ID" diff --git a/run/django/test/e2e_test_setup.yaml b/run/django/test/e2e_test_setup.yaml index ddbdaca626e..77b40dd82b6 100644 --- a/run/django/test/e2e_test_setup.yaml +++ b/run/django/test/e2e_test_setup.yaml @@ -30,10 +30,13 @@ steps: args: - "-c" - | + echo -n "${_DB_PASS}" > db_password ./test/retry.sh "gcloud sql users create ${_DB_USER} \ + --password $(cat db_password) \ --instance ${_DB_INSTANCE} \ --project ${PROJECT_ID}" \ - "gcloud sql users describe ${_DB_USER} --instance ${_DB_INSTANCE}" + "gcloud sql users list --filter \"name=${_DB_USER}\" --instance ${_DB_INSTANCE}" + rm db_password - id: "Create a dedicated storage bucket" name: "gcr.io/cloud-builders/gcloud" @@ -54,33 +57,32 @@ steps: - "-c" - | echo " - DATABASE_URL=postgres://${_DB_USER}:${_DB_PASS}@//cloudsql/{PROJET_ID}:{_REGION}:{_DB_INSTANCE}/{POSTGRES_DATABASE} + DATABASE_URL=postgres://${_DB_USER}:${_DB_PASS}@//cloudsql/${_CLOUD_SQL_CONNECTION_NAME}/${_DB_NAME} GS_BUCKET_NAME=${_STORAGE_BUCKET} - SECRET_KEY=$(cat /dev/urandom | LC_ALL=C tr -dc '[:alpha:]' | fold -w 30 | head -n1)" > ${_SECRET_SETTINGS_NAME} + SECRET_KEY=$(cat /dev/urandom | LC_ALL=C tr -dc '[:alpha:]' | fold -w 30 | head -n1) + PASSWORD_NAME=${_SECRET_PASSWORD_NAME}" > ${_SECRET_SETTINGS_NAME} ./test/retry.sh "gcloud secrets create ${_SECRET_SETTINGS_NAME} \ --project $PROJECT_ID \ --data-file=${_SECRET_SETTINGS_NAME}" \ "gcloud secrets describe ${_SECRET_SETTINGS_NAME}" - PROJECT_NUMBER=$(gcloud projects list --filter "name=${PROJET_ID}" --format "value(projectNumber)") - CLOUDBUILD_SA=serviceAccount:${PROJECT_NUMBER}@cloudbuild.gserviceaccount.com - gcloud secrets add-iam-policy-binding ${_SECRET_SETTINGS_NAME} \ - --member ${CLOUDBUILD_SA} \ + --member serviceAccount:$(gcloud projects list --filter "name=${PROJECT_ID}" --format "value(projectNumber)")@cloudbuild.gserviceaccount.com \ --role roles/secretmanager.secretAccessor \ --project ${PROJECT_ID} rm ${_SECRET_SETTINGS_NAME} - cat ${_SECRET_PASSWORD_VALUE} > ${_SECRET_PASSWORD_NAME} + echo -n "${_SECRET_PASSWORD_VALUE}" > ${_SECRET_PASSWORD_NAME} - gcloud secrets create ${_SECRET_PASSWORD_NAME} \ - --project ${PROJECT_ID} \ - --data-file=${_SECRET_PASSWORD_NAME} + ./test/retry.sh "gcloud secrets create ${_SECRET_PASSWORD_NAME} \ + --project $PROJECT_ID \ + --data-file=${_SECRET_PASSWORD_NAME}" \ + "gcloud secrets describe ${_SECRET_PASSWORD_NAME}" gcloud secrets add-iam-policy-binding ${_SECRET_PASSWORD_NAME} \ - --member $CLOUDBUILD_SA \ + --member serviceAccount:$(gcloud projects list --filter "name=${PROJECT_ID}" --format "value(projectNumber)")@cloudbuild.gserviceaccount.com \ --role roles/secretmanager.secretAccessor \ --project ${PROJECT_ID} @@ -107,11 +109,13 @@ steps: args: [ "-i", - "gcr.io/$PROJECT_ID/${_SERVICE_NAME}", + "gcr.io/$PROJECT_ID/${_SERVICE}:${_VERSION}", "-s", - "${PROJECT_ID}:${_REGION}:${_INSTANCE_NAME}", + "${_CLOUD_SQL_CONNECTION_NAME}", "-e", "SETTINGS_NAME=${_SECRET_SETTINGS_NAME}", + "-e", + "PASSWORD_NAME=${_SECRET_PASSWORD_NAME}", "--", "python", "manage.py", @@ -123,9 +127,9 @@ steps: args: [ "-i", - "gcr.io/$PROJECT_ID/${_SERVICE_NAME}", + "gcr.io/$PROJECT_ID/${_SERVICE}:${_VERSION}", "-s", - "${PROJECT_ID}:${_REGION}:${_INSTANCE_NAME}", + "${_CLOUD_SQL_CONNECTION_NAME}", "-e", "SETTINGS_NAME=${_SECRET_SETTINGS_NAME}", "--", From 63a2092626a82bf4a509e7678747965b04e19e77 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 8 Feb 2021 13:54:45 +1100 Subject: [PATCH 03/16] rev trampoline --- run/django/mysite/settings.py | 35 +++++++++++++++++++---------------- run/django/test/e2e_test.py | 2 +- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index dbd726a45eb..6051da194c3 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -37,22 +37,25 @@ env.read_env(env_file) else: # TODO(glasnt) removed trampoline for testing - # [START cloudrun_django_secretconfig] - import google.auth - from google.cloud import secretmanager - - _, project = google.auth.default() - - if project: - client = secretmanager.SecretManagerServiceClient() - - SETTINGS_NAME = os.environ.get("SETTINGS_NAME", "django_settings") - name = f"projects/{project}/secrets/{SETTINGS_NAME}/versions/latest" - payload = client.access_secret_version(name=name).payload.data.decode( - "UTF-8" - ) - env = environ.Env() - env.read_env(io.StringIO(payload)) + if os.getenv('TRAMPOLINE_CI', None): + payload = f"SECRET_KEY=a\nGS_BUCKET_NAME=none\nDATABASE_URL=sqlite://{os.path.join(BASE_DIR, 'db.sqlite3')}" + else: + # [START cloudrun_django_secretconfig] + import google.auth + from google.cloud import secretmanager + + _, project = google.auth.default() + + if project: + client = secretmanager.SecretManagerServiceClient() + + SETTINGS_NAME = os.environ.get("SETTINGS_NAME", "django_settings") + name = f"projects/{project}/secrets/{SETTINGS_NAME}/versions/latest" + payload = client.access_secret_version(name=name).payload.data.decode( + "UTF-8" + ) + env = environ.Env() + env.read_env(io.StringIO(payload)) # [END cloudrun_django_secretconfig] else: raise Exception("No environment configuration found.") diff --git a/run/django/test/e2e_test.py b/run/django/test/e2e_test.py index 68b09d89043..e715ca7d8fc 100644 --- a/run/django/test/e2e_test.py +++ b/run/django/test/e2e_test.py @@ -1,4 +1,4 @@ -# Copyright 2021 Google, LLC. +# Copyright 2021 Google LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 5f35c07ec73313691807e48914da18dd60ed77d9 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 8 Feb 2021 14:04:14 +1100 Subject: [PATCH 04/16] format --- run/django/mysite/settings.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 6051da194c3..4b2f8a31755 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -36,8 +36,7 @@ env = environ.Env() env.read_env(env_file) else: - # TODO(glasnt) removed trampoline for testing - if os.getenv('TRAMPOLINE_CI', None): + if os.getenv("TRAMPOLINE_CI", None): payload = f"SECRET_KEY=a\nGS_BUCKET_NAME=none\nDATABASE_URL=sqlite://{os.path.join(BASE_DIR, 'db.sqlite3')}" else: # [START cloudrun_django_secretconfig] @@ -57,8 +56,7 @@ env = environ.Env() env.read_env(io.StringIO(payload)) # [END cloudrun_django_secretconfig] - else: - raise Exception("No environment configuration found.") + SECRET_KEY = env("SECRET_KEY") @@ -117,10 +115,18 @@ # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ - {"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, - {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", }, - {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", }, - {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", }, + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, ] From 1fd0c33dacc29c54a0522d4dd8f970ca2c96da3e Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 8 Feb 2021 14:12:50 +1100 Subject: [PATCH 05/16] revert --- .../mysite/migrations/0001_createsuperuser.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py index 8d58b666802..b41d45af4b1 100644 --- a/run/django/mysite/migrations/0001_createsuperuser.py +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -29,16 +29,18 @@ def createsuperuser(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> Non Dynamically create an admin user as part of a migration Password is pulled from Secret Manger (previously created as part of tutorial) """ - # TODO(glasnt): removed trampoline CI for testing. - client = secretmanager.SecretManagerServiceClient() - - # Get project value for identifying current context - _, project = google.auth.default() - - # Retrieve the previously stored admin password - PASSWORD_NAME = os.environ.get("PASSWORD_NAME", "superuser_password") - name = f"projects/{project}/secrets/{PASSWORD_NAME}/versions/latest" - admin_pass = client.access_secret_version(name=name).payload.data.decode("UTF-8") + if os.getenv("TRAMPOLINE_CI", None): + admin_password = "test" + else: + client = secretmanager.SecretManagerServiceClient() + + # Get project value for identifying current context + _, project = google.auth.default() + + # Retrieve the previously stored admin password + PASSWORD_NAME = os.environ.get("PASSWORD_NAME", "superuser_password") + name = f"projects/{project}/secrets/{PASSWORD_NAME}/versions/latest" + admin_pass = client.access_secret_version(name=name).payload.data.decode("UTF-8") # Create a new user using acquired password, stripping any accidentally stored newline characters User.objects.create_superuser("admin", password=admin_pass.strip()) From 99c49d77741da24616d200125e6ff9ce836d42ca Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 8 Feb 2021 14:13:07 +1100 Subject: [PATCH 06/16] fix --- run/django/mysite/migrations/0001_createsuperuser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py index b41d45af4b1..11b1c205eb7 100644 --- a/run/django/mysite/migrations/0001_createsuperuser.py +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -30,7 +30,7 @@ def createsuperuser(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> Non Password is pulled from Secret Manger (previously created as part of tutorial) """ if os.getenv("TRAMPOLINE_CI", None): - admin_password = "test" + admin_pass = "test" else: client = secretmanager.SecretManagerServiceClient() From 4d92d653a5d7a7b4003c3223d12efeb4a488836a Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 8 Feb 2021 15:00:42 +1100 Subject: [PATCH 07/16] Workaround: create user in worker, not cloudbuild permissions do not currently permit cloudbuild to create sqlusers --- run/django/test/e2e_test.py | 46 +++++++++++++++++++++++++-- run/django/test/e2e_test_cleanup.yaml | 4 --- run/django/test/e2e_test_setup.yaml | 14 -------- 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/run/django/test/e2e_test.py b/run/django/test/e2e_test.py index e715ca7d8fc..c20bb1e9511 100644 --- a/run/django/test/e2e_test.py +++ b/run/django/test/e2e_test.py @@ -63,8 +63,26 @@ SECRET_SETTINGS_NAME = f"django_settings-{SUFFIX}" SECRET_PASSWORD_NAME = f"superuser_password-{SUFFIX}" + @pytest.fixture def deployed_service() -> str: + # NOTE(glasnt): cloudbuild doesn't have user create rights. + subprocess.run( + [ + "gcloud", + "sql", + "users", + "create", + POSTGRES_USER, + "--password", + POSTGRES_PASSWORD, + "--instance", + POSTGRES_INSTANCE_NAME, + "--project", + GOOGLE_CLOUD_PROJECT, + ], + check=True, + ) substitutions = [ f"_SERVICE={SERVICE}," @@ -131,7 +149,6 @@ def deployed_service() -> str: f"_PLATFORM={PLATFORM}," f"_REGION={REGION}," f"_DB_NAME={POSTGRES_DATABASE}," - f"_DB_USER={POSTGRES_USER}," f"_DB_INSTANCE={POSTGRES_INSTANCE_NAME}," f"_SECRET_SETTINGS_NAME={SECRET_SETTINGS_NAME}," f"_SECRET_PASSWORD_NAME={SECRET_PASSWORD_NAME}," @@ -153,7 +170,24 @@ def deployed_service() -> str: + substitutions, check=True, ) - + + # Remove manually created database user. + subprocess.run( + [ + "gcloud", + "sql", + "users", + "delete", + POSTGRES_USER, + "--instance", + POSTGRES_INSTANCE_NAME, + "--project", + GOOGLE_CLOUD_PROJECT, + "--quiet", + ], + check=True, + ) + @pytest.fixture def service_url_auth_token(deployed_service: str) -> Iterator[Tuple[str, str]]: @@ -183,7 +217,13 @@ def service_url_auth_token(deployed_service: str) -> Iterator[Tuple[str, str]]: ) auth_token = ( subprocess.run( - ["gcloud", "auth", "print-identity-token", "--project", GOOGLE_CLOUD_PROJECT], + [ + "gcloud", + "auth", + "print-identity-token", + "--project", + GOOGLE_CLOUD_PROJECT, + ], stdout=subprocess.PIPE, check=True, ) diff --git a/run/django/test/e2e_test_cleanup.yaml b/run/django/test/e2e_test_cleanup.yaml index 858a35a4303..40a9b0528c6 100644 --- a/run/django/test/e2e_test_cleanup.yaml +++ b/run/django/test/e2e_test_cleanup.yaml @@ -34,9 +34,6 @@ steps: ./test/retry.sh "gcloud sql databases describe ${_DB_NAME} --instance ${_DB_INSTANCE} --project $PROJECT_ID" \ "gcloud sql databases delete ${_DB_NAME} --instance ${_DB_INSTANCE} --quiet --project $PROJECT_ID" - - ./test/retry.sh "gcloud sql users list --filter \"name=${_DB_USER}\" --instance ${_DB_INSTANCE}" \ - "gcloud sql users delete ${_DB_USER} --instance ${_DB_INSTANCE} --quiet --project $PROJECT_ID" substitutions: @@ -45,7 +42,6 @@ substitutions: _REGION: us-central1 _PLATFORM: managed _DB_NAME: django - _DB_USER: django _DB_INSTANCE: django-instance _SECRET_SETTINGS_NAME: django_settings _SECRET_PASSWORD_NAME: admin_password diff --git a/run/django/test/e2e_test_setup.yaml b/run/django/test/e2e_test_setup.yaml index 77b40dd82b6..4d1b08692f9 100644 --- a/run/django/test/e2e_test_setup.yaml +++ b/run/django/test/e2e_test_setup.yaml @@ -24,20 +24,6 @@ steps: --project ${PROJECT_ID}" \ "gcloud sql databases describe ${_DB_NAME} --instance ${_DB_INSTANCE}" - - id: "Create a dedicated database user" - name: "gcr.io/cloud-builders/gcloud" - entrypoint: "/bin/bash" - args: - - "-c" - - | - echo -n "${_DB_PASS}" > db_password - ./test/retry.sh "gcloud sql users create ${_DB_USER} \ - --password $(cat db_password) \ - --instance ${_DB_INSTANCE} \ - --project ${PROJECT_ID}" \ - "gcloud sql users list --filter \"name=${_DB_USER}\" --instance ${_DB_INSTANCE}" - rm db_password - - id: "Create a dedicated storage bucket" name: "gcr.io/cloud-builders/gcloud" entrypoint: "/bin/bash" From 56d350ac9a36473b78a759f980275accbafa16d4 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 8 Feb 2021 15:08:32 +1100 Subject: [PATCH 08/16] minor cleanup --- .../mysite/migrations/0001_createsuperuser.py | 1 + run/django/mysite/settings.py | 14 ++++++---- run/django/test/e2e_test.py | 28 ++----------------- 3 files changed, 11 insertions(+), 32 deletions(-) diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py index 11b1c205eb7..05ff4221801 100644 --- a/run/django/mysite/migrations/0001_createsuperuser.py +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -30,6 +30,7 @@ def createsuperuser(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> Non Password is pulled from Secret Manger (previously created as part of tutorial) """ if os.getenv("TRAMPOLINE_CI", None): + # We are in CI, so just create a placeholder user for unit testing. admin_pass = "test" else: client = secretmanager.SecretManagerServiceClient() diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 4b2f8a31755..b800d87cf84 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -31,13 +31,16 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) env_file = os.path.join(BASE_DIR, ".env") + +env = environ.Env() # If no .env has been provided, pull it from Secret Manager if os.path.isfile(".env"): - env = environ.Env() env.read_env(env_file) else: + # Create local settings if running with CI, for unit testing if os.getenv("TRAMPOLINE_CI", None): - payload = f"SECRET_KEY=a\nGS_BUCKET_NAME=none\nDATABASE_URL=sqlite://{os.path.join(BASE_DIR, 'db.sqlite3')}" + placeholder = f"SECRET_KEY=a\nGS_BUCKET_NAME=none\nDATABASE_URL=sqlite://{os.path.join(BASE_DIR, 'db.sqlite3')}" + env.read_env(io.StringIO(placeholder)) else: # [START cloudrun_django_secretconfig] import google.auth @@ -53,9 +56,8 @@ payload = client.access_secret_version(name=name).payload.data.decode( "UTF-8" ) - env = environ.Env() - env.read_env(io.StringIO(payload)) -# [END cloudrun_django_secretconfig] + env.read_env(io.StringIO(payload)) + # [END cloudrun_django_secretconfig] SECRET_KEY = env("SECRET_KEY") @@ -107,7 +109,7 @@ # [START cloudrun_django_dbconfig] -# Use django-environ to define the connection string +# Use django-environ to parse the connection string DATABASES = {"default": env.db()} # [END cloudrun_django_dbconfig] diff --git a/run/django/test/e2e_test.py b/run/django/test/e2e_test.py index c20bb1e9511..5d9a60e5cdd 100644 --- a/run/django/test/e2e_test.py +++ b/run/django/test/e2e_test.py @@ -66,7 +66,7 @@ @pytest.fixture def deployed_service() -> str: - # NOTE(glasnt): cloudbuild doesn't have user create rights. + # NOTE(glasnt): cloudbuild doesn't have user create rights. subprocess.run( [ "gcloud", @@ -116,30 +116,6 @@ def deployed_service() -> str: check=True, ) - service_url = ( - subprocess.run( - [ - "gcloud", - "run", - "services", - "describe", - SERVICE, - "--project", - GOOGLE_CLOUD_PROJECT, - "--platform", - PLATFORM, - "--region", - REGION, - "--format", - "value(status.url)", - ], - stdout=subprocess.PIPE, - check=True, - ) - .stdout.strip() - .decode() - ) - yield SERVICE # Cleanup @@ -171,7 +147,7 @@ def deployed_service() -> str: check=True, ) - # Remove manually created database user. + # Remove manually created database user. subprocess.run( [ "gcloud", From ea8555e1c54c45cf7a04dc43fde94213ce071e38 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 8 Feb 2021 15:15:35 +1100 Subject: [PATCH 09/16] black --- run/django/mysite/migrations/0001_createsuperuser.py | 10 ++++++---- run/django/mysite/settings.py | 12 +++--------- run/django/polls/test_polls.py | 4 ++-- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py index 05ff4221801..27a354ef691 100644 --- a/run/django/mysite/migrations/0001_createsuperuser.py +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -30,8 +30,8 @@ def createsuperuser(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> Non Password is pulled from Secret Manger (previously created as part of tutorial) """ if os.getenv("TRAMPOLINE_CI", None): - # We are in CI, so just create a placeholder user for unit testing. - admin_pass = "test" + # We are in CI, so just create a placeholder user for unit testing. + admin_password = "test" else: client = secretmanager.SecretManagerServiceClient() @@ -41,10 +41,12 @@ def createsuperuser(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> Non # Retrieve the previously stored admin password PASSWORD_NAME = os.environ.get("PASSWORD_NAME", "superuser_password") name = f"projects/{project}/secrets/{PASSWORD_NAME}/versions/latest" - admin_pass = client.access_secret_version(name=name).payload.data.decode("UTF-8") + admin_password = client.access_secret_version(name=name).payload.data.decode( + "UTF-8" + ) # Create a new user using acquired password, stripping any accidentally stored newline characters - User.objects.create_superuser("admin", password=admin_pass.strip()) + User.objects.create_superuser("admin", password=admin_password.strip()) class Migration(migrations.Migration): diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index b800d87cf84..9512b18cd81 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -120,15 +120,9 @@ { "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, - { - "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", - }, - { - "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", - }, - { - "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", - }, + {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",}, + {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",}, + {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",}, ] diff --git a/run/django/polls/test_polls.py b/run/django/polls/test_polls.py index e02f4a34f44..81085e6fb0c 100644 --- a/run/django/polls/test_polls.py +++ b/run/django/polls/test_polls.py @@ -19,6 +19,6 @@ class PollViewTests(TestCase): def test_index_view(self: PollViewTests) -> None: - response = self.client.get('/') + response = self.client.get("/") assert response.status_code == 200 - assert 'Hello, world' in str(response.content) + assert "Hello, world" in str(response.content) From 6f6023126579daceb30028f08ed20ec97fec53eb Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 8 Feb 2021 15:20:21 +1100 Subject: [PATCH 10/16] black is not flake8 --- run/django/mysite/settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 9512b18cd81..11332426390 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -120,9 +120,9 @@ { "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, - {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",}, - {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",}, - {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",}, + {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", }, + {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", }, + {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", }, ] From 4d947588914d3403ae59d385e796006fcd4d3675 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 8 Feb 2021 15:34:25 +1100 Subject: [PATCH 11/16] databases cannot be deleted by cloudbuild rn --- run/django/test/e2e_test.py | 18 +++++++++++++++++- run/django/test/e2e_test_cleanup.yaml | 3 --- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/run/django/test/e2e_test.py b/run/django/test/e2e_test.py index 5d9a60e5cdd..9c9cc26b5ca 100644 --- a/run/django/test/e2e_test.py +++ b/run/django/test/e2e_test.py @@ -147,7 +147,23 @@ def deployed_service() -> str: check=True, ) - # Remove manually created database user. + # Remove manually created database and user. + subprocess.run( + [ + "gcloud", + "sql", + "databases", + "delete", + POSTGRES_DATABASE, + "--instance", + POSTGRES_INSTANCE, + "--project", + GOOGLE_CLOUD_PROJECT, + "--quiet", + ], + check=True, + ) + subprocess.run( [ "gcloud", diff --git a/run/django/test/e2e_test_cleanup.yaml b/run/django/test/e2e_test_cleanup.yaml index 40a9b0528c6..065967f0656 100644 --- a/run/django/test/e2e_test_cleanup.yaml +++ b/run/django/test/e2e_test_cleanup.yaml @@ -32,8 +32,6 @@ steps: ./test/retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM}" \ "gcloud run services delete ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM} --quiet" - ./test/retry.sh "gcloud sql databases describe ${_DB_NAME} --instance ${_DB_INSTANCE} --project $PROJECT_ID" \ - "gcloud sql databases delete ${_DB_NAME} --instance ${_DB_INSTANCE} --quiet --project $PROJECT_ID" substitutions: @@ -41,7 +39,6 @@ substitutions: _VERSION: manual _REGION: us-central1 _PLATFORM: managed - _DB_NAME: django _DB_INSTANCE: django-instance _SECRET_SETTINGS_NAME: django_settings _SECRET_PASSWORD_NAME: admin_password From deb6e07d2905ffa41e871941890e9813fa6df1ee Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 8 Feb 2021 15:44:55 +1100 Subject: [PATCH 12/16] remove unused subvar --- run/django/test/e2e_test_cleanup.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/run/django/test/e2e_test_cleanup.yaml b/run/django/test/e2e_test_cleanup.yaml index 065967f0656..0ebb8bab112 100644 --- a/run/django/test/e2e_test_cleanup.yaml +++ b/run/django/test/e2e_test_cleanup.yaml @@ -39,6 +39,5 @@ substitutions: _VERSION: manual _REGION: us-central1 _PLATFORM: managed - _DB_INSTANCE: django-instance _SECRET_SETTINGS_NAME: django_settings _SECRET_PASSWORD_NAME: admin_password From 73cca2bf4f0242153f183ed1885f4046a55726af Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 8 Feb 2021 15:57:45 +1100 Subject: [PATCH 13/16] cleanup --- run/django/test/e2e_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/run/django/test/e2e_test.py b/run/django/test/e2e_test.py index 9c9cc26b5ca..d18a27d85f8 100644 --- a/run/django/test/e2e_test.py +++ b/run/django/test/e2e_test.py @@ -124,8 +124,6 @@ def deployed_service() -> str: f"_SERVICE={SERVICE}," f"_PLATFORM={PLATFORM}," f"_REGION={REGION}," - f"_DB_NAME={POSTGRES_DATABASE}," - f"_DB_INSTANCE={POSTGRES_INSTANCE_NAME}," f"_SECRET_SETTINGS_NAME={SECRET_SETTINGS_NAME}," f"_SECRET_PASSWORD_NAME={SECRET_PASSWORD_NAME}," ] From a60ae4bfcee3e11231c2650dd889048f37d43954 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 10 Feb 2021 08:51:27 +1100 Subject: [PATCH 14/16] Restore setup/cleanup to cloudbuild --- run/django/test/e2e_test.py | 53 ++------------------------- run/django/test/e2e_test_cleanup.yaml | 36 ++++++++++-------- run/django/test/e2e_test_setup.yaml | 14 +++++++ 3 files changed, 38 insertions(+), 65 deletions(-) diff --git a/run/django/test/e2e_test.py b/run/django/test/e2e_test.py index d18a27d85f8..34f3b4760b7 100644 --- a/run/django/test/e2e_test.py +++ b/run/django/test/e2e_test.py @@ -66,23 +66,6 @@ @pytest.fixture def deployed_service() -> str: - # NOTE(glasnt): cloudbuild doesn't have user create rights. - subprocess.run( - [ - "gcloud", - "sql", - "users", - "create", - POSTGRES_USER, - "--password", - POSTGRES_PASSWORD, - "--instance", - POSTGRES_INSTANCE_NAME, - "--project", - GOOGLE_CLOUD_PROJECT, - ], - check=True, - ) substitutions = [ f"_SERVICE={SERVICE}," @@ -124,6 +107,9 @@ def deployed_service() -> str: f"_SERVICE={SERVICE}," f"_PLATFORM={PLATFORM}," f"_REGION={REGION}," + f"_DB_USER={POSTGRES_USER}," + f"_DB_NAME={POSTGRES_DATABASE}," + f"_DB_INSTANCE={POSTGRES_INSTANCE_NAME}," f"_SECRET_SETTINGS_NAME={SECRET_SETTINGS_NAME}," f"_SECRET_PASSWORD_NAME={SECRET_PASSWORD_NAME}," ] @@ -145,39 +131,6 @@ def deployed_service() -> str: check=True, ) - # Remove manually created database and user. - subprocess.run( - [ - "gcloud", - "sql", - "databases", - "delete", - POSTGRES_DATABASE, - "--instance", - POSTGRES_INSTANCE, - "--project", - GOOGLE_CLOUD_PROJECT, - "--quiet", - ], - check=True, - ) - - subprocess.run( - [ - "gcloud", - "sql", - "users", - "delete", - POSTGRES_USER, - "--instance", - POSTGRES_INSTANCE_NAME, - "--project", - GOOGLE_CLOUD_PROJECT, - "--quiet", - ], - check=True, - ) - @pytest.fixture def service_url_auth_token(deployed_service: str) -> Iterator[Tuple[str, str]]: diff --git a/run/django/test/e2e_test_cleanup.yaml b/run/django/test/e2e_test_cleanup.yaml index 0ebb8bab112..b60953e973c 100644 --- a/run/django/test/e2e_test_cleanup.yaml +++ b/run/django/test/e2e_test_cleanup.yaml @@ -13,31 +13,37 @@ # limitations under the License. steps: + - id: "Delete resources" + name: "gcr.io/cloud-builders/gcloud" + entrypoint: "/bin/bash" + args: + - "-c" + - | + ./test/retry.sh "gcloud secrets describe ${_SECRET_SETTINGS_NAME}" \ + "gcloud secrets delete ${_SECRET_SETTINGS_NAME} --quiet --project $PROJECT_ID" -- id: 'Delete resources' - name: 'gcr.io/cloud-builders/gcloud' - entrypoint: '/bin/bash' - args: - - '-c' - - | - ./test/retry.sh "gcloud secrets describe ${_SECRET_SETTINGS_NAME}" \ - "gcloud secrets delete ${_SECRET_SETTINGS_NAME} --quiet --project $PROJECT_ID" + ./test/retry.sh "gcloud secrets describe ${_SECRET_PASSWORD_NAME}" \ + "gcloud secrets delete ${_SECRET_PASSWORD_NAME} --quiet --project $PROJECT_ID" - ./test/retry.sh "gcloud secrets describe ${_SECRET_PASSWORD_NAME}" \ - "gcloud secrets delete ${_SECRET_PASSWORD_NAME} --quiet --project $PROJECT_ID" + ./test/retry.sh "gcloud container images describe gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" \ + "gcloud container images delete gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} --quiet" - ./test/retry.sh "gcloud container images describe gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" \ - "gcloud container images delete gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} --quiet" + ./test/retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM}" \ + "gcloud run services delete ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM} --quiet" - ./test/retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM}" \ - "gcloud run services delete ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM} --quiet" + ./test/retry.sh "gcloud sql databases describe ${_DB_NAME} --instance ${_DB_INSTANCE} --project $PROJECT_ID" \ + "gcloud sql databases delete ${_DB_NAME} --instance ${_DB_INSTANCE} --quiet --project $PROJECT_ID" - + ./test/retry.sh "gcloud sql users list --filter \"name=${_DB_USER}\" --instance ${_DB_INSTANCE}" \ + "gcloud sql users delete ${_DB_USER} --instance ${_DB_INSTANCE} --quiet --project $PROJECT_ID" substitutions: _SERVICE: django _VERSION: manual _REGION: us-central1 _PLATFORM: managed + _DB_USER: django + _DB_NAME: django + _DB_INSTANCE: django-instance _SECRET_SETTINGS_NAME: django_settings _SECRET_PASSWORD_NAME: admin_password diff --git a/run/django/test/e2e_test_setup.yaml b/run/django/test/e2e_test_setup.yaml index 4d1b08692f9..77b40dd82b6 100644 --- a/run/django/test/e2e_test_setup.yaml +++ b/run/django/test/e2e_test_setup.yaml @@ -24,6 +24,20 @@ steps: --project ${PROJECT_ID}" \ "gcloud sql databases describe ${_DB_NAME} --instance ${_DB_INSTANCE}" + - id: "Create a dedicated database user" + name: "gcr.io/cloud-builders/gcloud" + entrypoint: "/bin/bash" + args: + - "-c" + - | + echo -n "${_DB_PASS}" > db_password + ./test/retry.sh "gcloud sql users create ${_DB_USER} \ + --password $(cat db_password) \ + --instance ${_DB_INSTANCE} \ + --project ${PROJECT_ID}" \ + "gcloud sql users list --filter \"name=${_DB_USER}\" --instance ${_DB_INSTANCE}" + rm db_password + - id: "Create a dedicated storage bucket" name: "gcr.io/cloud-builders/gcloud" entrypoint: "/bin/bash" From 2e1e652d967928816fb572d7c6ddac45af15564d Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 10 Feb 2021 08:51:40 +1100 Subject: [PATCH 15/16] Update retry.sh to handle two-arg failure, ultimate failure --- run/django/test/retry.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/run/django/test/retry.sh b/run/django/test/retry.sh index bca6fae257d..a90388bf6cf 100755 --- a/run/django/test/retry.sh +++ b/run/django/test/retry.sh @@ -27,6 +27,10 @@ # # ./retry.sh "gcloud RESOURCE EXISTS?" "gcloud ACTION" # +# +# Note: +# $# - the number of command-line arguments passed +# $? - the exit value of the last command ## # Usage: try "cmd1" "cmd2" @@ -37,6 +41,7 @@ runIfSuccessful() { if [ $? -eq 0 ]; then echo "running: $2" $($2 > /dev/null) + else return 1 fi } @@ -59,7 +64,7 @@ do if ((attempt_num==max_attempts)) then echo "Attempt $attempt_num / $max_attempts failed! No more retries left!" - exit + exit 1 else echo "Attempt $attempt_num / $max_attempts failed!" sleep $((attempt_num++)) From 38ce341f40975c9903ede27aa912f54654b00811 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Thu, 11 Feb 2021 08:55:55 +1100 Subject: [PATCH 16/16] Change use of retry.sh to be create, or describe/delete --- run/django/test/e2e_test_setup.yaml | 15 +++++---------- run/django/test/retry.sh | 2 -- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/run/django/test/e2e_test_setup.yaml b/run/django/test/e2e_test_setup.yaml index 77b40dd82b6..d3e0e15e905 100644 --- a/run/django/test/e2e_test_setup.yaml +++ b/run/django/test/e2e_test_setup.yaml @@ -21,8 +21,7 @@ steps: - | ./test/retry.sh "gcloud sql databases create ${_DB_NAME} \ --instance ${_DB_INSTANCE} \ - --project ${PROJECT_ID}" \ - "gcloud sql databases describe ${_DB_NAME} --instance ${_DB_INSTANCE}" + --project ${PROJECT_ID}" - id: "Create a dedicated database user" name: "gcr.io/cloud-builders/gcloud" @@ -34,8 +33,7 @@ steps: ./test/retry.sh "gcloud sql users create ${_DB_USER} \ --password $(cat db_password) \ --instance ${_DB_INSTANCE} \ - --project ${PROJECT_ID}" \ - "gcloud sql users list --filter \"name=${_DB_USER}\" --instance ${_DB_INSTANCE}" + --project ${PROJECT_ID}" rm db_password - id: "Create a dedicated storage bucket" @@ -47,8 +45,7 @@ steps: ./test/retry.sh "gsutil mb \ -l ${_REGION} \ -p ${PROJECT_ID} \ - gs://${_STORAGE_BUCKET}" \ - "gsutil ls gs://${_STORAGE_BUCKET}" + gs://${_STORAGE_BUCKET}" - id: "Add Django secrets to Secret Manager" name: "gcr.io/cloud-builders/gcloud" @@ -64,8 +61,7 @@ steps: ./test/retry.sh "gcloud secrets create ${_SECRET_SETTINGS_NAME} \ --project $PROJECT_ID \ - --data-file=${_SECRET_SETTINGS_NAME}" \ - "gcloud secrets describe ${_SECRET_SETTINGS_NAME}" + --data-file=${_SECRET_SETTINGS_NAME}" gcloud secrets add-iam-policy-binding ${_SECRET_SETTINGS_NAME} \ --member serviceAccount:$(gcloud projects list --filter "name=${PROJECT_ID}" --format "value(projectNumber)")@cloudbuild.gserviceaccount.com \ @@ -78,8 +74,7 @@ steps: ./test/retry.sh "gcloud secrets create ${_SECRET_PASSWORD_NAME} \ --project $PROJECT_ID \ - --data-file=${_SECRET_PASSWORD_NAME}" \ - "gcloud secrets describe ${_SECRET_PASSWORD_NAME}" + --data-file=${_SECRET_PASSWORD_NAME}" gcloud secrets add-iam-policy-binding ${_SECRET_PASSWORD_NAME} \ --member serviceAccount:$(gcloud projects list --filter "name=${PROJECT_ID}" --format "value(projectNumber)")@cloudbuild.gserviceaccount.com \ diff --git a/run/django/test/retry.sh b/run/django/test/retry.sh index a90388bf6cf..67ed25ad447 100755 --- a/run/django/test/retry.sh +++ b/run/django/test/retry.sh @@ -41,7 +41,6 @@ runIfSuccessful() { if [ $? -eq 0 ]; then echo "running: $2" $($2 > /dev/null) - else return 1 fi } @@ -64,7 +63,6 @@ do if ((attempt_num==max_attempts)) then echo "Attempt $attempt_num / $max_attempts failed! No more retries left!" - exit 1 else echo "Attempt $attempt_num / $max_attempts failed!" sleep $((attempt_num++))