diff --git a/run/django/.gcloudignore b/run/django/.gcloudignore index 4c49bd78f1d..2217a41e75b 100644 --- a/run/django/.gcloudignore +++ b/run/django/.gcloudignore @@ -1 +1,2 @@ .env +venv diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py deleted file mode 100644 index 9a161d5a118..00000000000 --- a/run/django/e2e_test.py +++ /dev/null @@ -1,406 +0,0 @@ -# Copyright 2020 Google, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This test creates a Cloud SQL instance, a Cloud Storage bucket, associated -# secrets, and deploys a Django service - -import os -import subprocess -from typing import Iterator, List, Tuple -import uuid - -from google.cloud import secretmanager_v1 as sm -import pytest -import requests - -# Unique suffix to create distinct service names -SUFFIX = uuid.uuid4().hex[:10] - -PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] -REGION = "us-central1" -POSTGRES_INSTANCE = os.environ["POSTGRES_INSTANCE"] - -# Most commands in this test require the short instance form -if ":" in POSTGRES_INSTANCE: - POSTGRES_INSTANCE = POSTGRES_INSTANCE.split(":")[-1] - -CLOUD_STORAGE_BUCKET = f"{PROJECT}-media-{SUFFIX}" - -POSTGRES_DATABASE = f"polls-{SUFFIX}" -POSTGRES_USER = f"django-{SUFFIX}" -POSTGRES_PASSWORD = uuid.uuid4().hex[:26] - -ADMIN_NAME = "admin" -ADMIN_PASSWORD = uuid.uuid4().hex[:26] - -SECRET_SETTINGS_NAME = f"django_settings-{SUFFIX}" -SECRET_PASSWORD_NAME = f"superuser_password-{SUFFIX}" - - -@pytest.fixture -def project_number() -> Iterator[str]: - projectnum = ( - subprocess.run( - [ - "gcloud", - "projects", - "list", - "--filter", - f"name={PROJECT}", - "--format", - "value(projectNumber)", - ], - stdout=subprocess.PIPE, - check=True, - ) - .stdout.strip() - .decode() - ) - yield projectnum - - -@pytest.fixture -def postgres_host() -> Iterator[str]: - # Create database - subprocess.run( - [ - "gcloud", - "sql", - "databases", - "create", - POSTGRES_DATABASE, - "--instance", - POSTGRES_INSTANCE, - "--project", - PROJECT, - ], - check=True, - ) - # Create User - # NOTE Creating a user via the tutorial method is not automatable. - subprocess.run( - [ - "gcloud", - "sql", - "users", - "create", - POSTGRES_USER, - "--password", - POSTGRES_PASSWORD, - "--instance", - POSTGRES_INSTANCE, - "--project", - PROJECT, - ], - check=True, - ) - yield POSTGRES_INSTANCE - - subprocess.run( - [ - "gcloud", - "sql", - "databases", - "delete", - POSTGRES_DATABASE, - "--instance", - POSTGRES_INSTANCE, - "--project", - PROJECT, - "--quiet", - ], - check=True, - ) - - subprocess.run( - [ - "gcloud", - "sql", - "users", - "delete", - POSTGRES_USER, - "--instance", - POSTGRES_INSTANCE, - "--project", - PROJECT, - "--quiet", - ], - check=True, - ) - - -@pytest.fixture -def media_bucket() -> Iterator[str]: - # Create storage bucket - subprocess.run( - ["gsutil", "mb", "-l", REGION, "-p", PROJECT, f"gs://{CLOUD_STORAGE_BUCKET}"], - check=True, - ) - - yield CLOUD_STORAGE_BUCKET - - # Recursively delete assets and bucket (does not take a -p flag, apparently) - subprocess.run( - ["gsutil", "-m", "rm", "-r", f"gs://{CLOUD_STORAGE_BUCKET}"], - check=True, - ) - - -@pytest.fixture -def secrets(project_number: str) -> Iterator[str]: - # Create a number of secrets and allow Google Cloud services access to them - - def create_secret(name: str, value: str) -> None: - secret = client.create_secret( - request={ - "parent": f"projects/{PROJECT}", - "secret": {"replication": {"automatic": {}}}, - "secret_id": name, - } - ) - - client.add_secret_version( - request={"parent": secret.name, "payload": {"data": value.encode("UTF-8")}} - ) - - def allow_access(name: str, member: str) -> None: - subprocess.run( - [ - "gcloud", - "secrets", - "add-iam-policy-binding", - name, - "--member", - member, - "--role", - "roles/secretmanager.secretAccessor", - "--project", - PROJECT, - ], - check=True, - ) - - client = sm.SecretManagerServiceClient() - secret_key = uuid.uuid4().hex[:56] - settings = f""" -DATABASE_URL=postgres://{POSTGRES_USER}:{POSTGRES_PASSWORD}@//cloudsql/{PROJECT}:{REGION}:{POSTGRES_INSTANCE}/{POSTGRES_DATABASE} -GS_BUCKET_NAME={CLOUD_STORAGE_BUCKET} -SECRET_KEY={secret_key} -PASSWORD_NAME={SECRET_PASSWORD_NAME} - """ - - create_secret(SECRET_SETTINGS_NAME, settings) - allow_access( - SECRET_SETTINGS_NAME, - f"serviceAccount:{project_number}-compute@developer.gserviceaccount.com", - ) - allow_access( - SECRET_SETTINGS_NAME, - f"serviceAccount:{project_number}@cloudbuild.gserviceaccount.com", - ) - - create_secret(SECRET_PASSWORD_NAME, ADMIN_PASSWORD) - allow_access( - SECRET_PASSWORD_NAME, - f"serviceAccount:{project_number}@cloudbuild.gserviceaccount.com", - ) - - yield SECRET_SETTINGS_NAME - - # delete secrets - subprocess.run( - [ - "gcloud", - "secrets", - "delete", - SECRET_PASSWORD_NAME, - "--project", - PROJECT, - "--quiet", - ], - check=True, - ) - subprocess.run( - [ - "gcloud", - "secrets", - "delete", - SECRET_SETTINGS_NAME, - "--project", - PROJECT, - "--quiet", - ], - check=True, - ) - - -@pytest.fixture -def container_image(postgres_host: str, media_bucket: str, secrets: str) -> Iterator[str]: - # Build container image for Cloud Run deployment - image_name = f"gcr.io/{PROJECT}/polls-{SUFFIX}" - service_name = f"polls-{SUFFIX}" - cloudbuild_config = "cloudmigrate.yaml" - subprocess.run( - [ - "gcloud", - "builds", - "submit", - "--config", - cloudbuild_config, - "--substitutions", - ( - f"_INSTANCE_NAME={postgres_host}," - f"_REGION={REGION}," - f"_SERVICE_NAME={service_name}," - f"_SECRET_SETTINGS_NAME={SECRET_SETTINGS_NAME}" - ), - "--project", - PROJECT, - ], - check=True, - ) - yield image_name - - # Delete container image - subprocess.run( - [ - "gcloud", - "container", - "images", - "delete", - image_name, - "--quiet", - "--project", - PROJECT, - ], - check=True, - ) - - -@pytest.fixture -def deployed_service(container_image: str) -> Iterator[str]: - # Deploy image to Cloud Run - service_name = f"polls-{SUFFIX}" - subprocess.run( - [ - "gcloud", - "run", - "deploy", - service_name, - "--image", - container_image, - "--platform=managed", - "--no-allow-unauthenticated", - "--region", - REGION, - "--add-cloudsql-instances", - f"{PROJECT}:{REGION}:{POSTGRES_INSTANCE}", - "--set-env-vars", - f"SETTINGS_NAME={SECRET_SETTINGS_NAME}", - "--project", - PROJECT, - ], - check=True, - ) - yield service_name - - # Delete Cloud Run service - subprocess.run( - [ - "gcloud", - "run", - "services", - "delete", - service_name, - "--platform=managed", - "--region=us-central1", - "--quiet", - "--project", - PROJECT, - ], - check=True, - ) - - -@pytest.fixture -def service_url_auth_token(deployed_service: str) -> Iterator[Tuple[str, str]]: - # Get Cloud Run service URL and auth token - service_url = ( - subprocess.run( - [ - "gcloud", - "run", - "services", - "describe", - deployed_service, - "--platform", - "managed", - "--region", - REGION, - "--format", - "value(status.url)", - "--project", - PROJECT, - ], - stdout=subprocess.PIPE, - check=True, - ) - .stdout.strip() - .decode() - ) - auth_token = ( - subprocess.run( - ["gcloud", "auth", "print-identity-token", "--project", PROJECT], - stdout=subprocess.PIPE, - check=True, - ) - .stdout.strip() - .decode() - ) - - yield service_url, auth_token - - # no deletion needed - - -def test_end_to_end(service_url_auth_token: List[str]) -> None: - service_url, auth_token = service_url_auth_token - headers = {"Authorization": f"Bearer {auth_token}"} - login_slug = "/admin/login/?next=/admin/" - client = requests.session() - - # Check homepage - response = client.get(service_url, headers=headers) - body = response.text - - assert response.status_code == 200 - assert "Hello, world" in body - - # Load login page, collecting csrf token - client.get(service_url + login_slug, headers=headers) - csrftoken = client.cookies["csrftoken"] - - # Log into Django admin - payload = { - "username": ADMIN_NAME, - "password": ADMIN_PASSWORD, - "csrfmiddlewaretoken": csrftoken, - } - response = client.post(service_url + login_slug, data=payload, headers=headers) - body = response.text - - # Check Django admin landing page - assert response.status_code == 200 - assert "Site administration" in body - assert "Polls" in body diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py index 149d21c19d1..27a354ef691 100644 --- a/run/django/mysite/migrations/0001_createsuperuser.py +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -15,11 +15,13 @@ # [START cloudrun_django_superuser] import os +from django.contrib.auth.models import User from django.db import migrations from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps + import google.auth -from google.cloud import secretmanager_v1 +from google.cloud import secretmanager def createsuperuser(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: @@ -28,9 +30,10 @@ def createsuperuser(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> Non Password is pulled from Secret Manger (previously created as part of tutorial) """ if os.getenv("TRAMPOLINE_CI", None): + # We are in CI, so just create a placeholder user for unit testing. admin_password = "test" else: - client = secretmanager_v1.SecretManagerServiceClient() + client = secretmanager.SecretManagerServiceClient() # Get project value for identifying current context _, project = google.auth.default() @@ -42,17 +45,15 @@ def createsuperuser(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> Non "UTF-8" ) - # Create a new user using acquired password - from django.contrib.auth.models import User - - User.objects.create_superuser("admin", password=admin_password) + # Create a new user using acquired password, stripping any accidentally stored newline characters + User.objects.create_superuser("admin", password=admin_password.strip()) class Migration(migrations.Migration): initial = True - dependencies = [] - operations = [migrations.RunPython(createsuperuser)] + + # [END cloudrun_django_superuser] diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 3f160e827a0..11332426390 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -23,6 +23,7 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/3.0/ref/settings/ """ +import io import os import environ @@ -30,32 +31,34 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) env_file = os.path.join(BASE_DIR, ".env") -# If no .env has been provided, pull it from Secret Manager, storing it locally -if not os.path.isfile(".env"): - if os.getenv('TRAMPOLINE_CI', None): - payload = f"SECRET_KEY=a\nGS_BUCKET_NAME=none\nDATABASE_URL=sqlite://{os.path.join(BASE_DIR, 'db.sqlite3')}" + +env = environ.Env() +# If no .env has been provided, pull it from Secret Manager +if os.path.isfile(".env"): + env.read_env(env_file) +else: + # Create local settings if running with CI, for unit testing + if os.getenv("TRAMPOLINE_CI", None): + placeholder = f"SECRET_KEY=a\nGS_BUCKET_NAME=none\nDATABASE_URL=sqlite://{os.path.join(BASE_DIR, 'db.sqlite3')}" + env.read_env(io.StringIO(placeholder)) else: # [START cloudrun_django_secretconfig] import google.auth - from google.cloud import secretmanager_v1 + from google.cloud import secretmanager _, project = google.auth.default() if project: - client = secretmanager_v1.SecretManagerServiceClient() + client = secretmanager.SecretManagerServiceClient() SETTINGS_NAME = os.environ.get("SETTINGS_NAME", "django_settings") name = f"projects/{project}/secrets/{SETTINGS_NAME}/versions/latest" payload = client.access_secret_version(name=name).payload.data.decode( "UTF-8" ) + env.read_env(io.StringIO(payload)) + # [END cloudrun_django_secretconfig] - with open(env_file, "w") as f: - f.write(payload) - -env = environ.Env() -env.read_env(env_file) -# [END cloudrun_django_secretconfig] SECRET_KEY = env("SECRET_KEY") @@ -106,7 +109,7 @@ # [START cloudrun_django_dbconfig] -# Use django-environ to define the connection string +# Use django-environ to parse the connection string DATABASES = {"default": env.db()} # [END cloudrun_django_dbconfig] @@ -114,7 +117,9 @@ # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ - {"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", }, {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", }, {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", }, diff --git a/run/django/polls/test_polls.py b/run/django/polls/test_polls.py index e02f4a34f44..81085e6fb0c 100644 --- a/run/django/polls/test_polls.py +++ b/run/django/polls/test_polls.py @@ -19,6 +19,6 @@ class PollViewTests(TestCase): def test_index_view(self: PollViewTests) -> None: - response = self.client.get('/') + response = self.client.get("/") assert response.status_code == 200 - assert 'Hello, world' in str(response.content) + assert "Hello, world" in str(response.content) diff --git a/run/django/test/e2e_test.py b/run/django/test/e2e_test.py new file mode 100644 index 00000000000..34f3b4760b7 --- /dev/null +++ b/run/django/test/e2e_test.py @@ -0,0 +1,212 @@ +# Copyright 2021 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This test creates a Cloud SQL instance, a Cloud Storage bucket, associated +# secrets, and deploys a Django service + +import os +import subprocess +from typing import Iterator, List, Tuple +import uuid + +import pytest +import requests + +# Unique suffix to create distinct service names +SUFFIX = uuid.uuid4().hex[:10] + +SAMPLE_VERSION = os.environ.get("SAMPLE_VERSION", None) +GOOGLE_CLOUD_PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +REGION = "us-central1" +PLATFORM = "managed" + +SERVICE = f"polls-{SUFFIX}" + +# Retreieve Cloud SQL test config +POSTGRES_INSTANCE = os.environ.get("POSTGRES_INSTANCE", None) +if not POSTGRES_INSTANCE: + raise Exception("'POSTGRES_INSTANCE' env var not found") + +# Presuming POSTGRES_INSTANCE comes in the form project:region:instance +# Require the short form in some cases. +# POSTGRES_INSTANCE_FULL: project:region:instance +# POSTGRES_INSTANCE_NAME: instance only +if ":" in POSTGRES_INSTANCE: + POSTGRES_INSTANCE_FULL = POSTGRES_INSTANCE + POSTGRES_INSTANCE_NAME = POSTGRES_INSTANCE.split(":")[-1] +else: + POSTGRES_INSTANCE_FULL = f"{GOOGLE_CLOUD_PROJECT}:{REGION}:{POSTGRES_INSTANCE}" + POSTGRES_INSTANCE_NAME = POSTGRES_INSTANCE + +POSTGRES_DATABASE = f"django-database-{SUFFIX}" + +CLOUD_STORAGE_BUCKET = f"{GOOGLE_CLOUD_PROJECT}-media-{SUFFIX}" + +POSTGRES_DATABASE = f"polls-{SUFFIX}" +POSTGRES_USER = f"django-{SUFFIX}" +POSTGRES_PASSWORD = uuid.uuid4().hex[:26] + +ADMIN_NAME = "admin" +ADMIN_PASSWORD = uuid.uuid4().hex[:26] + +SECRET_SETTINGS_NAME = f"django_settings-{SUFFIX}" +SECRET_PASSWORD_NAME = f"superuser_password-{SUFFIX}" + + +@pytest.fixture +def deployed_service() -> str: + + substitutions = [ + f"_SERVICE={SERVICE}," + f"_PLATFORM={PLATFORM}," + f"_REGION={REGION}," + f"_STORAGE_BUCKET={CLOUD_STORAGE_BUCKET}," + f"_DB_NAME={POSTGRES_DATABASE}," + f"_DB_USER={POSTGRES_USER}," + f"_DB_PASS={POSTGRES_PASSWORD}," + f"_DB_INSTANCE={POSTGRES_INSTANCE_NAME}," + f"_SECRET_SETTINGS_NAME={SECRET_SETTINGS_NAME}," + f"_SECRET_PASSWORD_NAME={SECRET_PASSWORD_NAME}," + f"_SECRET_PASSWORD_VALUE={ADMIN_PASSWORD}," + f"_CLOUD_SQL_CONNECTION_NAME={POSTGRES_INSTANCE_FULL}" + ] + if SAMPLE_VERSION: + substitutions.append(f",_VERSION={SAMPLE_VERSION}") + + subprocess.run( + [ + "gcloud", + "builds", + "submit", + "--project", + GOOGLE_CLOUD_PROJECT, + "--config", + "./test/e2e_test_setup.yaml", + "--substitutions", + ] + + substitutions, + check=True, + ) + + yield SERVICE + + # Cleanup + + substitutions = [ + f"_SERVICE={SERVICE}," + f"_PLATFORM={PLATFORM}," + f"_REGION={REGION}," + f"_DB_USER={POSTGRES_USER}," + f"_DB_NAME={POSTGRES_DATABASE}," + f"_DB_INSTANCE={POSTGRES_INSTANCE_NAME}," + f"_SECRET_SETTINGS_NAME={SECRET_SETTINGS_NAME}," + f"_SECRET_PASSWORD_NAME={SECRET_PASSWORD_NAME}," + ] + if SAMPLE_VERSION: + substitutions.append(f"_SAMPLE_VERSION={SAMPLE_VERSION}") + + subprocess.run( + [ + "gcloud", + "builds", + "submit", + "--project", + GOOGLE_CLOUD_PROJECT, + "--config", + "./test/e2e_test_cleanup.yaml", + "--substitutions", + ] + + substitutions, + check=True, + ) + + +@pytest.fixture +def service_url_auth_token(deployed_service: str) -> Iterator[Tuple[str, str]]: + # Get Cloud Run service URL and auth token + service_url = ( + subprocess.run( + [ + "gcloud", + "run", + "services", + "describe", + deployed_service, + "--platform", + "managed", + "--region", + REGION, + "--format", + "value(status.url)", + "--project", + GOOGLE_CLOUD_PROJECT, + ], + stdout=subprocess.PIPE, + check=True, + ) + .stdout.strip() + .decode() + ) + auth_token = ( + subprocess.run( + [ + "gcloud", + "auth", + "print-identity-token", + "--project", + GOOGLE_CLOUD_PROJECT, + ], + stdout=subprocess.PIPE, + check=True, + ) + .stdout.strip() + .decode() + ) + + yield service_url, auth_token + + # no deletion needed + + +def test_end_to_end(service_url_auth_token: List[str]) -> None: + service_url, auth_token = service_url_auth_token + headers = {"Authorization": f"Bearer {auth_token}"} + login_slug = "/admin/login/?next=/admin/" + client = requests.session() + + # Check homepage + response = client.get(service_url, headers=headers) + body = response.text + + assert response.status_code == 200 + assert "Hello, world" in body + + # Load login page, collecting csrf token + client.get(service_url + login_slug, headers=headers) + csrftoken = client.cookies["csrftoken"] + + # Log into Django admin + payload = { + "username": ADMIN_NAME, + "password": ADMIN_PASSWORD, + "csrfmiddlewaretoken": csrftoken, + } + response = client.post(service_url + login_slug, data=payload, headers=headers) + body = response.text + + # Check Django admin landing page + assert response.status_code == 200 + assert "Please enter the correct username and password" not in body + assert "Site administration" in body + assert "Polls" in body diff --git a/run/django/test/e2e_test_cleanup.yaml b/run/django/test/e2e_test_cleanup.yaml new file mode 100644 index 00000000000..b60953e973c --- /dev/null +++ b/run/django/test/e2e_test_cleanup.yaml @@ -0,0 +1,49 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + - id: "Delete resources" + name: "gcr.io/cloud-builders/gcloud" + entrypoint: "/bin/bash" + args: + - "-c" + - | + ./test/retry.sh "gcloud secrets describe ${_SECRET_SETTINGS_NAME}" \ + "gcloud secrets delete ${_SECRET_SETTINGS_NAME} --quiet --project $PROJECT_ID" + + ./test/retry.sh "gcloud secrets describe ${_SECRET_PASSWORD_NAME}" \ + "gcloud secrets delete ${_SECRET_PASSWORD_NAME} --quiet --project $PROJECT_ID" + + ./test/retry.sh "gcloud container images describe gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" \ + "gcloud container images delete gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} --quiet" + + ./test/retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM}" \ + "gcloud run services delete ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM} --quiet" + + ./test/retry.sh "gcloud sql databases describe ${_DB_NAME} --instance ${_DB_INSTANCE} --project $PROJECT_ID" \ + "gcloud sql databases delete ${_DB_NAME} --instance ${_DB_INSTANCE} --quiet --project $PROJECT_ID" + + ./test/retry.sh "gcloud sql users list --filter \"name=${_DB_USER}\" --instance ${_DB_INSTANCE}" \ + "gcloud sql users delete ${_DB_USER} --instance ${_DB_INSTANCE} --quiet --project $PROJECT_ID" + +substitutions: + _SERVICE: django + _VERSION: manual + _REGION: us-central1 + _PLATFORM: managed + _DB_USER: django + _DB_NAME: django + _DB_INSTANCE: django-instance + _SECRET_SETTINGS_NAME: django_settings + _SECRET_PASSWORD_NAME: admin_password diff --git a/run/django/test/e2e_test_setup.yaml b/run/django/test/e2e_test_setup.yaml new file mode 100644 index 00000000000..d3e0e15e905 --- /dev/null +++ b/run/django/test/e2e_test_setup.yaml @@ -0,0 +1,170 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + - id: "Create a dedicated database" + name: "gcr.io/cloud-builders/gcloud" + entrypoint: "/bin/bash" + args: + - "-c" + - | + ./test/retry.sh "gcloud sql databases create ${_DB_NAME} \ + --instance ${_DB_INSTANCE} \ + --project ${PROJECT_ID}" + + - id: "Create a dedicated database user" + name: "gcr.io/cloud-builders/gcloud" + entrypoint: "/bin/bash" + args: + - "-c" + - | + echo -n "${_DB_PASS}" > db_password + ./test/retry.sh "gcloud sql users create ${_DB_USER} \ + --password $(cat db_password) \ + --instance ${_DB_INSTANCE} \ + --project ${PROJECT_ID}" + rm db_password + + - id: "Create a dedicated storage bucket" + name: "gcr.io/cloud-builders/gcloud" + entrypoint: "/bin/bash" + args: + - "-c" + - | + ./test/retry.sh "gsutil mb \ + -l ${_REGION} \ + -p ${PROJECT_ID} \ + gs://${_STORAGE_BUCKET}" + + - id: "Add Django secrets to Secret Manager" + name: "gcr.io/cloud-builders/gcloud" + entrypoint: "/bin/bash" + args: + - "-c" + - | + echo " + DATABASE_URL=postgres://${_DB_USER}:${_DB_PASS}@//cloudsql/${_CLOUD_SQL_CONNECTION_NAME}/${_DB_NAME} + GS_BUCKET_NAME=${_STORAGE_BUCKET} + SECRET_KEY=$(cat /dev/urandom | LC_ALL=C tr -dc '[:alpha:]' | fold -w 30 | head -n1) + PASSWORD_NAME=${_SECRET_PASSWORD_NAME}" > ${_SECRET_SETTINGS_NAME} + + ./test/retry.sh "gcloud secrets create ${_SECRET_SETTINGS_NAME} \ + --project $PROJECT_ID \ + --data-file=${_SECRET_SETTINGS_NAME}" + + gcloud secrets add-iam-policy-binding ${_SECRET_SETTINGS_NAME} \ + --member serviceAccount:$(gcloud projects list --filter "name=${PROJECT_ID}" --format "value(projectNumber)")@cloudbuild.gserviceaccount.com \ + --role roles/secretmanager.secretAccessor \ + --project ${PROJECT_ID} + + rm ${_SECRET_SETTINGS_NAME} + + echo -n "${_SECRET_PASSWORD_VALUE}" > ${_SECRET_PASSWORD_NAME} + + ./test/retry.sh "gcloud secrets create ${_SECRET_PASSWORD_NAME} \ + --project $PROJECT_ID \ + --data-file=${_SECRET_PASSWORD_NAME}" + + gcloud secrets add-iam-policy-binding ${_SECRET_PASSWORD_NAME} \ + --member serviceAccount:$(gcloud projects list --filter "name=${PROJECT_ID}" --format "value(projectNumber)")@cloudbuild.gserviceaccount.com \ + --role roles/secretmanager.secretAccessor \ + --project ${PROJECT_ID} + + rm ${_SECRET_PASSWORD_NAME} + + - id: "Build Container Image" + name: "gcr.io/cloud-builders/docker" + entrypoint: "/bin/bash" + args: + - "-c" + - | + ./test/retry.sh "docker build -t gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} ." + + - id: "Push Container Image" + name: "gcr.io/cloud-builders/docker" + entrypoint: "/bin/bash" + args: + - "-c" + - | + ./test/retry.sh "docker push gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" + + - id: "Migrate database" + name: "gcr.io/google-appengine/exec-wrapper" + args: + [ + "-i", + "gcr.io/$PROJECT_ID/${_SERVICE}:${_VERSION}", + "-s", + "${_CLOUD_SQL_CONNECTION_NAME}", + "-e", + "SETTINGS_NAME=${_SECRET_SETTINGS_NAME}", + "-e", + "PASSWORD_NAME=${_SECRET_PASSWORD_NAME}", + "--", + "python", + "manage.py", + "migrate", + ] + + - id: "Collect static" + name: "gcr.io/google-appengine/exec-wrapper" + args: + [ + "-i", + "gcr.io/$PROJECT_ID/${_SERVICE}:${_VERSION}", + "-s", + "${_CLOUD_SQL_CONNECTION_NAME}", + "-e", + "SETTINGS_NAME=${_SECRET_SETTINGS_NAME}", + "--", + "python", + "manage.py", + "collectstatic", + "--verbosity", + "2", + "--no-input", + ] + + - id: "Deploy to Cloud Run" + name: "gcr.io/cloud-builders/gcloud:latest" + entrypoint: /bin/bash + args: + - "-c" + - | + ./test/retry.sh "gcloud run deploy ${_SERVICE} \ + --project $PROJECT_ID \ + --image gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} \ + --no-allow-unauthenticated \ + --region ${_REGION} \ + --platform ${_PLATFORM} \ + --add-cloudsql-instances ${_CLOUD_SQL_CONNECTION_NAME} \ + --update-env-vars SETTINGS_NAME=${_SECRET_SETTINGS_NAME}" + +images: + - gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} + +substitutions: + _SERVICE: django + _VERSION: manual + _REGION: us-central1 + _PLATFORM: managed + _STORAGE_BUCKET: ${PROJECT_ID}-bucket + _DB_INSTANCE: django-instance + _CLOUD_SQL_CONNECTION_NAME: $PROJECT_ID:us-central1:django-instance + _DB_NAME: postgres + _DB_USER: postgres + _DB_PASS: password1234 + _SECRET_SETTINGS_NAME: django_settings + _SECRET_PASSWORD_NAME: admin_password + _SECRET_PASSWORD_VALUE: password diff --git a/run/django/test/retry.sh b/run/django/test/retry.sh new file mode 100755 index 00000000000..67ed25ad447 --- /dev/null +++ b/run/django/test/retry.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## +# retry.sh +# Provides utility function commonly needed across Cloud Build pipelines to +# retry commands on failure. +# +# Usage: +# 1. Retry single command: +# +# ./retry.sh "CMD" +# +# 2. Retry with check: +# +# ./retry.sh "gcloud RESOURCE EXISTS?" "gcloud ACTION" +# +# +# Note: +# $# - the number of command-line arguments passed +# $? - the exit value of the last command +## + +# Usage: try "cmd1" "cmd2" +# If first cmd executes successfully then execute second cmd +runIfSuccessful() { + echo "running: $1" + $($1 > /dev/null) + if [ $? -eq 0 ]; then + echo "running: $2" + $($2 > /dev/null) + fi +} + +# Define max retries +max_attempts=3; +attempt_num=1; + +arg1="$1" +arg2="$2" + +if [ $# -eq 1 ] +then + cmd="$arg1" +else + cmd="runIfSuccessful \"$arg1\" \"$arg2\"" +fi + +until eval $cmd +do + if ((attempt_num==max_attempts)) + then + echo "Attempt $attempt_num / $max_attempts failed! No more retries left!" + else + echo "Attempt $attempt_num / $max_attempts failed!" + sleep $((attempt_num++)) + fi +done