diff --git a/.gitignore b/.gitignore index 184297e92b1..dd8bffdf608 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ managed_vms/django_tutorial/static/* lib testing/resources/test-env.sh testing/resources/service-account.json +testing/resources/client-secrets.json secrets.tar .cache junit.xml diff --git a/.travis.yml b/.travis.yml index b43a7de8d63..4a147705cd5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ sudo: false language: python services: - - memcached - - mysql +- memcached +- mysql branches: only: - master @@ -13,15 +13,15 @@ env: global: - PATH=${PATH}:${HOME}/gcloud/google-cloud-sdk/bin - GOOGLE_APPLICATION_CREDENTIALS=${TRAVIS_BUILD_DIR}/testing/resources/service-account.json + - GOOGLE_CLIENT_SECRETS=${TRAVIS_BUILD_DIR}/testing/resources/client-secrets.json - GAE_PYTHONPATH=${HOME}/.cache/google_appengine - - secure: YIowCOMJ97rTcehKVT6Gi3u0Etm8s9+TBRGsNPJLgSF2zZdsh9IHcIc+tMDUMR3lpOe8y2a060RuODQcRsW1W1LIHej+ZE/gv6vATT6qNA3eKfKmZ9AyrpBO0fTOHlHrGBuU9ktBPR+iqvnq8MLWjnUozPFMJbuNBFITU7JP8jc= + - GAE_ROOT=${HOME}/.cache/ + - secure: Orp9Et2TIwCG/Hf59aa0NUDF1pNcwcS4TFulXX175918cFREOzf/cNZNg+Ui585ZRFjbifZdc858tVuCVd8XlxQPXQgp7bwB7nXs3lby3LYg4+HD83Gaz7KOWxRLWVor6IVn8OxeCzwl6fJkdmffsTTO9csC4yZ7izHr+u7hiO4= before_install: - openssl aes-256-cbc -k "$secrets_password" -in secrets.tar.enc -out secrets.tar -d - tar xvf secrets.tar install: - pip install tox coverage -- pip install -e git+https://github.com/GoogleCloudPlatform/python-repo-tools#egg=python-repo-tools -- gcp-python-repo-tools download-appengine-sdk `dirname "${GAE_PYTHONPATH}"` script: - source ${TRAVIS_BUILD_DIR}/testing/resources/test-env.sh - tox diff --git a/appengine/bigquery/main_test.py b/appengine/bigquery/main_test.py index 4d22d67d243..77e192b2dac 100644 --- a/appengine/bigquery/main_test.py +++ b/appengine/bigquery/main_test.py @@ -14,7 +14,7 @@ import re -from apiclient.http import HttpMock +from googleapiclient.http import HttpMock import main import mock import pytest @@ -23,7 +23,7 @@ @pytest.fixture def app(cloud_config, testbed): - main.PROJECTID = cloud_config.GCLOUD_PROJECT + main.PROJECTID = cloud_config.project return webtest.TestApp(main.app) diff --git a/appengine/conftest.py b/appengine/conftest.py index ff92c6fc95e..4fd85dcc906 100644 --- a/appengine/conftest.py +++ b/appengine/conftest.py @@ -1,35 +1,27 @@ -import pytest -import testing.appengine - - -def pytest_configure(config): - testing.appengine.setup_sdk_imports() - - -def pytest_runtest_call(item): - testing.appengine.import_appengine_config() - - -@pytest.yield_fixture -def testbed(): - testbed = testing.appengine.setup_testbed() - yield testbed - testbed.deactivate() - - -@pytest.fixture -def login(testbed): - def _login(email='user@example.com', id='123', is_admin=False): - testbed.setup_env( - user_email=email, - user_id=id, - user_is_admin='1' if is_admin else '0', - overwrite=True) - return _login - - -@pytest.fixture -def run_tasks(testbed): - def _run_tasks(app): - testing.appengine.run_tasks(testbed, app) - return _run_tasks +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Import py.test hooks and fixtures for App Engine +from gcp.testing.appengine import ( + login, + pytest_configure, + pytest_runtest_call, + run_tasks, + testbed) + +(login) +(pytest_configure) +(pytest_runtest_call) +(run_tasks) +(testbed) diff --git a/appengine/mailgun/main_test.py b/appengine/mailgun/main_test.py index 8270716b84f..de92f476f29 100644 --- a/appengine/mailgun/main_test.py +++ b/appengine/mailgun/main_test.py @@ -12,12 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +from googleapiclient.http import HttpMockSequence +import httplib2 import main +import mock import pytest -from testing import Http2Mock import webtest +class HttpMockSequenceWithCredentials(HttpMockSequence): + def add_credentials(self, *args): + pass + + @pytest.fixture def app(): return webtest.TestApp(main.app) @@ -29,27 +36,31 @@ def test_get(app): def test_post(app): - http = Http2Mock(responses=[{}]) + http = HttpMockSequenceWithCredentials([ + ({'status': '200'}, '')]) + patch_http = mock.patch.object(httplib2, 'Http', lambda: http) - with http: + with patch_http: response = app.post('/', { 'recipient': 'jonwayne@google.com', 'submit': 'Send simple email'}) assert response.status_int == 200 - http = Http2Mock(responses=[{}]) + http = HttpMockSequenceWithCredentials([ + ({'status': '200'}, '')]) - with http: + with patch_http: response = app.post('/', { 'recipient': 'jonwayne@google.com', 'submit': 'Send complex email'}) assert response.status_int == 200 - http = Http2Mock(responses=[{'status': 500, 'body': 'Test error'}]) + http = HttpMockSequenceWithCredentials([ + ({'status': '500'}, 'Test error')]) - with http, pytest.raises(Exception): + with patch_http, pytest.raises(Exception): app.post('/', { 'recipient': 'jonwayne@google.com', 'submit': 'Send simple email'}) diff --git a/appengine/storage/main_test.py b/appengine/storage/main_test.py index fd9b9fd32b1..e68fe66a16d 100644 --- a/appengine/storage/main_test.py +++ b/appengine/storage/main_test.py @@ -18,7 +18,7 @@ def test_get(cloud_config): - main.BUCKET_NAME = cloud_config.GCLOUD_PROJECT + main.BUCKET_NAME = cloud_config.project app = webtest.TestApp(main.app) response = app.get('/') diff --git a/bigquery/api/async_query_test.py b/bigquery/api/async_query_test.py index 8aabadd0fed..f8e9f06891d 100644 --- a/bigquery/api/async_query_test.py +++ b/bigquery/api/async_query_test.py @@ -22,7 +22,7 @@ def test_async_query(cloud_config, capsys): 'GROUP BY corpus;') main( - project_id=cloud_config.GCLOUD_PROJECT, + project_id=cloud_config.project, query_string=query, batch=False, num_retries=5, diff --git a/bigquery/api/export_data_to_cloud_storage_test.py b/bigquery/api/export_data_to_cloud_storage_test.py index 86204fc05f5..72a41069199 100644 --- a/bigquery/api/export_data_to_cloud_storage_test.py +++ b/bigquery/api/export_data_to_cloud_storage_test.py @@ -12,19 +12,19 @@ # limitations under the License. from export_data_to_cloud_storage import main -from testing import mark_flaky +from gcp.testing.flaky import flaky DATASET_ID = 'test_dataset' TABLE_ID = 'test_table' -@mark_flaky +@flaky def test_export_table_csv(cloud_config): cloud_storage_output_uri = \ - 'gs://{}/output.csv'.format(cloud_config.CLOUD_STORAGE_BUCKET) + 'gs://{}/output.csv'.format(cloud_config.storage_bucket) main( cloud_storage_output_uri, - cloud_config.GCLOUD_PROJECT, + cloud_config.project, DATASET_ID, TABLE_ID, num_retries=5, @@ -32,13 +32,13 @@ def test_export_table_csv(cloud_config): export_format="CSV") -@mark_flaky +@flaky def test_export_table_json(cloud_config): cloud_storage_output_uri = \ - 'gs://{}/output.json'.format(cloud_config.CLOUD_STORAGE_BUCKET) + 'gs://{}/output.json'.format(cloud_config.storage_bucket) main( cloud_storage_output_uri, - cloud_config.GCLOUD_PROJECT, + cloud_config.project, DATASET_ID, TABLE_ID, num_retries=5, @@ -46,13 +46,13 @@ def test_export_table_json(cloud_config): export_format="NEWLINE_DELIMITED_JSON") -@mark_flaky +@flaky def test_export_table_avro(cloud_config): cloud_storage_output_uri = \ - 'gs://{}/output.avro'.format(cloud_config.CLOUD_STORAGE_BUCKET) + 'gs://{}/output.avro'.format(cloud_config.storage_bucket) main( cloud_storage_output_uri, - cloud_config.GCLOUD_PROJECT, + cloud_config.project, DATASET_ID, TABLE_ID, num_retries=5, diff --git a/bigquery/api/getting_started_test.py b/bigquery/api/getting_started_test.py index d0ff01b71e0..8f0866f46d1 100644 --- a/bigquery/api/getting_started_test.py +++ b/bigquery/api/getting_started_test.py @@ -17,7 +17,7 @@ def test_main(cloud_config, capsys): - main(cloud_config.GCLOUD_PROJECT) + main(cloud_config.project) out, _ = capsys.readouterr() diff --git a/bigquery/api/list_datasets_projects_test.py b/bigquery/api/list_datasets_projects_test.py index 588086807b0..e096a0342b2 100644 --- a/bigquery/api/list_datasets_projects_test.py +++ b/bigquery/api/list_datasets_projects_test.py @@ -17,7 +17,7 @@ def test_main(cloud_config, capsys): - main(cloud_config.GCLOUD_PROJECT) + main(cloud_config.project) out, _ = capsys.readouterr() diff --git a/bigquery/api/load_data_by_post_test.py b/bigquery/api/load_data_by_post_test.py index 876c8748c9f..ca4d1ea38e9 100644 --- a/bigquery/api/load_data_by_post_test.py +++ b/bigquery/api/load_data_by_post_test.py @@ -13,14 +13,14 @@ import re +from gcp.testing.flaky import flaky from load_data_by_post import load_data -from testing import mark_flaky DATASET_ID = 'ephemeral_test_dataset' TABLE_ID = 'load_data_by_post' -@mark_flaky +@flaky def test_load_csv_data(cloud_config, resource, capsys): schema_path = resource('schema.json') data_path = resource('data.csv') @@ -28,7 +28,7 @@ def test_load_csv_data(cloud_config, resource, capsys): load_data( schema_path, data_path, - cloud_config.GCLOUD_PROJECT, + cloud_config.project, DATASET_ID, TABLE_ID ) @@ -39,7 +39,7 @@ def test_load_csv_data(cloud_config, resource, capsys): r'Waiting for job to finish.*Job complete.', re.DOTALL), out) -@mark_flaky +@flaky def test_load_json_data(cloud_config, resource, capsys): schema_path = resource('schema.json') data_path = resource('data.json') @@ -47,7 +47,7 @@ def test_load_json_data(cloud_config, resource, capsys): load_data( schema_path, data_path, - cloud_config.GCLOUD_PROJECT, + cloud_config.project, DATASET_ID, TABLE_ID ) diff --git a/bigquery/api/load_data_from_csv_test.py b/bigquery/api/load_data_from_csv_test.py index 0cc64adb423..6e7fc4323db 100644 --- a/bigquery/api/load_data_from_csv_test.py +++ b/bigquery/api/load_data_from_csv_test.py @@ -11,22 +11,21 @@ # See the License for the specific language governing permissions and # limitations under the License. - +from gcp.testing.flaky import flaky from load_data_from_csv import main -from testing import mark_flaky DATASET_ID = 'test_dataset' TABLE_ID = 'test_import_table' -@mark_flaky +@flaky def test_load_table(cloud_config, resource): cloud_storage_input_uri = 'gs://{}/data.csv'.format( - cloud_config.CLOUD_STORAGE_BUCKET) + cloud_config.storage_bucket) schema_file = resource('schema.json') main( - cloud_config.GCLOUD_PROJECT, + cloud_config.project, DATASET_ID, TABLE_ID, schema_file=schema_file, diff --git a/bigquery/api/streaming_test.py b/bigquery/api/streaming_test.py index de22c58d202..66347da7cc0 100644 --- a/bigquery/api/streaming_test.py +++ b/bigquery/api/streaming_test.py @@ -27,7 +27,7 @@ def test_stream_row_to_bigquery(cloud_config, resource, capsys): streaming.get_rows = lambda: rows streaming.main( - cloud_config.GCLOUD_PROJECT, + cloud_config.project, DATASET_ID, TABLE_ID, num_retries=5) diff --git a/bigquery/api/sync_query_test.py b/bigquery/api/sync_query_test.py index 7f1d196f2b6..02d84f7c9c1 100644 --- a/bigquery/api/sync_query_test.py +++ b/bigquery/api/sync_query_test.py @@ -22,7 +22,7 @@ def test_sync_query(cloud_config, capsys): 'GROUP BY corpus;') main( - project_id=cloud_config.GCLOUD_PROJECT, + project_id=cloud_config.project, query=query, timeout=30, num_retries=5) diff --git a/blog/introduction_to_data_models_in_cloud_datastore/blog_test.py b/blog/introduction_to_data_models_in_cloud_datastore/blog_test.py index 4959229e7b2..16f41c883ec 100644 --- a/blog/introduction_to_data_models_in_cloud_datastore/blog_test.py +++ b/blog/introduction_to_data_models_in_cloud_datastore/blog_test.py @@ -12,9 +12,9 @@ # limitations under the License. from blog import main -from testing import mark_flaky +from gcp.testing.flaky import flaky -@mark_flaky +@flaky def test_main(cloud_config): - main(cloud_config.GCLOUD_PROJECT) + main(cloud_config.project) diff --git a/blog/introduction_to_data_models_in_cloud_datastore/wiki_test.py b/blog/introduction_to_data_models_in_cloud_datastore/wiki_test.py index a4ff93d97f0..7db45420ebe 100644 --- a/blog/introduction_to_data_models_in_cloud_datastore/wiki_test.py +++ b/blog/introduction_to_data_models_in_cloud_datastore/wiki_test.py @@ -11,10 +11,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from testing import mark_flaky +from gcp.testing.flaky import flaky from wiki import main -@mark_flaky +@flaky def test_main(cloud_config): - main(cloud_config.GCLOUD_PROJECT) + main(cloud_config.project) diff --git a/cloud_logging/api/list_logs_test.py b/cloud_logging/api/list_logs_test.py index 031f0680662..f7f94cf84b1 100644 --- a/cloud_logging/api/list_logs_test.py +++ b/cloud_logging/api/list_logs_test.py @@ -17,6 +17,6 @@ def test_main(cloud_config, capsys): - list_logs.main(cloud_config.GCLOUD_PROJECT) + list_logs.main(cloud_config.project) out, _ = capsys.readouterr() assert re.search(re.compile(r'.*', re.S), out) diff --git a/compute/api/create_instance_test.py b/compute/api/create_instance_test.py index 70de82a33c1..f85dcf09d0e 100644 --- a/compute/api/create_instance_test.py +++ b/compute/api/create_instance_test.py @@ -14,14 +14,14 @@ import re from create_instance import main -from testing import mark_flaky +from gcp.testing.flaky import flaky -@mark_flaky +@flaky def test_main(cloud_config, capsys): main( - cloud_config.GCLOUD_PROJECT, - cloud_config.CLOUD_STORAGE_BUCKET, + cloud_config.project, + cloud_config.storage_bucket, 'us-central1-f', 'test-instance', wait=False) diff --git a/conftest.py b/conftest.py index 049118194e6..92a808d07b0 100644 --- a/conftest.py +++ b/conftest.py @@ -1,13 +1,30 @@ import os import pytest -from testing.cloud import Config, get_resource_path + + +class Namespace: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) @pytest.fixture(scope='session') def cloud_config(): - """Provides a configuration option as a proxy to environment variables.""" - return Config() + """Provides a configuration object as a proxy to environment variables.""" + return Namespace( + project=os.environ.get('GCLOUD_PROJECT'), + storage_bucket=os.environ.get('CLOUD_STORAGE_BUCKET'), + client_secrets=os.environ.get('GOOGLE_CLIENT_SECRETS')) + + +def get_resource_path(resource, local_path): + local_resource_path = os.path.join(local_path, 'resources', *resource) + + if os.path.exists(local_resource_path): + return local_resource_path + else: + raise EnvironmentError('Resource {} not found.'.format( + os.path.join(*resource))) @pytest.fixture(scope='module') diff --git a/datastore/api/snippets_test.py b/datastore/api/snippets_test.py index 9bb65dbbc3d..974578f9ef4 100644 --- a/datastore/api/snippets_test.py +++ b/datastore/api/snippets_test.py @@ -14,9 +14,9 @@ import time from gcloud import datastore +from gcp.testing.flaky import flaky import pytest import snippets -from testing import mark_flaky class CleanupClient(datastore.Client): @@ -43,19 +43,19 @@ def put_multi(self, *args, **kwargs): @pytest.yield_fixture def client(cloud_config): - client = CleanupClient(cloud_config.GCLOUD_PROJECT) + client = CleanupClient(cloud_config.project) yield client client.cleanup() @pytest.yield_fixture def waiting_client(cloud_config): - client = WaitingClient(cloud_config.GCLOUD_PROJECT) + client = WaitingClient(cloud_config.project) yield client client.cleanup() -@mark_flaky +@flaky class TestDatastoreSnippets: # These tests mostly just test the absence of exceptions. def test_incomplete_key(self, client): diff --git a/datastore/api/tasks_test.py b/datastore/api/tasks_test.py index 98358997030..d8901d8f244 100644 --- a/datastore/api/tasks_test.py +++ b/datastore/api/tasks_test.py @@ -12,14 +12,14 @@ # limitations under the License. from gcloud import datastore +from gcp.testing.flaky import flaky import pytest import tasks -from testing import mark_flaky @pytest.yield_fixture def client(cloud_config): - client = datastore.Client(cloud_config.GCLOUD_PROJECT) + client = datastore.Client(cloud_config.project) yield client @@ -29,12 +29,12 @@ def client(cloud_config): [x.key for x in client.query(kind='Task').fetch()]) -@mark_flaky +@flaky def test_create_client(cloud_config): - tasks.create_client(cloud_config.GCLOUD_PROJECT) + tasks.create_client(cloud_config.project) -@mark_flaky +@flaky def test_add_task(client): task_key = tasks.add_task(client, 'Test task') task = client.get(task_key) @@ -42,7 +42,7 @@ def test_add_task(client): assert task['description'] == 'Test task' -@mark_flaky +@flaky def test_mark_done(client): task_key = tasks.add_task(client, 'Test task') tasks.mark_done(client, task_key.id) @@ -51,7 +51,7 @@ def test_mark_done(client): assert task['done'] -@mark_flaky +@flaky def test_list_tasks(client): task1_key = tasks.add_task(client, 'Test task 1') task2_key = tasks.add_task(client, 'Test task 2') @@ -59,14 +59,14 @@ def test_list_tasks(client): assert [x.key for x in task_list] == [task1_key, task2_key] -@mark_flaky +@flaky def test_delete_task(client): task_key = tasks.add_task(client, 'Test task 1') tasks.delete_task(client, task_key.id) assert client.get(task_key) is None -@mark_flaky +@flaky def test_format_tasks(client): task1_key = tasks.add_task(client, 'Test task 1') tasks.add_task(client, 'Test task 2') diff --git a/monitoring/api/auth_test.py b/monitoring/api/auth_test.py index 4d0d37a029a..917e33e870d 100644 --- a/monitoring/api/auth_test.py +++ b/monitoring/api/auth_test.py @@ -17,7 +17,7 @@ def test_main(cloud_config, capsys): - auth.main(cloud_config.GCLOUD_PROJECT) + auth.main(cloud_config.project) output, _ = capsys.readouterr() assert re.search( diff --git a/scripts/encrypt-secrets.sh b/scripts/encrypt-secrets.sh index 56e6f2d63a1..6bdbf9421d4 100755 --- a/scripts/encrypt-secrets.sh +++ b/scripts/encrypt-secrets.sh @@ -17,7 +17,7 @@ read -s -p "Enter password for encryption: " password echo -tar cvf secrets.tar testing/resources/{service-account.json,test-env.sh} +tar cvf secrets.tar testing/resources/{service-account.json,client-secrets.json,test-env.sh} openssl aes-256-cbc -k "$password" -in secrets.tar -out secrets.tar.enc rm secrets.tar diff --git a/secrets.tar.enc b/secrets.tar.enc index 24e60b17ab7..3dc345619bd 100644 Binary files a/secrets.tar.enc and b/secrets.tar.enc differ diff --git a/storage/api/compose_objects_test.py b/storage/api/compose_objects_test.py index 11fb41be1ab..131adc6279d 100644 --- a/storage/api/compose_objects_test.py +++ b/storage/api/compose_objects_test.py @@ -16,7 +16,7 @@ def test_main(cloud_config, resource): main( - cloud_config.CLOUD_STORAGE_BUCKET, + cloud_config.storage_bucket, 'dest.txt', [resource('file1.txt'), resource('file2.txt')] diff --git a/storage/api/list_objects_test.py b/storage/api/list_objects_test.py index 5fdab835301..9910bba57e8 100644 --- a/storage/api/list_objects_test.py +++ b/storage/api/list_objects_test.py @@ -15,4 +15,4 @@ def test_main(cloud_config): - main(cloud_config.CLOUD_STORAGE_BUCKET) + main(cloud_config.storage_bucket) diff --git a/testing/__init__.py b/testing/__init__.py deleted file mode 100644 index f5c1f76ce62..00000000000 --- a/testing/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .cloud import CloudTest -from .flaky import flaky_filter, mark_flaky -from .utils import capture_stdout, Http2Mock - - -__all__ = [ - 'capture_stdout', - 'CloudTest', - 'flaky_filter', - 'Http2Mock', - 'mark_flaky', -] diff --git a/testing/appengine.py b/testing/appengine.py deleted file mode 100644 index 92b721afd2b..00000000000 --- a/testing/appengine.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Common testing tools for Google App Engine tests. -""" - -import sys -import tempfile - -try: - APPENGINE_AVAILABLE = True - from google.appengine.datastore import datastore_stub_util - from google.appengine.ext import testbed - from google.appengine.api import namespace_manager -except ImportError: - APPENGINE_AVAILABLE = False - - -def setup_sdk_imports(): - """Sets up appengine SDK third-party imports.""" - if 'google' in sys.modules: - # Some packages, such as protobuf, clobber the google - # namespace package. This prevents that. - reload(sys.modules['google']) - - # This sets up google-provided libraries. - import dev_appserver - dev_appserver.fix_sys_path() - - # Fixes timezone and other os-level items. - import google.appengine.tools.os_compat - (google.appengine.tools.os_compat) - - -def import_appengine_config(): - """Imports an application appengine_config.py. This is used to - mimic the behavior of the runtime.""" - try: - import appengine_config - (appengine_config) - except ImportError: - pass - - -def setup_testbed(): - # Setup the datastore and memcache stub. - # First, create an instance of the Testbed class. - tb = testbed.Testbed() - # Then activate the testbed, which prepares the service stubs for - # use. - tb.activate() - # Create a consistency policy that will simulate the High - # Replication consistency model. - policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy( - probability=0) - # Initialize the datastore stub with this policy. - tb.init_datastore_v3_stub( - datastore_file=tempfile.mkstemp()[1], - consistency_policy=policy) - tb.init_memcache_stub() - - # Setup remaining stubs. - tb.init_app_identity_stub() - tb.init_blobstore_stub() - tb.init_user_stub() - tb.init_logservice_stub() - # tb.init_taskqueue_stub(root_path='tests/resources') - tb.init_taskqueue_stub() - tb.taskqueue_stub = tb.get_stub(testbed.TASKQUEUE_SERVICE_NAME) - - return tb - - -def run_tasks(testbed, app): - tasks = testbed.taskqueue_stub.get_filtered_tasks() - for task in tasks: - namespace = task.headers.get('X-AppEngine-Current-Namespace', '') - previous_namespace = namespace_manager.get_namespace() - try: - namespace_manager.set_namespace(namespace) - app.post( - task.url, - task.extract_params(), - headers={ - k: v for k, v in task.headers.iteritems() - if k.startswith('X-AppEngine')}) - finally: - namespace_manager.set_namespace(previous_namespace) diff --git a/testing/cloud.py b/testing/cloud.py deleted file mode 100644 index 039f9901ae7..00000000000 --- a/testing/cloud.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Common testing tools for cloud samples. -""" - -import inspect -import os -import unittest - -from .utils import silence_requests - - -GLOBAL_RESOURCE_PATH = os.path.join( - os.path.abspath(os.path.dirname(__file__)), 'resources') - - -# Note: these values must also be whitelisted in tox.ini. -ENVIRONMENT_VARIABLES = frozenset(( - 'GCLOUD_PROJECT', - 'CLOUD_STORAGE_BUCKET', -)) - - -class Config(object): - def __getattr__(self, name): - if name not in ENVIRONMENT_VARIABLES: - raise AttributeError( - 'Environment variable {} is not in the whitelist.' - .format(name)) - - if name not in os.environ: - raise EnvironmentError( - 'Environment variable {} not set.'.format(name)) - - return os.environ[name] - - -def get_resource_path(resource, local_path): - global_resource_path = os.path.join(GLOBAL_RESOURCE_PATH, *resource) - local_resource_path = os.path.join(local_path, 'resources', *resource) - - if os.path.exists(local_resource_path): - return local_resource_path - - if os.path.exists(global_resource_path): - return global_resource_path - - raise EnvironmentError('Resource {} not found.'.format( - os.path.join(*resource))) - - -class CloudTest(unittest.TestCase): - """Common base class for cloud tests.""" - def __init__(self, *args, **kwargs): - super(CloudTest, self).__init__(*args, **kwargs) - self.config = Config() - - @classmethod - def setUpClass(self): - silence_requests() - - def resource_path(self, *resource): - local_path = os.path.dirname(inspect.getfile(self.__class__)) - return get_resource_path(resource, local_path) diff --git a/testing/flaky.py b/testing/flaky.py deleted file mode 100644 index 2174a9a832c..00000000000 --- a/testing/flaky.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Tools for dealing with flaky tests. -""" -from __future__ import absolute_import - - -from flaky import flaky -import gcloud -import pytest - - -def flaky_filter(e, *args): - """Used by mark_flaky to retry on remote service errors.""" - exception_class, exception_instance, traceback = e - return isinstance( - exception_instance, - (gcloud.exceptions.GCloudError,)) - - -def mark_flaky(f): - """Makes a test retry on remote service errors.""" - return flaky(max_runs=3, rerun_filter=flaky_filter)( - pytest.mark.flaky(f)) diff --git a/testing/utils.py b/testing/utils.py deleted file mode 100644 index 8c3d075c322..00000000000 --- a/testing/utils.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Common testing utilities between samples -""" - -import contextlib -import logging -import sys - -import httplib2 -from six.moves import cStringIO - - -def silence_requests(): - """Silence requests' noisy logging.""" - logging.getLogger("requests").setLevel(logging.WARNING) - logging.getLogger("urrlib3").setLevel(logging.WARNING) - - -@contextlib.contextmanager -def capture_stdout(): - """Capture stdout to a StringIO object.""" - fake_stdout = cStringIO() - old_stdout = sys.stdout - - try: - sys.stdout = fake_stdout - yield fake_stdout - finally: - sys.stdout = old_stdout - - -class Http2Mock(object): - """Mock httplib2.Http""" - - def __init__(self, responses): - self.responses = responses - - def add_credentials(self, user, pwd): - self.credentials = (user, pwd) - - def request(self, token_uri, method, body, headers=None, *args, **kwargs): - response = self.responses.pop(0) - self.status = response.get('status', 200) - self.body = response.get('body', '') - self.headers = response.get('headers', '') - return (self, self.body) - - def __enter__(self): - self.httplib2_orig = httplib2.Http - httplib2.Http = self - return self - - def __exit__(self, exc_type, exc_value, traceback): - httplib2.Http = self.httplib2_orig - - def __call__(self, *args, **kwargs): - return self diff --git a/tox.ini b/tox.ini index e63665c4fa1..5e8939b42b4 100644 --- a/tox.ini +++ b/tox.ini @@ -36,6 +36,7 @@ commands = [testenv:py27] deps = + git+https://github.com/GoogleCloudPlatform/python-repo-tools -rrequirements-py27-dev.txt commands = python scripts/run-tests.py {posargs:{[testenv]examples}} @@ -52,6 +53,7 @@ commands = [testenv:py34] basepython = python3.4 deps = + git+https://github.com/GoogleCloudPlatform/python-repo-tools -rrequirements-py34-dev.txt commands = python scripts/run-tests.py {posargs:{[testenv]examples}} @@ -68,6 +70,7 @@ commands = [testenv:gae] deps = + git+https://github.com/GoogleCloudPlatform/python-repo-tools -rrequirements-py27-dev.txt examples = appengine/app_identity/signing @@ -86,17 +89,19 @@ examples = appengine/ndb/transactions appengine/storage commands = + gcprepotools download-appengine-sdk {env:GAE_ROOT} # Create a lib directory, otherwise, the vendor library will explode. mkdir -p lib python scripts/run-tests.py {posargs:{[testenv:gae]examples}} setenv = - PYTHONPATH=.:{env:GAE_PYTHONPATH:} + PYTHONPATH=.:{env:GAE_ROOT:}/google_appengine whitelist_externals = mkdir [testenv:gae-all] deps = {[testenv:gae]deps} commands = + gcprepotools download-appengine-sdk {env:GAE_ROOT} # Create a lib directory, otherwise, the vendor library will explode. mkdir -p lib python scripts/run-tests.py \ @@ -104,28 +109,30 @@ commands = --run-slow \ {posargs:{[testenv:gae]examples}} setenv = - PYTHONPATH=.:{env:GAE_PYTHONPATH:} + PYTHONPATH=.:{env:GAE_ROOT:}/google_appengine whitelist_externals = mkdir [testenv:reqcheck] +deps= + git+https://github.com/GoogleCloudPlatform/python-repo-tools commands = bash -c "find . -name requirements\*.txt |\ - xargs -P 4 -n 1 gcp-python-repo-tools check-requirements" + xargs -P 4 -n 1 gcprepotools check-requirements" whitelist_externals = bash find xargs - gcp-python-repo-tools [testenv:requpdate] +deps= + git+https://github.com/GoogleCloudPlatform/python-repo-tools commands = bash -c "find . -name requirements\*.txt |\ - xargs -P 4 -n 1 gcp-python-repo-tools update-requirements" + xargs -P 4 -n 1 gcprepotools update-requirements" whitelist_externals = bash find xargs - gcp-python-repo-tools [flake8] exclude=container_engine/django_tutorial/polls/migrations/*