Merge pull request #462 from GoogleCloudPlatform/fixup-nox

Author: dpebot

appengine/standard/conftest.py

@@ -12,16 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+
 # Import py.test hooks and fixtures for App Engine
 from gcp.testing.appengine import (
     login,
     pytest_configure,
     pytest_runtest_call,
     run_tasks,
     testbed)
+import six
 
 (login)
 (pytest_configure)
 (pytest_runtest_call)
 (run_tasks)
 (testbed)
+
+
+def pytest_ignore_collect(path, config):
+    """Skip App Engine tests in python 3 or if no SDK is available."""
+    if 'appengine/standard' in str(path):
+        if six.PY3:
+            return True
+        if 'GAE_SDK_PATH' not in os.environ:
+            return True
+    return False

bigtable/hello/main.py

@@ -70,7 +70,7 @@ def main(project_id, instance_id, table_id):
             row = table.row(row_key)
             row.set_cell(
                 column_family_id,
-                column_id.encode('utf-8'),
+                column_id,
                 value.encode('utf-8'))
             row.commit()
         # [END writing_rows]
@@ -79,7 +79,7 @@ def main(project_id, instance_id, table_id):
         print('Getting a single greeting by row key.')
         key = 'greeting0'
         row = table.read_row(key.encode('utf-8'))
-        value = row.cells[column_family_id][column_id.encode('utf-8')][0].value
+        value = row.cells[column_family_id][column_id][0].value
         print('\t{}: {}'.format(key, value.decode('utf-8')))
         # [END getting_a_row]
 
@@ -90,7 +90,7 @@ def main(project_id, instance_id, table_id):
 
         for row_key, row in partial_rows.rows.items():
             key = row_key.decode('utf-8')
-            cell = row.cells[column_family_id][column_id.encode('utf-8')][0]
+            cell = row.cells[column_family_id][column_id][0]
             value = cell.value.decode('utf-8')
             print('\t{}: {}'.format(key, value))
         # [END scanning_all_rows]

bigtable/hello/main_test.py

@@ -13,21 +13,13 @@
 # limitations under the License.
 
 import random
-import re
-import sys
 
 from main import main
 
-import pytest
-
-TABLE_NAME_FORMAT = 'Hello-Bigtable-{}'
+TABLE_NAME_FORMAT = 'hell-bigtable-system-tests-{}'
 TABLE_NAME_RANGE = 10000
 
 
-@pytest.mark.skipif(
-    sys.version_info >= (3, 0),
-    reason=("grpc doesn't yet support python3 "
-            'https://github.com/grpc/grpc/issues/282'))
 def test_main(cloud_config, capsys):
     table_name = TABLE_NAME_FORMAT.format(
         random.randrange(TABLE_NAME_RANGE))
@@ -37,12 +29,10 @@ def test_main(cloud_config, capsys):
         table_name)
 
     out, _ = capsys.readouterr()
-    assert re.search(
-        re.compile(r'Creating the Hello-Bigtable-[0-9]+ table\.'), out)
-    assert re.search(re.compile(r'Writing some greetings to the table\.'), out)
-    assert re.search(re.compile(r'Getting a single greeting by row key.'), out)
-    assert re.search(re.compile(r'greeting0: Hello World!'), out)
-    assert re.search(re.compile(r'Scanning for all greetings'), out)
-    assert re.search(re.compile(r'greeting1: Hello Cloud Bigtable!'), out)
-    assert re.search(
-        re.compile(r'Deleting the Hello-Bigtable-[0-9]+ table\.'), out)
+    assert 'Creating the {} table.'.format(table_name) in out
+    assert 'Writing some greetings to the table.' in out
+    assert 'Getting a single greeting by row key.' in out
+    assert 'Hello World!' in out
+    assert 'Scanning for all greetings' in out
+    assert 'Hello Cloud Bigtable!' in out
+    assert 'Deleting the {} table.'.format(table_name) in out

bigtable/hello_happybase/main.py

@@ -77,16 +77,16 @@ def main(project_id, instance_id, table_name):
 
         # [START getting_a_row]
         print('Getting a single greeting by row key.')
-        key = 'greeting0'
+        key = 'greeting0'.encode('utf-8')
         row = table.row(key)
-        print('\t{}: {}'.format(key, row[column_name]))
+        print('\t{}: {}'.format(key, row[column_name.encode('utf-8')]))
         # [END getting_a_row]
 
         # [START scanning_all_rows]
         print('Scanning for all greetings:')
 
         for key, row in table.scan():
-            print('\t{}: {}'.format(key, row[column_name]))
+            print('\t{}: {}'.format(key, row[column_name.encode('utf-8')]))
         # [END scanning_all_rows]
 
         # [START deleting_a_table]

bigtable/hello_happybase/main_test.py

@@ -13,21 +13,13 @@
 # limitations under the License.
 
 import random
-import re
-import sys
 
 from main import main
 
-import pytest
-
-TABLE_NAME_FORMAT = 'Hello-Bigtable-{}'
+TABLE_NAME_FORMAT = 'hello_happybase-system-tests-{}'
 TABLE_NAME_RANGE = 10000
 
 
-@pytest.mark.skipif(
-    sys.version_info >= (3, 0),
-    reason=("grpc doesn't yet support python3 "
-            'https://github.com/grpc/grpc/issues/282'))
 def test_main(cloud_config, capsys):
     table_name = TABLE_NAME_FORMAT.format(
         random.randrange(TABLE_NAME_RANGE))
@@ -37,12 +29,10 @@ def test_main(cloud_config, capsys):
         table_name)
 
     out, _ = capsys.readouterr()
-    assert re.search(
-        re.compile(r'Creating the Hello-Bigtable-[0-9]+ table\.'), out)
-    assert re.search(re.compile(r'Writing some greetings to the table\.'), out)
-    assert re.search(re.compile(r'Getting a single greeting by row key.'), out)
-    assert re.search(re.compile(r'greeting0: Hello World!'), out)
-    assert re.search(re.compile(r'Scanning for all greetings'), out)
-    assert re.search(re.compile(r'greeting1: Hello Cloud Bigtable!'), out)
-    assert re.search(
-        re.compile(r'Deleting the Hello-Bigtable-[0-9]+ table\.'), out)
+    assert 'Creating the {} table.'.format(table_name) in out
+    assert 'Writing some greetings to the table.' in out
+    assert 'Getting a single greeting by row key.' in out
+    assert 'Hello World!' in out
+    assert 'Scanning for all greetings' in out
+    assert 'Hello Cloud Bigtable!' in out
+    assert 'Deleting the {} table.'.format(table_name) in out

nox.py

@@ -30,7 +30,6 @@
 """
 
 import fnmatch
-import itertools
 import os
 import subprocess
 import tempfile
@@ -46,16 +45,6 @@
     '-x', '--no-success-flaky-report', '--cov', '--cov-config',
     '.coveragerc', '--cov-append', '--cov-report=']
 
-# Blacklists of samples to ingnore.
-# Bigtable and Speech are disabled because they use gRPC, which does not yet
-# support Python 3. See: https://github.com/grpc/grpc/issues/282
-TESTS_BLACKLIST = set((
-    './appengine/standard',
-    './bigtable',
-    './speech',
-    './testing'))
-APPENGINE_BLACKLIST = set()
-
 
 # Libraries that only work on Python 2.7
 PY27_ONLY_LIBRARIES = ['mysql-python']
@@ -132,8 +121,11 @@ def filter_samples(sample_dirs, changed_files):
 def setup_appengine(session):
     """Installs the App Engine SDK."""
     # Install the app engine sdk and setup import paths.
+    if session.interpreter.startswith('python3'):
+        return
+
     gae_root = os.environ.get('GAE_ROOT', tempfile.gettempdir())
-    session.env['PYTHONPATH'] = os.path.join(gae_root, 'google_appengine')
+    session.env['GAE_SDK_PATH'] = os.path.join(gae_root, 'google_appengine')
     session.run('gcprepotools', 'download-appengine-sdk', gae_root)
 
     # Create a lib directory to prevent the GAE vendor library from
@@ -143,8 +135,8 @@ def setup_appengine(session):
 
 
 def run_tests_in_sesssion(
-        session, interpreter, use_appengine=False, skip_flaky=False,
-        changed_only=False, sample_directories=None):
+        session, interpreter, sample_directories, use_appengine=True,
+        skip_flaky=False, changed_only=False):
     """This is the main function for executing tests.
 
     It:
@@ -173,13 +165,6 @@ def run_tests_in_sesssion(
     if skip_flaky:
         pytest_args.append('-m not slow and not flaky')
 
-    # session.posargs is any leftover arguments from the command line,
-    # which allows users to run a particular test instead of all of them.
-    if session.posargs:
-        sample_directories = session.posargs
-    elif sample_directories is None:
-        sample_directories = collect_sample_dirs('.', TESTS_BLACKLIST)
-
     if changed_only:
         changed_files = get_changed_files()
         sample_directories = filter_samples(
@@ -204,43 +189,38 @@ def run_tests_in_sesssion(
 
 @nox.parametrize('interpreter', ['python2.7', 'python3.4'])
 def session_tests(session, interpreter):
-    """Runs tests"""
-    run_tests_in_sesssion(session, interpreter)
-
+    """Runs tests for all non-gae standard samples."""
+    # session.posargs is any leftover arguments from the command line,
+    # which allows users to run a particular test instead of all of them.
+    sample_directories = session.posargs
+    if not sample_directories:
+        sample_directories = collect_sample_dirs('.')
 
-def session_gae(session):
-    """Runs test for GAE Standard samples."""
     run_tests_in_sesssion(
-        session, 'python2.7', use_appengine=True,
-        sample_directories=collect_sample_dirs(
-            'appengine/standard',
-            APPENGINE_BLACKLIST))
+        session, interpreter, sample_directories)
 
 
-def session_grpc(session):
-    """Runs tests for samples that need grpc."""
-    # TODO: Remove this when grpc supports Python 3.
+def session_gae(session):
+    """Runs test for GAE Standard samples."""
+    sample_directories = collect_sample_dirs('appengine/standard')
     run_tests_in_sesssion(
-        session,
-        'python2.7',
-        sample_directories=itertools.chain(
-            collect_sample_dirs('speech'),
-            collect_sample_dirs('bigtable')))
+        session, 'python2.7', sample_directories, use_appengine=True)
 
 
 @nox.parametrize('subsession', ['gae', 'tests'])
 def session_travis(session, subsession):
     """On travis, just run with python3.4 and don't run slow or flaky tests."""
     if subsession == 'tests':
+        sample_directories = collect_sample_dirs(
+            '.', set('./appengine/standard'))
         run_tests_in_sesssion(
-            session, 'python3.4', skip_flaky=True, changed_only=True)
+            session, 'python3.4', sample_directories,
+            skip_flaky=True, changed_only=True)
     else:
+        sample_directories = collect_sample_dirs('appengine/standard')
         run_tests_in_sesssion(
-            session, 'python2.7', use_appengine=True, skip_flaky=True,
-            changed_only=True,
-            sample_directories=collect_sample_dirs(
-                'appengine/standard',
-                APPENGINE_BLACKLIST))
+            session, 'python2.7', sample_directories,
+            skip_flaky=True, changed_only=True)
 
 
 def session_lint(session):

speech/api/speech_async_grpc_test.py

@@ -13,17 +13,12 @@
 
 import argparse
 import re
-import sys
 
 import pytest
 from speech_async_grpc import _gcs_uri
 from speech_async_grpc import main
 
 
-@pytest.mark.skipif(
-        sys.version_info >= (3, 0),
-        reason=("grpc doesn't yet support python3 "
-                'https://github.com/grpc/grpc/issues/282'))
 def test_main(cloud_config, capsys):
     input_uri = 'gs://{}/speech/audio.flac'.format(cloud_config.storage_bucket)
 

speech/api/speech_grpc_test.py

@@ -12,17 +12,12 @@
 # limitations under the License.
 
 import re
-import sys
 
 import pytest
 from speech_grpc import _gcs_uri
 from speech_grpc import main
 
 
-@pytest.mark.skipif(
-        sys.version_info >= (3, 0),
-        reason=("grpc doesn't yet support python3 "
-                'https://github.com/grpc/grpc/issues/282'))
 def test_main(cloud_config, capsys):
     input_uri = 'gs://{}/speech/audio.flac'.format(cloud_config.storage_bucket)
 

speech/api/speech_streaming_test.py

@@ -14,11 +14,8 @@
 import contextlib
 import io
 import re
-import sys
 import time
 
-import pytest
-
 import speech_streaming
 
 
@@ -57,10 +54,6 @@ def mock_audio_stream(channels, rate, chunk):
     return mock_audio_stream
 
 
-@pytest.mark.skipif(
-        sys.version_info >= (3, 0),
-        reason=("grpc doesn't yet support python3 "
-                'https://github.com/grpc/grpc/issues/282'))
 def test_main(resource, monkeypatch, capsys):
     monkeypatch.setattr(
         speech_streaming, 'record_audio',