Merge pull request #237 from GoogleCloudPlatform/nox

Auto-detect directories to test.

Author: jerjou

.coveragerc

@@ -8,6 +8,7 @@ include =
     datastore/*
     managed_vms/*
     monitoring/*
+    speech/*
     storage/*
 [report]
 exclude_lines =

TESTING.md

@@ -4,10 +4,10 @@ The tests in this repository are system tests and run against live services, the
 
 Before you can run tests locally you must have:
 
-* The latest [tox](https://tox.readthedocs.org/en/latest/),
+* The latest [nox](https://nox.readthedocs.org/en/latest/),
   [pip](https://pypi.python.org/pypi/pip), and [gcp-python-repo-tools](https://pypi.python.org/pypi/gcp-python-repo-tools) installed.
 
-        $ sudo pip install --upgrade tox pip gcp-python-repo-tools
+        $ sudo pip install --upgrade nox-automation pip gcp-python-repo-tools
 
 * The [Google Cloud SDK](https://cloud.google.com/sdk/) installed. You
   can do so with the following command:
@@ -50,29 +50,35 @@ If you want to run the Google App Engine tests, you will need:
 
 ### Test environments
 
-We use [tox](https://tox.readthedocs.org/en/latest/) to configure
-multiple python environments:
+We use [nox](https://nox.readthedocs.org/en/latest/) to configure
+multiple python sessions:
 
-* ``py27`` and ``py34`` contains tests for samples that run in a normal Python 2.7 pr 3.4 environment. This is everything outside of the ``appengine`` directory that isn't slow or flaky.
-* ``py27-all`` and ``py34-all`` runs all tests except for App Engine tests. This can time some time and some tests are flaky.
-* ``gae`` contains tests for samples that run only in Google App Engine. This is (mostly) everything in the ``appengine`` directory.
-* ``pep8`` just runs the linter.
+* ``tests`` contains tests for samples that run in a normal Python 2.7 or 3.4
+  environment. This is everything outside of the ``appengine`` directory. It's
+  parameterized to run all the tests using the 2.7 and 3.4 interpreters.
+* ``gae`` contains tests for samples that run only in Google App Engine. This is
+  (mostly) everything in the ``appengine`` directory.
+* ``lint`` just runs the linter.
 
-To run tests for a particular environment, invoke tox with the ``-e``
-flag:
+To see a list of the available sessions:
 
-    tox -e py27
+    nox -l
 
-To run one particular test suite or provide additional parameters to
-``py.test``, invoke tox like this:
+To run tests for a particular session, with a particular parameter, invoke nox
+with the ``-s`` flag:
 
-    toxe -e py27 -- storage/api
+    nox -s "tests(interpreter='python2.7')"
 
-### Adding new tests
+To run one particular session or provide additional parameters to ``py.test``,
+invoke nox like this:
+
+    nox -s tests -- storage/api
 
-When adding a new top-level directory, be sure to edit ``.coveragerc`` and ``tox.ini`` to include it in tests and coverage reporting.
+### Adding new tests
+When adding a new top-level directory, be sure to edit ``.coveragerc`` to
+include it in coverage reporting.
 
 To add new tests that require Google App Engine, please place them in
 the ``appengine`` directory if possible. If you place them elsewhere,
-you will need to modify ``tox.ini`` to make the environments
+you will need to modify ``nox.py`` to make the environments
 appropriately run or ignore your test.

appengine/localtesting/test_datastore.py renamed to appengine/localtesting/datastore_test.py

@@ -25,44 +25,7 @@
     '-x', '--no-success-flaky-report', '--cov', '--cov-config',
     '.coveragerc', '--cov-append', '--cov-report=']
 
-SAMPLES = [
-    'bigquery/api',
-    'blog/introduction_to_data_models_in_cloud_datastore',
-    'cloud_logging/api',
-    'compute/api',
-    'compute/autoscaler/demo',
-    'datastore/api',
-    'managed_vms/cloudsql',
-    'managed_vms/datastore',
-    'managed_vms/disk',
-    'managed_vms/extending_runtime',
-    'managed_vms/hello_world',
-    'managed_vms/hello_world_compat',
-    'managed_vms/memcache',
-    'managed_vms/pubsub',
-    'managed_vms/static_files',
-    'managed_vms/storage',
-    'monitoring/api',
-    'storage/api',
-]
-
-GAE_SAMPLES = [
-    'appengine/app_identity/signing',
-    'appengine/bigquery',
-    'appengine/blobstore',
-    'appengine/cloudsql',
-    'appengine/images',
-    'appengine/localtesting',
-    'appengine/logging/reading_logs',
-    'appengine/logging/writing_logs',
-    'appengine/mailgun',
-    'appengine/memcache/guestbook',
-    'appengine/multitenancy',
-    'appengine/ndb/modeling',
-    'appengine/ndb/overview',
-    'appengine/ndb/transactions',
-    'appengine/storage',
-]
+SESSION_TESTS_BLACKLIST = set(('appengine', 'testing'))
 
 
 def session_lint(session):
@@ -94,6 +57,21 @@ def session_reqcheck(session):
         session.run('gcprepotools', command, reqfile)
 
 
+def collect_sample_dirs(start_dir, blacklist=set()):
+    """Recursively collects a list of dirs that contain tests."""
+    # Collect all the directories that have tests in them.
+    for parent, subdirs, files in os.walk(start_dir):
+        if any(f for f in files if f[-8:] == '_test.py'):
+            # Don't recurse further, since py.test will do that.
+            del subdirs[:]
+            # This dir has tests in it. yield it.
+            yield parent
+        else:
+            # Filter out dirs we don't want to recurse into
+            subdirs[:] = [s for s in subdirs
+                          if s[0].isalpha() and s not in blacklist]
+
+
 @nox.parametrize('interpreter', ['python2.7', 'python3.4'])
 def session_tests(session, interpreter, extra_pytest_args=None):
     session.interpreter = interpreter
@@ -106,7 +84,8 @@ def session_tests(session, interpreter, extra_pytest_args=None):
 
     # session.posargs is any leftover arguments from the command line, which
     # allows users to run a particular test instead of all of them.
-    for sample in (session.posargs or SAMPLES):
+    for sample in (session.posargs or
+                   collect_sample_dirs('.', SESSION_TESTS_BLACKLIST)):
         session.run(
             'py.test', sample,
             *pytest_args,
@@ -130,7 +109,7 @@ def session_gae(session, extra_pytest_args=None):
 
     pytest_args = COMMON_PYTEST_ARGS + (extra_pytest_args or [])
 
-    for sample in (session.posargs or GAE_SAMPLES):
+    for sample in (session.posargs or collect_sample_dirs('appengine')):
         session.run(
             'py.test', sample,
             *pytest_args,