test: enable tests to run on kokoro (#134)

Co-authored-by: Ilya Gurov <[email protected]>

Author: skuruppu

.circleci/config.yml

@@ -1,7 +1 @@
 # Format: //devtools/kokoro/config/proto/build.proto
-
-# Disable system tests.
-env_vars: {
-    key: "RUN_SYSTEM_TESTS"
-    value: "false"
-}

.kokoro/presubmit/presubmit.cfg

@@ -82,11 +82,11 @@ def create_test_instance():
     instance = CLIENT.instance(instance_id, instance_config, labels=labels)
 
     created_op = instance.create()
-    created_op.result(120)  # block until completion
+    created_op.result(1800)  # block until completion
 
     database = instance.database("compliance-test")
     created_op = database.create()
-    created_op.result(120)
+    created_op.result(1800)
 
     config = configparser.ConfigParser()
     url = "spanner:///projects/{project}/instances/{instance_id}/databases/compliance-test".format(

create_test_database.py

@@ -17,6 +17,14 @@
 
 
 class Requirements(SuiteRequirements):
+    @property
+    def sane_rowcount(self):
+        return exclusions.closed()
+
+    @property
+    def sane_multi_rowcount(self):
+        return exclusions.closed()
+
     @property
     def foreign_key_constraint_name_reflection(self):
         return exclusions.open()

google/cloud/sqlalchemy_spanner/requirements.py

@@ -394,7 +394,7 @@ class SpannerDialect(DefaultDialect):
     execute_sequence_format = list
 
     supports_alter = True
-    supports_sane_rowcount = True
+    supports_sane_rowcount = False
     supports_sane_multi_rowcount = False
     supports_default_values = False
     supports_sequences = True

google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py

@@ -14,24 +14,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import configparser
-import os
 import re
+import sys
 
 from google.cloud import spanner
 
-config = configparser.ConfigParser()
-if os.path.exists("test.cfg"):
-    config.read("test.cfg")
-else:
-    config.read("setup.cfg")
-db_url = config.get("db", "default")
 
-project = re.findall(r"projects(.*?)instances", db_url)
-instance_id = re.findall(r"instances(.*?)databases", db_url)
+def main(argv):
+  db_url = argv[0]
 
-client = spanner.Client(project="".join(project).replace("/", ""))
-instance = client.instance(instance_id="".join(instance_id).replace("/", ""))
-database = instance.database("compliance-test")
+  project = re.findall(r"projects(.*?)instances", db_url)
+  instance_id = re.findall(r"instances(.*?)databases", db_url)
 
-database.update_ddl(["DROP TABLE account", "DROP TABLE alembic_version"]).result(120)
+  client = spanner.Client(project="".join(project).replace("/", ""))
+  instance = client.instance(instance_id="".join(instance_id).replace("/", ""))
+  database = instance.database("compliance-test")
+
+  database.update_ddl(["DROP TABLE account", "DROP TABLE alembic_version"]).result(120)
+
+
+if __name__ == "__main__":
+   main(sys.argv[1:])

migration_test_cleanup.py

@@ -147,12 +147,20 @@ def migration_test(session):
     session.install("-e", ".")
     session.install("alembic")
 
+    project = os.getenv(
+        "GOOGLE_CLOUD_PROJECT", os.getenv("PROJECT_ID", "emulator-test-project"),
+    )
+    db_url = (
+        f"spanner:///projects/{project}/instances/"
+        "sqlalchemy-dialect-test/databases/compliance-test"
+    )
+
     config = configparser.ConfigParser()
     if os.path.exists("test.cfg"):
         config.read("test.cfg")
     else:
         config.read("setup.cfg")
-    db_url = config.get("db", "default")
+    db_url = config.get("db", "default", fallback=db_url)
 
     session.run("alembic", "init", "test_migration")
 
@@ -181,7 +189,7 @@ def migration_test(session):
     # clearing the migration data
     os.remove("alembic.ini")
     shutil.rmtree("test_migration")
-    session.run("python", "migration_test_cleanup.py")
+    session.run("python", "migration_test_cleanup.py", db_url)
     if os.path.exists("test.cfg"):
         os.remove("test.cfg")
 

noxfile.py

@@ -12,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import configparser
 import datetime
+import os
 import uuid
 
 import pytest
@@ -30,11 +32,21 @@
 
 @pytest.fixture
 def db_url():
-    return (
-        "spanner:///projects/appdev-soda-spanner-staging/instances/"
+    project = os.getenv(
+        "GOOGLE_CLOUD_PROJECT", os.getenv("PROJECT_ID", "emulator-test-project"),
+    )
+    db_url = (
+        f"spanner:///projects/{project}/instances/"
         "sqlalchemy-dialect-test/databases/compliance-test"
     )
 
+    config = configparser.ConfigParser()
+    if os.path.exists("test.cfg"):
+        config.read("test.cfg")
+    else:
+        config.read("setup.cfg")
+    return config.get("db", "default", fallback=db_url)
+
 
 @pytest.fixture
 def table_id():

samples/conftest.py

@@ -24,6 +24,3 @@ python_files=test/*test_*.py
 [sqla_testing]
 requirement_cls=google.cloud.sqlalchemy_spanner.requirements:Requirements
 profile_file=test/profiles.txt
-
-[db]
-default=spanner:///projects/appdev-soda-spanner-staging/instances/sqlalchemy-dialect-test/databases/compliance-test

setup.cfg

@@ -5,7 +5,9 @@
 # https://developers.google.com/open-source/licenses/bsd
 
 
+import configparser
 import mock
+import os
 from sqlalchemy.testing import fixtures
 
 try:
@@ -29,6 +31,24 @@
 _TEST_OT_PROVIDER_INITIALIZED = False
 
 
+PROJECT = os.getenv(
+    "GOOGLE_CLOUD_PROJECT", os.getenv("PROJECT_ID", "emulator-test-project"),
+)
+DB_URL = (
+    f"spanner:///projects/{PROJECT}/instances/"
+    "sqlalchemy-dialect-test/databases/compliance-test"
+)
+
+
+def get_db_url():
+    config = configparser.ConfigParser()
+    if os.path.exists("test.cfg"):
+        config.read("test.cfg")
+    else:
+        config.read("setup.cfg")
+    return config.get("db", "default", fallback=DB_URL)
+
+
 def get_test_ot_exporter():
     global _TEST_OT_EXPORTER
 

test/_helpers.py

@@ -111,6 +111,7 @@
     UnicodeTextTest as _UnicodeTextTest,
     _UnicodeFixture as _UnicodeFixtureTest,
 )
+from test._helpers import get_db_url
 
 config.test_schema = ""
 
@@ -678,7 +679,7 @@ def define_temp_tables(cls, metadata):
             Column("foo", sqlalchemy.INT),
             sqlalchemy.Index("user_tmp_uq", "name", unique=True),
             sqlalchemy.Index("user_tmp_ix", "foo"),
-            **kw
+            **kw,
         )
         if (
             testing.requires.view_reflection.enabled
@@ -1508,10 +1509,7 @@ class InterleavedTablesTest(fixtures.TestBase):
     """
 
     def setUp(self):
-        self._engine = create_engine(
-            "spanner:///projects/appdev-soda-spanner-staging/instances/"
-            "sqlalchemy-dialect-test/databases/compliance-test"
-        )
+        self._engine = create_engine(get_db_url())
         self._metadata = MetaData(bind=self._engine)
 
     def test_interleave(self):
@@ -1560,10 +1558,7 @@ class UserAgentTest(fixtures.TestBase):
     """Check that SQLAlchemy dialect uses correct user agent."""
 
     def setUp(self):
-        self._engine = create_engine(
-            "spanner:///projects/appdev-soda-spanner-staging/instances/"
-            "sqlalchemy-dialect-test/databases/compliance-test"
-        )
+        self._engine = create_engine(get_db_url())
         self._metadata = MetaData(bind=self._engine)
 
     def test_user_agent(self):
@@ -1583,10 +1578,7 @@ class ExecutionOptionsTest(fixtures.TestBase):
     """
 
     def setUp(self):
-        self._engine = create_engine(
-            "spanner:///projects/appdev-soda-spanner-staging/instances/"
-            "sqlalchemy-dialect-test/databases/compliance-test"
-        )
+        self._engine = create_engine(get_db_url())
         self._metadata = MetaData(bind=self._engine)
 
         self._table = Table(