From 6151d748b51b3a4cbd2d8379c2fc5b482040a05d Mon Sep 17 00:00:00 2001 From: Shanika Kuruppu Date: Thu, 20 May 2021 14:57:02 +1000 Subject: [PATCH 01/10] test: add back emulator test workflow --- .github/workflows/test_suite.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/test_suite.yml diff --git a/.github/workflows/test_suite.yml b/.github/workflows/test_suite.yml new file mode 100644 index 00000000..7a6d4653 --- /dev/null +++ b/.github/workflows/test_suite.yml @@ -0,0 +1,31 @@ +on: + push: + branches: + - main + pull_request: +name: SQLAlchemy Spanner dialect +jobs: + tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install nox + run: python -m pip install nox + - name: Run SQLAlchemy tests + run: nox + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: appdev-soda-spanner-staging + From 4170b4b15b42408c67f01be68ef8218d98fe6f81 Mon Sep 17 00:00:00 2001 From: Shanika Kuruppu Date: Thu, 20 May 2021 15:01:24 +1000 Subject: [PATCH 02/10] fix: create project for tests against emulator --- create_test_database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/create_test_database.py b/create_test_database.py index d4a523de..7ce3e1aa 100644 --- a/create_test_database.py +++ b/create_test_database.py @@ -57,10 +57,10 @@ def prep_instance(): # Filter out non "us" locations configs = [config for config in configs if "-us-" in config.name] - instance_config = configs[0].name + instance_config = 'us-central1' if USE_EMULATOR else configs[0].name create_time = str(int(time.time())) unique_resource_id = '%s%d' % ('-', 1000 * time.time()) - instance_id = "sqlalchemy-test" + unique_resource_id + instance_id = 'sqlalchemy-dialect-test' if USE_EMULATOR else "sqlalchemy-test" + unique_resource_id labels = {"python-spanner-sqlalchemy-systest": "true", "created": create_time} instance = CLIENT.instance(instance_id, instance_config, labels=labels) From 35c93429a0cd20c1f7d232f242264a530aac37d7 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Thu, 20 May 2021 11:37:11 +0300 Subject: [PATCH 03/10] skip numeric --- test/test_suite.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/test_suite.py b/test/test_suite.py index 971bceee..44f59774 100644 --- a/test/test_suite.py +++ b/test/test_suite.py @@ -14,9 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import decimal import operator +import os import pytest -import decimal import pytz import sqlalchemy @@ -1008,6 +1009,10 @@ def test_text_roundtrip(self): eq_(row, ("some text",)) +@pytest.mark.skipif( + os.getenv("SPANNER_EMULATOR_HOST"), + reason="Numeric type isn't supported by emulator yet", +) class NumericTest(_NumericTest): @emits_warning(r".*does \*not\* support Decimal objects natively") def test_render_literal_numeric(self): From b9a6d541e7e49eda1594415098581b59ab55b494 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Thu, 27 May 2021 11:02:17 +0300 Subject: [PATCH 04/10] Revert "skip numeric" This reverts commit 976b73081e8464a258b327513ecea6c94bffadfc. --- test/test_suite.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/test/test_suite.py b/test/test_suite.py index 44f59774..971bceee 100644 --- a/test/test_suite.py +++ b/test/test_suite.py @@ -14,10 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import decimal import operator -import os import pytest +import decimal import pytz import sqlalchemy @@ -1009,10 +1008,6 @@ def test_text_roundtrip(self): eq_(row, ("some text",)) -@pytest.mark.skipif( - os.getenv("SPANNER_EMULATOR_HOST"), - reason="Numeric type isn't supported by emulator yet", -) class NumericTest(_NumericTest): @emits_warning(r".*does \*not\* support Decimal objects natively") def test_render_literal_numeric(self): From 5283379df379639a80eb9003fc8e6f0645f37c0b Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Thu, 27 May 2021 11:09:42 +0300 Subject: [PATCH 05/10] fix --- test/test_suite.py | 143 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/test/test_suite.py b/test/test_suite.py index 971bceee..40786f60 100644 --- a/test/test_suite.py +++ b/test/test_suite.py @@ -649,6 +649,149 @@ class UnicodeTextTest(UnicodeFixtureTest, _UnicodeTextTest): class ComponentReflectionTest(_ComponentReflectionTest): + @classmethod + def define_reflected_tables(cls, metadata, schema): + if schema: + schema_prefix = schema + "." + else: + schema_prefix = "" + + if testing.requires.self_referential_foreign_keys.enabled: + users = Table( + "users", + metadata, + Column("user_id", sqlalchemy.INT, primary_key=True), + Column("test1", sqlalchemy.CHAR(5), nullable=False), + Column("test2", sqlalchemy.Float(5), nullable=False), + Column( + "parent_user_id", + sqlalchemy.Integer, + sqlalchemy.ForeignKey( + "%susers.user_id" % schema_prefix, name="user_id_fk" + ), + ), + schema=schema, + test_needs_fk=True, + ) + else: + users = Table( + "users", + metadata, + Column("user_id", sqlalchemy.INT, primary_key=True), + Column("test1", sqlalchemy.CHAR(5), nullable=False), + Column("test2", sqlalchemy.Float(5), nullable=False), + schema=schema, + test_needs_fk=True, + ) + + Table( + "dingalings", + metadata, + Column("dingaling_id", sqlalchemy.Integer, primary_key=True), + Column( + "address_id", + sqlalchemy.Integer, + sqlalchemy.ForeignKey("%semail_addresses.address_id" % schema_prefix), + ), + Column("data", sqlalchemy.String(30)), + schema=schema, + test_needs_fk=True, + ) + Table( + "email_addresses", + metadata, + Column("address_id", sqlalchemy.Integer, primary_key=True), + Column( + "remote_user_id", + sqlalchemy.Integer, + sqlalchemy.ForeignKey(users.c.user_id), + ), + Column("email_address", sqlalchemy.String(20)), + sqlalchemy.PrimaryKeyConstraint("address_id", name="email_ad_pk"), + schema=schema, + test_needs_fk=True, + ) + Table( + "comment_test", + metadata, + Column("id", sqlalchemy.Integer, primary_key=True, comment="id comment"), + Column("data", sqlalchemy.String(20), comment="data % comment"), + Column( + "d2", + sqlalchemy.String(20), + comment=r"""Comment types type speedily ' " \ '' Fun!""", + ), + schema=schema, + comment=r"""the test % ' " \ table comment""", + ) + + if testing.requires.cross_schema_fk_reflection.enabled: + if schema is None: + Table( + "local_table", + metadata, + Column("id", sqlalchemy.Integer, primary_key=True), + Column("data", sqlalchemy.String(20)), + Column( + "remote_id", + ForeignKey("%s.remote_table_2.id" % testing.config.test_schema), + ), + test_needs_fk=True, + schema=config.db.dialect.default_schema_name, + ) + else: + Table( + "remote_table", + metadata, + Column("id", sqlalchemy.Integer, primary_key=True), + Column( + "local_id", + ForeignKey( + "%s.local_table.id" % config.db.dialect.default_schema_name + ), + ), + Column("data", sqlalchemy.String(20)), + schema=schema, + test_needs_fk=True, + ) + Table( + "remote_table_2", + metadata, + Column("id", sqlalchemy.Integer, primary_key=True), + Column("data", sqlalchemy.String(20)), + schema=schema, + test_needs_fk=True, + ) + + if testing.requires.index_reflection.enabled: + cls.define_index(metadata, users) + + if not schema: + # test_needs_fk is at the moment to force MySQL InnoDB + # noncol_idx_test_nopk = Table( + # "noncol_idx_test_nopk", + # metadata, + # Column("q", sqlalchemy.String(5)), + # test_needs_fk=True, + # ) + + noncol_idx_test_pk = Table( + "noncol_idx_test_pk", + metadata, + Column("id", sqlalchemy.Integer, primary_key=True), + Column("q", sqlalchemy.String(5)), + test_needs_fk=True, + ) + + if testing.requires.indexes_with_ascdesc.enabled: + # Index("noncol_idx_nopk", noncol_idx_test_nopk.c.q.desc()) + sqlalchemy.Index("noncol_idx_pk", noncol_idx_test_pk.c.q.desc()) + + if testing.requires.view_column_reflection.enabled: + cls.define_views(metadata, schema) + if not schema and testing.requires.temp_table_reflection.enabled: + cls.define_temp_tables(metadata) + @classmethod def define_temp_tables(cls, metadata): """ From 6b8656f28e09230d0e1dbb02eaad023ded9f0a93 Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 10 Jun 2021 13:21:30 -0400 Subject: [PATCH 06/10] chore: use dynamic instance_id --- create_test_database.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/create_test_database.py b/create_test_database.py index 8606c2b2..267b002e 100644 --- a/create_test_database.py +++ b/create_test_database.py @@ -56,13 +56,14 @@ def reap_old_instances(): def prep_instance(): configs = list(CLIENT.list_instance_configs()) - # Filter out non "us" locations - configs = [config for config in configs if "-us-" in config.name] + if not USE_EMULATOR: + # Filter out non "us" locations + configs = [config for config in configs if "-us-" in config.name] - instance_config = 'us-central1' if USE_EMULATOR else configs[0].name + instance_config = configs[0].name create_time = str(int(time.time())) unique_resource_id = '%s%d' % ('-', 1000 * time.time()) - instance_id = 'sqlalchemy-dialect-test' if USE_EMULATOR else "sqlalchemy-test" + unique_resource_id + instance_id = "sqlalchemy-test" + unique_resource_id labels = {"python-spanner-sqlalchemy-systest": "true", "created": create_time} instance = CLIENT.instance(instance_id, instance_config, labels=labels) From 2d1bca5dfd54ddd5c52f95d78357abf322419c97 Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Mon, 14 Jun 2021 16:56:20 +0530 Subject: [PATCH 07/10] fix: uncomment and fix noncol_idx_test_nopk test --- test/test_suite.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test/test_suite.py b/test/test_suite.py index 2883cb9a..fcbda8cc 100644 --- a/test/test_suite.py +++ b/test/test_suite.py @@ -599,12 +599,13 @@ def define_reflected_tables(cls, metadata, schema): if not schema: # test_needs_fk is at the moment to force MySQL InnoDB - # noncol_idx_test_nopk = Table( - # "noncol_idx_test_nopk", - # metadata, - # Column("q", sqlalchemy.String(5)), - # test_needs_fk=True, - # ) + noncol_idx_test_nopk = Table( + "noncol_idx_test_nopk", + metadata, + Column("id", sqlalchemy.Integer, primary_key=True), + Column("q", sqlalchemy.String(5)), + test_needs_fk=True, + ) noncol_idx_test_pk = Table( "noncol_idx_test_pk", @@ -615,7 +616,7 @@ def define_reflected_tables(cls, metadata, schema): ) if testing.requires.indexes_with_ascdesc.enabled: - # Index("noncol_idx_nopk", noncol_idx_test_nopk.c.q.desc()) + sqlalchemy.Index("noncol_idx_nopk", noncol_idx_test_nopk.c.q.desc()) sqlalchemy.Index("noncol_idx_pk", noncol_idx_test_pk.c.q.desc()) if testing.requires.view_column_reflection.enabled: From b73d7bdf00955f179969f283d531c0c1853b4ef5 Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Mon, 14 Jun 2021 17:29:35 +0530 Subject: [PATCH 08/10] fix: add primary key to fix get_unique_constraints --- test/test_suite.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_suite.py b/test/test_suite.py index fcbda8cc..a450d77d 100644 --- a/test/test_suite.py +++ b/test/test_suite.py @@ -685,6 +685,7 @@ def _test_get_unique_constraints(self, schema=None): Table( "testtbl", orig_meta, + Column("id", sqlalchemy.Integer, primary_key=True), Column("a", sqlalchemy.String(20)), Column("b", sqlalchemy.String(30)), Column("c", sqlalchemy.Integer), From 575222a4365b8a88da7fcc45dd958d31efbcfe57 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Thu, 17 Jun 2021 11:39:18 +0300 Subject: [PATCH 09/10] Revert "fix" This reverts commit 5283379df379639a80eb9003fc8e6f0645f37c0b. --- test/test_suite.py | 143 --------------------------------------------- 1 file changed, 143 deletions(-) diff --git a/test/test_suite.py b/test/test_suite.py index 18efd5f5..eb789de7 100644 --- a/test/test_suite.py +++ b/test/test_suite.py @@ -480,149 +480,6 @@ class UnicodeTextTest(UnicodeFixtureTest, _UnicodeTextTest): class ComponentReflectionTest(_ComponentReflectionTest): - @classmethod - def define_reflected_tables(cls, metadata, schema): - if schema: - schema_prefix = schema + "." - else: - schema_prefix = "" - - if testing.requires.self_referential_foreign_keys.enabled: - users = Table( - "users", - metadata, - Column("user_id", sqlalchemy.INT, primary_key=True), - Column("test1", sqlalchemy.CHAR(5), nullable=False), - Column("test2", sqlalchemy.Float(5), nullable=False), - Column( - "parent_user_id", - sqlalchemy.Integer, - sqlalchemy.ForeignKey( - "%susers.user_id" % schema_prefix, name="user_id_fk" - ), - ), - schema=schema, - test_needs_fk=True, - ) - else: - users = Table( - "users", - metadata, - Column("user_id", sqlalchemy.INT, primary_key=True), - Column("test1", sqlalchemy.CHAR(5), nullable=False), - Column("test2", sqlalchemy.Float(5), nullable=False), - schema=schema, - test_needs_fk=True, - ) - - Table( - "dingalings", - metadata, - Column("dingaling_id", sqlalchemy.Integer, primary_key=True), - Column( - "address_id", - sqlalchemy.Integer, - sqlalchemy.ForeignKey("%semail_addresses.address_id" % schema_prefix), - ), - Column("data", sqlalchemy.String(30)), - schema=schema, - test_needs_fk=True, - ) - Table( - "email_addresses", - metadata, - Column("address_id", sqlalchemy.Integer, primary_key=True), - Column( - "remote_user_id", - sqlalchemy.Integer, - sqlalchemy.ForeignKey(users.c.user_id), - ), - Column("email_address", sqlalchemy.String(20)), - sqlalchemy.PrimaryKeyConstraint("address_id", name="email_ad_pk"), - schema=schema, - test_needs_fk=True, - ) - Table( - "comment_test", - metadata, - Column("id", sqlalchemy.Integer, primary_key=True, comment="id comment"), - Column("data", sqlalchemy.String(20), comment="data % comment"), - Column( - "d2", - sqlalchemy.String(20), - comment=r"""Comment types type speedily ' " \ '' Fun!""", - ), - schema=schema, - comment=r"""the test % ' " \ table comment""", - ) - - if testing.requires.cross_schema_fk_reflection.enabled: - if schema is None: - Table( - "local_table", - metadata, - Column("id", sqlalchemy.Integer, primary_key=True), - Column("data", sqlalchemy.String(20)), - Column( - "remote_id", - ForeignKey("%s.remote_table_2.id" % testing.config.test_schema), - ), - test_needs_fk=True, - schema=config.db.dialect.default_schema_name, - ) - else: - Table( - "remote_table", - metadata, - Column("id", sqlalchemy.Integer, primary_key=True), - Column( - "local_id", - ForeignKey( - "%s.local_table.id" % config.db.dialect.default_schema_name - ), - ), - Column("data", sqlalchemy.String(20)), - schema=schema, - test_needs_fk=True, - ) - Table( - "remote_table_2", - metadata, - Column("id", sqlalchemy.Integer, primary_key=True), - Column("data", sqlalchemy.String(20)), - schema=schema, - test_needs_fk=True, - ) - - if testing.requires.index_reflection.enabled: - cls.define_index(metadata, users) - - if not schema: - # test_needs_fk is at the moment to force MySQL InnoDB - # noncol_idx_test_nopk = Table( - # "noncol_idx_test_nopk", - # metadata, - # Column("q", sqlalchemy.String(5)), - # test_needs_fk=True, - # ) - - noncol_idx_test_pk = Table( - "noncol_idx_test_pk", - metadata, - Column("id", sqlalchemy.Integer, primary_key=True), - Column("q", sqlalchemy.String(5)), - test_needs_fk=True, - ) - - if testing.requires.indexes_with_ascdesc.enabled: - # Index("noncol_idx_nopk", noncol_idx_test_nopk.c.q.desc()) - sqlalchemy.Index("noncol_idx_pk", noncol_idx_test_pk.c.q.desc()) - - if testing.requires.view_column_reflection.enabled: - cls.define_views(metadata, schema) - if not schema and testing.requires.temp_table_reflection.enabled: - cls.define_temp_tables(metadata) - @classmethod def define_temp_tables(cls, metadata): """ From 9d1d13e865fdb94e86d1d65f0944cad392126f3a Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Thu, 17 Jun 2021 11:46:16 +0300 Subject: [PATCH 10/10] Revert "Revert "fix"" This reverts commit 575222a4365b8a88da7fcc45dd958d31efbcfe57. --- test/test_suite.py | 143 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/test/test_suite.py b/test/test_suite.py index 11a4aa6f..52441290 100644 --- a/test/test_suite.py +++ b/test/test_suite.py @@ -480,6 +480,149 @@ class UnicodeTextTest(UnicodeFixtureTest, _UnicodeTextTest): class ComponentReflectionTest(_ComponentReflectionTest): + @classmethod + def define_reflected_tables(cls, metadata, schema): + if schema: + schema_prefix = schema + "." + else: + schema_prefix = "" + + if testing.requires.self_referential_foreign_keys.enabled: + users = Table( + "users", + metadata, + Column("user_id", sqlalchemy.INT, primary_key=True), + Column("test1", sqlalchemy.CHAR(5), nullable=False), + Column("test2", sqlalchemy.Float(5), nullable=False), + Column( + "parent_user_id", + sqlalchemy.Integer, + sqlalchemy.ForeignKey( + "%susers.user_id" % schema_prefix, name="user_id_fk" + ), + ), + schema=schema, + test_needs_fk=True, + ) + else: + users = Table( + "users", + metadata, + Column("user_id", sqlalchemy.INT, primary_key=True), + Column("test1", sqlalchemy.CHAR(5), nullable=False), + Column("test2", sqlalchemy.Float(5), nullable=False), + schema=schema, + test_needs_fk=True, + ) + + Table( + "dingalings", + metadata, + Column("dingaling_id", sqlalchemy.Integer, primary_key=True), + Column( + "address_id", + sqlalchemy.Integer, + sqlalchemy.ForeignKey("%semail_addresses.address_id" % schema_prefix), + ), + Column("data", sqlalchemy.String(30)), + schema=schema, + test_needs_fk=True, + ) + Table( + "email_addresses", + metadata, + Column("address_id", sqlalchemy.Integer, primary_key=True), + Column( + "remote_user_id", + sqlalchemy.Integer, + sqlalchemy.ForeignKey(users.c.user_id), + ), + Column("email_address", sqlalchemy.String(20)), + sqlalchemy.PrimaryKeyConstraint("address_id", name="email_ad_pk"), + schema=schema, + test_needs_fk=True, + ) + Table( + "comment_test", + metadata, + Column("id", sqlalchemy.Integer, primary_key=True, comment="id comment"), + Column("data", sqlalchemy.String(20), comment="data % comment"), + Column( + "d2", + sqlalchemy.String(20), + comment=r"""Comment types type speedily ' " \ '' Fun!""", + ), + schema=schema, + comment=r"""the test % ' " \ table comment""", + ) + + if testing.requires.cross_schema_fk_reflection.enabled: + if schema is None: + Table( + "local_table", + metadata, + Column("id", sqlalchemy.Integer, primary_key=True), + Column("data", sqlalchemy.String(20)), + Column( + "remote_id", + ForeignKey("%s.remote_table_2.id" % testing.config.test_schema), + ), + test_needs_fk=True, + schema=config.db.dialect.default_schema_name, + ) + else: + Table( + "remote_table", + metadata, + Column("id", sqlalchemy.Integer, primary_key=True), + Column( + "local_id", + ForeignKey( + "%s.local_table.id" % config.db.dialect.default_schema_name + ), + ), + Column("data", sqlalchemy.String(20)), + schema=schema, + test_needs_fk=True, + ) + Table( + "remote_table_2", + metadata, + Column("id", sqlalchemy.Integer, primary_key=True), + Column("data", sqlalchemy.String(20)), + schema=schema, + test_needs_fk=True, + ) + + if testing.requires.index_reflection.enabled: + cls.define_index(metadata, users) + + if not schema: + # test_needs_fk is at the moment to force MySQL InnoDB + # noncol_idx_test_nopk = Table( + # "noncol_idx_test_nopk", + # metadata, + # Column("q", sqlalchemy.String(5)), + # test_needs_fk=True, + # ) + + noncol_idx_test_pk = Table( + "noncol_idx_test_pk", + metadata, + Column("id", sqlalchemy.Integer, primary_key=True), + Column("q", sqlalchemy.String(5)), + test_needs_fk=True, + ) + + if testing.requires.indexes_with_ascdesc.enabled: + # Index("noncol_idx_nopk", noncol_idx_test_nopk.c.q.desc()) + sqlalchemy.Index("noncol_idx_pk", noncol_idx_test_pk.c.q.desc()) + + if testing.requires.view_column_reflection.enabled: + cls.define_views(metadata, schema) + if not schema and testing.requires.temp_table_reflection.enabled: + cls.define_temp_tables(metadata) + @classmethod def define_temp_tables(cls, metadata): """