YQB FQ: integration tests for MySQL (#6573)

Author: vitalyisaev2

ydb/library/yql/providers/generic/connector/tests/common_test_cases/base.py

@@ -19,36 +19,53 @@ class BaseTestCase:
 
     @property
     def name(self) -> str:
-        return f'{self.name_}_{EProtocol.Name(self.protocol)}'
+        match self.data_source_kind:
+            case EDataSourceKind.CLICKHOUSE:
+                # ClickHouse has two kinds of network protocols: NATIVE and HTTP,
+                # so we append protocol name to the test case name
+                return f'{self.name_}_{EProtocol.Name(self.protocol)}'
+            case EDataSourceKind.MYSQL:
+                return self.name_
+            case EDataSourceKind.POSTGRESQL:
+                return self.name_
+            case EDataSourceKind.YDB:
+                return self.name_
+            case _:
+                raise Exception(f'invalid data source: {self.data_source_kind}')
 
     @property
     def database(self) -> Database:
         '''
-        We want to create a distinct database on every test case
+        For PG/CH we create a distinct database on every test case.
+        For YDB/MySQL we use single predefined database.
         '''
-        return Database(self.name, self.data_source_kind)
+        match self.data_source_kind:
+            case EDataSourceKind.CLICKHOUSE:
+                return Database(self.name, self.data_source_kind)
+            case EDataSourceKind.MYSQL:
+                return Database("db", self.data_source_kind)
+            case EDataSourceKind.POSTGRESQL:
+                return Database(self.name, self.data_source_kind)
+            case EDataSourceKind.YDB:
+                return Database("local", self.data_source_kind)
 
     @functools.cached_property
-    def _table_name(self) -> str:
+    def table_name(self) -> str:
         '''
-        In general, we cannot use test case name as table name because of special symbols,
-        so we provide a random table name instead.
+        For some kinds of RDBMS we cannot use test case name as table name because of special symbols,
+        so we provide a random table name instead where necessary.
         '''
         match self.data_source_kind:
-            case EDataSourceKind.POSTGRESQL:
-                return 't' + hashlib.sha256(str(random.randint(0, 65536)).encode('ascii')).hexdigest()[:8]
             case EDataSourceKind.CLICKHOUSE:
-                return 't' + hashlib.sha256(str(random.randint(0, 65536)).encode('ascii')).hexdigest()[:8]
+                return 't' + make_random_string(8)
+            case EDataSourceKind.MYSQL:
+                return self.name
+            case EDataSourceKind.POSTGRESQL:
+                return 't' + make_random_string(8)
             case EDataSourceKind.YDB:
                 return self.name
-
-    @property
-    def sql_table_name(self) -> str:
-        return self._table_name
-
-    @property
-    def qualified_table_name(self) -> str:
-        return self._table_name
+            case _:
+                raise Exception(f'invalid data source: {self.data_source_kind}')
 
     @property
     def pragmas_sql_string(self) -> str:
@@ -66,13 +83,15 @@ def generic_settings(self) -> GenericSettings:
                     clickhouse_clusters=[
                         GenericSettings.ClickHouseCluster(database=self.database.name, protocol=EProtocol.NATIVE)
                     ],
-                    postgresql_clusters=[],
                 )
-
+            case EDataSourceKind.MYSQL:
+                return GenericSettings(
+                    date_time_format=EDateTimeFormat.YQL_FORMAT,
+                    mysql_clusters=[GenericSettings.MySQLCluster(database=self.database.name)],
+                )
             case EDataSourceKind.POSTGRESQL:
                 return GenericSettings(
                     date_time_format=EDateTimeFormat.YQL_FORMAT,
-                    clickhouse_clusters=[],
                     postgresql_clusters=[GenericSettings.PostgreSQLCluster(database=self.database.name, schema=None)],
                 )
             case EDataSourceKind.YDB:
@@ -82,3 +101,7 @@ def generic_settings(self) -> GenericSettings:
                 )
             case _:
                 raise Exception(f'invalid data source: {self.data_source_kind}')
+
+
+def make_random_string(length: int) -> str:
+    return hashlib.sha256(str(random.randint(0, 65536)).encode('ascii')).hexdigest()[:length]

ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_missing_database.py

@@ -1,25 +1,32 @@
 from typing import List
+from dataclasses import dataclass
 
 from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol
 from ydb.library.yql.providers.generic.connector.tests.common_test_cases.base import BaseTestCase
+from ydb.library.yql.providers.generic.connector.tests.utils.settings import GenericSettings
 
 
-TestCase = BaseTestCase
+@dataclass
+class TestCase(BaseTestCase):
+    @property
+    def generic_settings(self) -> GenericSettings:
+        gs = super().generic_settings
 
+        # Overload setting for MySQL database
+        if self.data_source_kind == EDataSourceKind.MYSQL:
+            for cluster in gs.mysql_clusters:
+                cluster.database = "missing_database"
 
-class Factory:
-    def make_test_cases(self, data_source_kind: EDataSourceKind) -> List[TestCase]:
-        test_cases = []
-
-        test_case_name = 'missing_database'
+        return gs
 
-        test_case = TestCase(
-            name_=test_case_name,
-            data_source_kind=data_source_kind,
-            protocol=EProtocol.NATIVE,
-            pragmas=dict(),
-        )
 
-        test_cases.append(test_case)
-
-        return test_cases
+class Factory:
+    def make_test_cases(self, data_source_kind: EDataSourceKind) -> List[TestCase]:
+        return [
+            TestCase(
+                name_="missing_database",
+                data_source_kind=data_source_kind,
+                protocol=EProtocol.NATIVE,
+                pragmas=dict(),
+            )
+        ]

ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_positive_common.py

@@ -8,6 +8,7 @@
 from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
 from ydb.library.yql.providers.generic.connector.tests.utils.generate import generate_table_data
 import ydb.library.yql.providers.generic.connector.tests.utils.types.clickhouse as clickhouse
+import ydb.library.yql.providers.generic.connector.tests.utils.types.mysql as mysql
 import ydb.library.yql.providers.generic.connector.tests.utils.types.postgresql as postgresql
 import ydb.library.yql.providers.generic.connector.tests.utils.types.ydb as Ydb
 from ydb.library.yql.providers.generic.connector.tests.utils.schema import (
@@ -69,12 +70,16 @@ def _column_selection(self) -> Sequence[TestCase]:
                 Column(
                     name='COL1',
                     ydb_type=Type.INT32,
-                    data_source_type=DataSourceType(ch=clickhouse.Int32(), pg=postgresql.Int4(), ydb=Ydb.Int32()),
+                    data_source_type=DataSourceType(
+                        ch=clickhouse.Int32(), pg=postgresql.Int4(), ydb=Ydb.Int32(), my=mysql.Integer()
+                    ),
                 ),
                 Column(
                     name='col2',
                     ydb_type=Type.INT32,
-                    data_source_type=DataSourceType(ch=clickhouse.Int32(), pg=postgresql.Int4(), ydb=Ydb.Int32()),
+                    data_source_type=DataSourceType(
+                        ch=clickhouse.Int32(), pg=postgresql.Int4(), ydb=Ydb.Int32(), my=mysql.Integer()
+                    ),
                 ),
             )
         )
@@ -92,6 +97,7 @@ def _column_selection(self) -> Sequence[TestCase]:
                     EDataSourceKind.CLICKHOUSE,
                     EDataSourceKind.POSTGRESQL,
                     EDataSourceKind.YDB,
+                    EDataSourceKind.MYSQL,
                 ),
             ),
             # SELECT COL1 FROM table
@@ -105,6 +111,7 @@ def _column_selection(self) -> Sequence[TestCase]:
                     EDataSourceKind.CLICKHOUSE,
                     # NOTE: YQ-2264: doesn't work for PostgreSQL because of implicit cast to lowercase (COL1 -> col1)
                     EDataSourceKind.YDB,
+                    EDataSourceKind.MYSQL,
                 ),
             ),
             # SELECT col1 FROM table
@@ -127,6 +134,7 @@ def _column_selection(self) -> Sequence[TestCase]:
                     EDataSourceKind.CLICKHOUSE,
                     EDataSourceKind.POSTGRESQL,
                     EDataSourceKind.YDB,
+                    EDataSourceKind.MYSQL,
                 ),
             ),
             # SELECT col2, COL1 FROM table
@@ -140,6 +148,7 @@ def _column_selection(self) -> Sequence[TestCase]:
                     EDataSourceKind.CLICKHOUSE,
                     # NOTE: YQ-2264: doesn't work for PostgreSQL because of implicit cast to lowercase (COL1 -> col1)
                     EDataSourceKind.YDB,
+                    EDataSourceKind.MYSQL,
                 ),
             ),
             # SELECT col2, col1 FROM table
@@ -163,6 +172,7 @@ def _column_selection(self) -> Sequence[TestCase]:
                     EDataSourceKind.CLICKHOUSE,
                     # NOTE: YQ-2264: doesn't work for PostgreSQL because of implicit cast to lowercase (COL1 -> col1)
                     EDataSourceKind.YDB,
+                    EDataSourceKind.MYSQL,
                 ),
             ),
             # Select the same column multiple times with different aliases
@@ -183,6 +193,7 @@ def _column_selection(self) -> Sequence[TestCase]:
                     EDataSourceKind.CLICKHOUSE,
                     EDataSourceKind.POSTGRESQL,
                     EDataSourceKind.YDB,
+                    EDataSourceKind.MYSQL,
                 ),
             ),
         )
@@ -279,19 +290,22 @@ def make_test_cases(self, data_source_kind: EDataSourceKind) -> Sequence[TestCas
             EDataSourceKind.CLICKHOUSE: [EProtocol.NATIVE, EProtocol.HTTP],
             EDataSourceKind.POSTGRESQL: [EProtocol.NATIVE],
             EDataSourceKind.YDB: [EProtocol.NATIVE],
+            EDataSourceKind.MYSQL: [EProtocol.NATIVE],
         }
 
         base_test_cases = None
 
-        if data_source_kind == EDataSourceKind.YDB:
+        if data_source_kind in [EDataSourceKind.YDB, EDataSourceKind.MYSQL]:
             base_test_cases = self._column_selection()
-        else:
+        elif data_source_kind in [EDataSourceKind.CLICKHOUSE, EDataSourceKind.POSTGRESQL]:
             base_test_cases = list(
                 itertools.chain(
                     self._column_selection(),
                     self._large_table(),
                 )
             )
+        else:
+            raise f'Unexpected data source kind: {data_source_kind}'
 
         test_cases = []
         for base_tc in base_test_cases:

ydb/library/yql/providers/generic/connector/tests/common_test_cases/ya.make

@@ -5,11 +5,6 @@ IF (AUTOCHECK)
     NO_LINT()
 ENDIF()
 
-IF (OPENSOURCE)
-    # YQ-3351: enabling python style checks only for opensource
-    STYLE_PYTHON()
-ENDIF()
-
 PY_SRCS(
     base.py
     select_missing_database.py

ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml

@@ -12,7 +12,7 @@ services:
     - 8123
   fq-connector-go:
     container_name: fq-tests-ch-fq-connector-go
-    image: ghcr.io/ydb-platform/fq-connector-go:v0.4.9@sha256:3a1fe086be50c0edbae2c2b284aee5ce76bd056d7f46cb460919ec37a6f8ab5c
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.4.17@sha256:3763344ab70f9a6b8c1546f15c0b31465590e8ac6636be15ca2d29c4f4cd9b19
     ports:
     - 2130
     volumes:

ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/test.py

@@ -42,19 +42,16 @@ def test_select_positive(
     "test_case", tc_collection.get('select_missing_database'), ids=tc_collection.ids('select_missing_database')
 )
 @pytest.mark.usefixtures("settings")
-@pytest.mark.usefixtures("clickhouse_client")
 def test_select_missing_database(
     request: pytest.FixtureRequest,
     settings: Settings,
     runner_type: str,
-    clickhouse_client: Client,
     test_case: select_missing_database.TestCase,
 ):
     runner = configure_runner(runner_type=runner_type, settings=settings)
-    scenario.select_missing_table(
+    scenario.select_missing_database(
         settings=settings,
         runner=runner,
-        client=clickhouse_client,
         test_case=test_case,
         test_name=request.node.name,
     )

ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/ya.make

@@ -37,8 +37,6 @@ ENDIF()
 INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc)
 
 IF (OPENSOURCE)
-    # YQ-3351: enabling python style checks only for opensource
-    STYLE_PYTHON()
     # Including of docker_compose/recipe.inc automatically converts these tests into LARGE, 
     # which makes it impossible to run them during precommit checks on Github CI. 
     # Next several lines forces these tests to be MEDIUM. To see discussion, visit YDBOPS-8928.

ydb/library/yql/providers/generic/connector/tests/datasource/mysql/collection.py

@@ -0,0 +1,37 @@
+from typing import Sequence, Mapping
+
+from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_database as select_missing_database
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_table as select_missing_table
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common as select_positive_common
+import select_datetime
+import select_positive
+
+# import select_positive_with_schema
+
+from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
+
+
+class Collection(object):
+    _test_cases: Mapping[str, Sequence]
+
+    def __init__(self, ss: Settings):
+        self._test_cases = {
+            'select_missing_database': select_missing_database.Factory().make_test_cases(EDataSourceKind.MYSQL),
+            'select_missing_table': select_missing_table.Factory().make_test_cases(EDataSourceKind.MYSQL),
+            'select_positive': select_positive.Factory().make_test_cases()
+            + select_positive_common.Factory(ss).make_test_cases(EDataSourceKind.MYSQL),
+            'select_datetime': select_datetime.Factory().make_test_cases(),
+        }
+
+    def get(self, key: str) -> Sequence:
+        if key not in self._test_cases:
+            raise ValueError(f'no such test: {key}')
+
+        return self._test_cases[key]
+
+    def ids(self, key: str) -> Sequence[str]:
+        if key not in self._test_cases:
+            raise ValueError(f'no such test: {key}')
+
+        return [tc.name for tc in self._test_cases[key]]

ydb/library/yql/providers/generic/connector/tests/datasource/mysql/conftest.py

@@ -0,0 +1,16 @@
+from typing import Final
+import pathlib
+
+import pytest
+
+
+from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind
+from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
+
+
+docker_compose_dir: Final = pathlib.Path("ydb/library/yql/providers/generic/connector/tests/datasource/mysql")
+
+
+@pytest.fixture
+def settings() -> Settings:
+    return Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EDataSourceKind.MYSQL])

ydb/library/yql/providers/generic/connector/tests/datasource/mysql/custom.cnf

@@ -0,0 +1,4 @@
+[mysqld]
+character-set-server = utf8
+collation-server = utf8_unicode_ci
+skip-character-set-client-handshake

ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml

@@ -0,0 +1,20 @@
+services:
+  fq-connector-go:
+    container_name: fq-tests-my-fq-connector-go
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.4.17@sha256:3763344ab70f9a6b8c1546f15c0b31465590e8ac6636be15ca2d29c4f4cd9b19
+    ports:
+    - 2130
+    volumes:
+    - ../../fq-connector-go/:/opt/ydb/cfg/
+  mysql:
+    container_name: fq-tests-my-mysql
+    environment:
+      MYSQL_DATABASE: db
+      MYSQL_ROOT_PASSWORD: password
+    image: mirror.gcr.io/library/mysql@sha256:1579fe3a97a436cc10824fc771a07fcedc92213e7ab7604eb5d2976ca419abc8
+    ports:
+    - 3306
+    volumes:
+    - ./init:/docker-entrypoint-initdb.d
+    - ./custom.cnf:/etc/mysql/conf.d/custom.cnf
+version: "3.4"