Skip to content

Commit f32d32d

Browse files
authored
YQB FQ: integration tests for MySQL (#6573)
1 parent 1e656fe commit f32d32d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+1587
-168
lines changed

ydb/library/yql/providers/generic/connector/tests/common_test_cases/base.py

Lines changed: 43 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -19,36 +19,53 @@ class BaseTestCase:
1919

2020
@property
2121
def name(self) -> str:
22-
return f'{self.name_}_{EProtocol.Name(self.protocol)}'
22+
match self.data_source_kind:
23+
case EDataSourceKind.CLICKHOUSE:
24+
# ClickHouse has two kinds of network protocols: NATIVE and HTTP,
25+
# so we append protocol name to the test case name
26+
return f'{self.name_}_{EProtocol.Name(self.protocol)}'
27+
case EDataSourceKind.MYSQL:
28+
return self.name_
29+
case EDataSourceKind.POSTGRESQL:
30+
return self.name_
31+
case EDataSourceKind.YDB:
32+
return self.name_
33+
case _:
34+
raise Exception(f'invalid data source: {self.data_source_kind}')
2335

2436
@property
2537
def database(self) -> Database:
2638
'''
27-
We want to create a distinct database on every test case
39+
For PG/CH we create a distinct database on every test case.
40+
For YDB/MySQL we use single predefined database.
2841
'''
29-
return Database(self.name, self.data_source_kind)
42+
match self.data_source_kind:
43+
case EDataSourceKind.CLICKHOUSE:
44+
return Database(self.name, self.data_source_kind)
45+
case EDataSourceKind.MYSQL:
46+
return Database("db", self.data_source_kind)
47+
case EDataSourceKind.POSTGRESQL:
48+
return Database(self.name, self.data_source_kind)
49+
case EDataSourceKind.YDB:
50+
return Database("local", self.data_source_kind)
3051

3152
@functools.cached_property
32-
def _table_name(self) -> str:
53+
def table_name(self) -> str:
3354
'''
34-
In general, we cannot use test case name as table name because of special symbols,
35-
so we provide a random table name instead.
55+
For some kinds of RDBMS we cannot use test case name as table name because of special symbols,
56+
so we provide a random table name instead where necessary.
3657
'''
3758
match self.data_source_kind:
38-
case EDataSourceKind.POSTGRESQL:
39-
return 't' + hashlib.sha256(str(random.randint(0, 65536)).encode('ascii')).hexdigest()[:8]
4059
case EDataSourceKind.CLICKHOUSE:
41-
return 't' + hashlib.sha256(str(random.randint(0, 65536)).encode('ascii')).hexdigest()[:8]
60+
return 't' + make_random_string(8)
61+
case EDataSourceKind.MYSQL:
62+
return self.name
63+
case EDataSourceKind.POSTGRESQL:
64+
return 't' + make_random_string(8)
4265
case EDataSourceKind.YDB:
4366
return self.name
44-
45-
@property
46-
def sql_table_name(self) -> str:
47-
return self._table_name
48-
49-
@property
50-
def qualified_table_name(self) -> str:
51-
return self._table_name
67+
case _:
68+
raise Exception(f'invalid data source: {self.data_source_kind}')
5269

5370
@property
5471
def pragmas_sql_string(self) -> str:
@@ -66,13 +83,15 @@ def generic_settings(self) -> GenericSettings:
6683
clickhouse_clusters=[
6784
GenericSettings.ClickHouseCluster(database=self.database.name, protocol=EProtocol.NATIVE)
6885
],
69-
postgresql_clusters=[],
7086
)
71-
87+
case EDataSourceKind.MYSQL:
88+
return GenericSettings(
89+
date_time_format=EDateTimeFormat.YQL_FORMAT,
90+
mysql_clusters=[GenericSettings.MySQLCluster(database=self.database.name)],
91+
)
7292
case EDataSourceKind.POSTGRESQL:
7393
return GenericSettings(
7494
date_time_format=EDateTimeFormat.YQL_FORMAT,
75-
clickhouse_clusters=[],
7695
postgresql_clusters=[GenericSettings.PostgreSQLCluster(database=self.database.name, schema=None)],
7796
)
7897
case EDataSourceKind.YDB:
@@ -82,3 +101,7 @@ def generic_settings(self) -> GenericSettings:
82101
)
83102
case _:
84103
raise Exception(f'invalid data source: {self.data_source_kind}')
104+
105+
106+
def make_random_string(length: int) -> str:
107+
return hashlib.sha256(str(random.randint(0, 65536)).encode('ascii')).hexdigest()[:length]
Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,32 @@
11
from typing import List
2+
from dataclasses import dataclass
23

34
from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol
45
from ydb.library.yql.providers.generic.connector.tests.common_test_cases.base import BaseTestCase
6+
from ydb.library.yql.providers.generic.connector.tests.utils.settings import GenericSettings
57

68

7-
TestCase = BaseTestCase
9+
@dataclass
10+
class TestCase(BaseTestCase):
11+
@property
12+
def generic_settings(self) -> GenericSettings:
13+
gs = super().generic_settings
814

15+
# Overload setting for MySQL database
16+
if self.data_source_kind == EDataSourceKind.MYSQL:
17+
for cluster in gs.mysql_clusters:
18+
cluster.database = "missing_database"
919

10-
class Factory:
11-
def make_test_cases(self, data_source_kind: EDataSourceKind) -> List[TestCase]:
12-
test_cases = []
13-
14-
test_case_name = 'missing_database'
20+
return gs
1521

16-
test_case = TestCase(
17-
name_=test_case_name,
18-
data_source_kind=data_source_kind,
19-
protocol=EProtocol.NATIVE,
20-
pragmas=dict(),
21-
)
2222

23-
test_cases.append(test_case)
24-
25-
return test_cases
23+
class Factory:
24+
def make_test_cases(self, data_source_kind: EDataSourceKind) -> List[TestCase]:
25+
return [
26+
TestCase(
27+
name_="missing_database",
28+
data_source_kind=data_source_kind,
29+
protocol=EProtocol.NATIVE,
30+
pragmas=dict(),
31+
)
32+
]

ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_positive_common.py

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
99
from ydb.library.yql.providers.generic.connector.tests.utils.generate import generate_table_data
1010
import ydb.library.yql.providers.generic.connector.tests.utils.types.clickhouse as clickhouse
11+
import ydb.library.yql.providers.generic.connector.tests.utils.types.mysql as mysql
1112
import ydb.library.yql.providers.generic.connector.tests.utils.types.postgresql as postgresql
1213
import ydb.library.yql.providers.generic.connector.tests.utils.types.ydb as Ydb
1314
from ydb.library.yql.providers.generic.connector.tests.utils.schema import (
@@ -69,12 +70,16 @@ def _column_selection(self) -> Sequence[TestCase]:
6970
Column(
7071
name='COL1',
7172
ydb_type=Type.INT32,
72-
data_source_type=DataSourceType(ch=clickhouse.Int32(), pg=postgresql.Int4(), ydb=Ydb.Int32()),
73+
data_source_type=DataSourceType(
74+
ch=clickhouse.Int32(), pg=postgresql.Int4(), ydb=Ydb.Int32(), my=mysql.Integer()
75+
),
7376
),
7477
Column(
7578
name='col2',
7679
ydb_type=Type.INT32,
77-
data_source_type=DataSourceType(ch=clickhouse.Int32(), pg=postgresql.Int4(), ydb=Ydb.Int32()),
80+
data_source_type=DataSourceType(
81+
ch=clickhouse.Int32(), pg=postgresql.Int4(), ydb=Ydb.Int32(), my=mysql.Integer()
82+
),
7883
),
7984
)
8085
)
@@ -92,6 +97,7 @@ def _column_selection(self) -> Sequence[TestCase]:
9297
EDataSourceKind.CLICKHOUSE,
9398
EDataSourceKind.POSTGRESQL,
9499
EDataSourceKind.YDB,
100+
EDataSourceKind.MYSQL,
95101
),
96102
),
97103
# SELECT COL1 FROM table
@@ -105,6 +111,7 @@ def _column_selection(self) -> Sequence[TestCase]:
105111
EDataSourceKind.CLICKHOUSE,
106112
# NOTE: YQ-2264: doesn't work for PostgreSQL because of implicit cast to lowercase (COL1 -> col1)
107113
EDataSourceKind.YDB,
114+
EDataSourceKind.MYSQL,
108115
),
109116
),
110117
# SELECT col1 FROM table
@@ -127,6 +134,7 @@ def _column_selection(self) -> Sequence[TestCase]:
127134
EDataSourceKind.CLICKHOUSE,
128135
EDataSourceKind.POSTGRESQL,
129136
EDataSourceKind.YDB,
137+
EDataSourceKind.MYSQL,
130138
),
131139
),
132140
# SELECT col2, COL1 FROM table
@@ -140,6 +148,7 @@ def _column_selection(self) -> Sequence[TestCase]:
140148
EDataSourceKind.CLICKHOUSE,
141149
# NOTE: YQ-2264: doesn't work for PostgreSQL because of implicit cast to lowercase (COL1 -> col1)
142150
EDataSourceKind.YDB,
151+
EDataSourceKind.MYSQL,
143152
),
144153
),
145154
# SELECT col2, col1 FROM table
@@ -163,6 +172,7 @@ def _column_selection(self) -> Sequence[TestCase]:
163172
EDataSourceKind.CLICKHOUSE,
164173
# NOTE: YQ-2264: doesn't work for PostgreSQL because of implicit cast to lowercase (COL1 -> col1)
165174
EDataSourceKind.YDB,
175+
EDataSourceKind.MYSQL,
166176
),
167177
),
168178
# Select the same column multiple times with different aliases
@@ -183,6 +193,7 @@ def _column_selection(self) -> Sequence[TestCase]:
183193
EDataSourceKind.CLICKHOUSE,
184194
EDataSourceKind.POSTGRESQL,
185195
EDataSourceKind.YDB,
196+
EDataSourceKind.MYSQL,
186197
),
187198
),
188199
)
@@ -279,19 +290,22 @@ def make_test_cases(self, data_source_kind: EDataSourceKind) -> Sequence[TestCas
279290
EDataSourceKind.CLICKHOUSE: [EProtocol.NATIVE, EProtocol.HTTP],
280291
EDataSourceKind.POSTGRESQL: [EProtocol.NATIVE],
281292
EDataSourceKind.YDB: [EProtocol.NATIVE],
293+
EDataSourceKind.MYSQL: [EProtocol.NATIVE],
282294
}
283295

284296
base_test_cases = None
285297

286-
if data_source_kind == EDataSourceKind.YDB:
298+
if data_source_kind in [EDataSourceKind.YDB, EDataSourceKind.MYSQL]:
287299
base_test_cases = self._column_selection()
288-
else:
300+
elif data_source_kind in [EDataSourceKind.CLICKHOUSE, EDataSourceKind.POSTGRESQL]:
289301
base_test_cases = list(
290302
itertools.chain(
291303
self._column_selection(),
292304
self._large_table(),
293305
)
294306
)
307+
else:
308+
raise f'Unexpected data source kind: {data_source_kind}'
295309

296310
test_cases = []
297311
for base_tc in base_test_cases:

ydb/library/yql/providers/generic/connector/tests/common_test_cases/ya.make

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,6 @@ IF (AUTOCHECK)
55
NO_LINT()
66
ENDIF()
77

8-
IF (OPENSOURCE)
9-
# YQ-3351: enabling python style checks only for opensource
10-
STYLE_PYTHON()
11-
ENDIF()
12-
138
PY_SRCS(
149
base.py
1510
select_missing_database.py

ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ services:
1212
- 8123
1313
fq-connector-go:
1414
container_name: fq-tests-ch-fq-connector-go
15-
image: ghcr.io/ydb-platform/fq-connector-go:v0.4.9@sha256:3a1fe086be50c0edbae2c2b284aee5ce76bd056d7f46cb460919ec37a6f8ab5c
15+
image: ghcr.io/ydb-platform/fq-connector-go:v0.4.17@sha256:3763344ab70f9a6b8c1546f15c0b31465590e8ac6636be15ca2d29c4f4cd9b19
1616
ports:
1717
- 2130
1818
volumes:

ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/test.py

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,19 +42,16 @@ def test_select_positive(
4242
"test_case", tc_collection.get('select_missing_database'), ids=tc_collection.ids('select_missing_database')
4343
)
4444
@pytest.mark.usefixtures("settings")
45-
@pytest.mark.usefixtures("clickhouse_client")
4645
def test_select_missing_database(
4746
request: pytest.FixtureRequest,
4847
settings: Settings,
4948
runner_type: str,
50-
clickhouse_client: Client,
5149
test_case: select_missing_database.TestCase,
5250
):
5351
runner = configure_runner(runner_type=runner_type, settings=settings)
54-
scenario.select_missing_table(
52+
scenario.select_missing_database(
5553
settings=settings,
5654
runner=runner,
57-
client=clickhouse_client,
5855
test_case=test_case,
5956
test_name=request.node.name,
6057
)

ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/ya.make

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,6 @@ ENDIF()
3737
INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc)
3838

3939
IF (OPENSOURCE)
40-
# YQ-3351: enabling python style checks only for opensource
41-
STYLE_PYTHON()
4240
# Including of docker_compose/recipe.inc automatically converts these tests into LARGE,
4341
# which makes it impossible to run them during precommit checks on Github CI.
4442
# Next several lines forces these tests to be MEDIUM. To see discussion, visit YDBOPS-8928.
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
from typing import Sequence, Mapping
2+
3+
from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind
4+
import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_database as select_missing_database
5+
import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_table as select_missing_table
6+
import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common as select_positive_common
7+
import select_datetime
8+
import select_positive
9+
10+
# import select_positive_with_schema
11+
12+
from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
13+
14+
15+
class Collection(object):
16+
_test_cases: Mapping[str, Sequence]
17+
18+
def __init__(self, ss: Settings):
19+
self._test_cases = {
20+
'select_missing_database': select_missing_database.Factory().make_test_cases(EDataSourceKind.MYSQL),
21+
'select_missing_table': select_missing_table.Factory().make_test_cases(EDataSourceKind.MYSQL),
22+
'select_positive': select_positive.Factory().make_test_cases()
23+
+ select_positive_common.Factory(ss).make_test_cases(EDataSourceKind.MYSQL),
24+
'select_datetime': select_datetime.Factory().make_test_cases(),
25+
}
26+
27+
def get(self, key: str) -> Sequence:
28+
if key not in self._test_cases:
29+
raise ValueError(f'no such test: {key}')
30+
31+
return self._test_cases[key]
32+
33+
def ids(self, key: str) -> Sequence[str]:
34+
if key not in self._test_cases:
35+
raise ValueError(f'no such test: {key}')
36+
37+
return [tc.name for tc in self._test_cases[key]]
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
from typing import Final
2+
import pathlib
3+
4+
import pytest
5+
6+
7+
from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind
8+
from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
9+
10+
11+
docker_compose_dir: Final = pathlib.Path("ydb/library/yql/providers/generic/connector/tests/datasource/mysql")
12+
13+
14+
@pytest.fixture
15+
def settings() -> Settings:
16+
return Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EDataSourceKind.MYSQL])
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[mysqld]
2+
character-set-server = utf8
3+
collation-server = utf8_unicode_ci
4+
skip-character-set-client-handshake
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
services:
2+
fq-connector-go:
3+
container_name: fq-tests-my-fq-connector-go
4+
image: ghcr.io/ydb-platform/fq-connector-go:v0.4.17@sha256:3763344ab70f9a6b8c1546f15c0b31465590e8ac6636be15ca2d29c4f4cd9b19
5+
ports:
6+
- 2130
7+
volumes:
8+
- ../../fq-connector-go/:/opt/ydb/cfg/
9+
mysql:
10+
container_name: fq-tests-my-mysql
11+
environment:
12+
MYSQL_DATABASE: db
13+
MYSQL_ROOT_PASSWORD: password
14+
image: mirror.gcr.io/library/mysql@sha256:1579fe3a97a436cc10824fc771a07fcedc92213e7ab7604eb5d2976ca419abc8
15+
ports:
16+
- 3306
17+
volumes:
18+
- ./init:/docker-entrypoint-initdb.d
19+
- ./custom.cnf:/etc/mysql/conf.d/custom.cnf
20+
version: "3.4"

0 commit comments

Comments
 (0)