Skip to content

Commit c58de52

Browse files
pashandor789pashandor789
authored
[KQP] Use query service in join order tests instead of scan service (#12033)
1 parent fbf506c commit c58de52

6 files changed

+133
-51
lines changed

‎ydb/core/kqp/ut/join/data/join_order/tpcds78_1000s.json

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,14 @@
1919
"table":"test\/ds\/store_sales"
2020
},
2121
{
22-
"op_name":"TableFullScan",
23-
"table":"test\/ds\/date_dim"
22+
"op_name":"Union",
23+
"args":
24+
[
25+
{
26+
"op_name":"TableFullScan",
27+
"table":"test\/ds\/date_dim"
28+
}
29+
]
2430
}
2531
]
2632
},
@@ -49,8 +55,14 @@
4955
]
5056
},
5157
{
52-
"op_name":"TableFullScan",
53-
"table":"test\/ds\/date_dim"
58+
"op_name":"Union",
59+
"args":
60+
[
61+
{
62+
"op_name":"TableFullScan",
63+
"table":"test\/ds\/date_dim"
64+
}
65+
]
5466
}
5567
]
5668
}
@@ -75,8 +87,14 @@
7587
]
7688
},
7789
{
78-
"op_name":"TableFullScan",
79-
"table":"test\/ds\/date_dim"
90+
"op_name":"Union",
91+
"args":
92+
[
93+
{
94+
"op_name":"TableFullScan",
95+
"table":"test\/ds\/date_dim"
96+
}
97+
]
8098
}
8199
]
82100
}

‎ydb/core/kqp/ut/join/data/queries/join_order_hints_complex.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ PRAGMA ydb.OptimizerHints =
1212
Rows(T U # 10e8)
1313
Rows(V # 1)
1414
JoinOrder( (R S) (T U) )
15+
JoinType(T U Broadcast)
1516
';
1617

1718
SELECT * FROM

‎ydb/core/kqp/ut/join/data/queries/join_order_hints_many_hint_trees.sql

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ PRAGMA ydb.OptimizerHints =
1010
Rows(V # 1)
1111
JoinOrder(T U)
1212
JoinOrder(R S)
13+
JoinType(R S Broadcast)
14+
JoinType(T U Broadcast)
1315
';
1416

1517
SELECT * FROM

‎ydb/core/kqp/ut/join/data/queries/tpch_random_join_view_just_works.sql

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
select v.o_orderkey from
2+
`/Root/tpch3_as_view` v
3+
join
4+
`/Root/orders` as o
5+
on
6+
v.o_orderkey = o.o_custkey;

‎ydb/core/kqp/ut/join/data/view/tpch_random_join_view.sql

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
CREATE VIEW `/Root/tpch3_as_view` WITH (security_invoker = TRUE) AS
2+
select
3+
c.c_mktsegment as c_mktsegment,
4+
o.o_orderdate as o_orderdate,
5+
o.o_shippriority as o_shippriority,
6+
o.o_orderkey as o_orderkey
7+
from
8+
`/Root/customer` as c
9+
join
10+
`/Root/orders` as o
11+
on
12+
c.c_custkey = o.o_custkey

‎ydb/core/kqp/ut/join/kqp_join_order_ut.cpp

Lines changed: 88 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -43,15 +43,22 @@ TString GetStatic(const TString& filePath) {
4343
return buffer.str();
4444
}
4545

46-
void CreateTables(TSession session, const TString& schemaPath, bool useColumnStore) {
46+
void CreateTables(NYdb::NQuery::TSession session, const TString& schemaPath, bool useColumnStore) {
4747
std::string query = GetStatic(schemaPath);
4848

4949
if (useColumnStore) {
5050
std::regex pattern(R"(CREATE TABLE [^\(]+ \([^;]*\))", std::regex::multiline);
5151
query = std::regex_replace(query, pattern, "$& WITH (STORE = COLUMN, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 16);");
5252
}
5353

54-
auto res = session.ExecuteSchemeQuery(TString(query)).GetValueSync();
54+
auto res = session.ExecuteQuery(TString(query), NYdb::NQuery::TTxControl::NoTx()).GetValueSync();
55+
res.GetIssues().PrintTo(Cerr);
56+
UNIT_ASSERT(res.IsSuccess());
57+
}
58+
59+
void CreateView(NYdb::NQuery::TSession session, const TString& viewPath) {
60+
std::string query = GetStatic(viewPath);
61+
auto res = session.ExecuteQuery(TString(query), NYdb::NQuery::TTxControl::NoTx()).GetValueSync();
5562
res.GetIssues().PrintTo(Cerr);
5663
UNIT_ASSERT(res.IsSuccess());
5764
}
@@ -69,7 +76,7 @@ TString GetPrettyJSON(const NJson::TJsonValue& json) {
6976
* A basic join order test. We define 5 tables sharing the same
7077
* key attribute and construct various full clique join queries
7178
*/
72-
static void CreateSampleTable(TSession session, bool useColumnStore) {
79+
static void CreateSampleTable(NYdb::NQuery::TSession session, bool useColumnStore) {
7380
CreateTables(session, "schema/rstuv.sql", useColumnStore);
7481

7582
CreateTables(session, "schema/tpch.sql", useColumnStore);
@@ -79,6 +86,8 @@ static void CreateSampleTable(TSession session, bool useColumnStore) {
7986
CreateTables(session, "schema/tpcc.sql", useColumnStore);
8087

8188
CreateTables(session, "schema/lookupbug.sql", useColumnStore);
89+
90+
CreateView(session, "view/tpch_random_join_view.sql");
8291
}
8392

8493
static TKikimrRunner GetKikimrWithJoinSettings(bool useStreamLookupJoin = false, TString stats = "", bool useCBO = true){
@@ -96,8 +105,11 @@ static TKikimrRunner GetKikimrWithJoinSettings(bool useStreamLookupJoin = false,
96105
appConfig.MutableTableServiceConfig()->SetEnableKqpDataQueryStreamIdxLookupJoin(useStreamLookupJoin);
97106
appConfig.MutableTableServiceConfig()->SetEnableConstantFolding(true);
98107
appConfig.MutableTableServiceConfig()->SetCompileTimeoutMs(TDuration::Minutes(10).MilliSeconds());
108+
appConfig.MutableFeatureFlags()->SetEnableViews(true);
99109
if (!useCBO) {
100110
appConfig.MutableTableServiceConfig()->SetDefaultCostBasedOptimizationLevel(0);
111+
} else {
112+
appConfig.MutableTableServiceConfig()->SetDefaultCostBasedOptimizationLevel(4);
101113
}
102114

103115
auto serverSettings = TKikimrSettings().SetAppConfig(appConfig);
@@ -125,7 +137,7 @@ class TChainTester {
125137
TChainTester(size_t chainSize)
126138
: Kikimr(GetKikimrWithJoinSettings(false, GetStats(chainSize)))
127139
, TableClient(Kikimr.GetTableClient())
128-
, Session(TableClient.CreateSession().GetValueSync().GetSession())
140+
, Session(TableClient.GetSession().GetValueSync().GetSession())
129141
, ChainSize(chainSize)
130142
{}
131143

@@ -198,47 +210,65 @@ class TChainTester {
198210

199211
void ExplainJoinOrderTestDataQueryWithStats(const TString& queryPath, const TString& statsPath, bool useStreamLookupJoin, bool useColumnStore, bool useCBO = true) {
200212
auto kikimr = GetKikimrWithJoinSettings(useStreamLookupJoin, GetStatic(statsPath), useCBO);
201-
auto db = kikimr.GetTableClient();
202-
auto session = db.CreateSession().GetValueSync().GetSession();
213+
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableViews(true);
214+
auto db = kikimr.GetQueryClient();
215+
auto session = db.GetSession().GetValueSync().GetSession();
203216

204217
CreateSampleTable(session, useColumnStore);
205218

206219
/* join with parameters */
207220
{
208221
const TString query = GetStatic(queryPath);
209222

210-
auto result = session.ExplainDataQuery(query).ExtractValueSync();
223+
auto result =
224+
session.ExecuteQuery(
225+
query,
226+
NYdb::NQuery::TTxControl::NoTx(),
227+
NYdb::NQuery::TExecuteQuerySettings().ExecMode(NQuery::EExecMode::Explain)
228+
).ExtractValueSync();
211229
result.GetIssues().PrintTo(Cerr);
212230
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
213-
PrintPlan(result.GetPlan());
231+
PrintPlan(*result.GetStats()->GetPlan());
214232
}
215233
}
216234

217235
void TestOlapEstimationRowsCorrectness(const TString& queryPath, const TString& statsPath) {
218236
auto kikimr = GetKikimrWithJoinSettings(false, GetStatic(statsPath));
219-
auto db = kikimr.GetTableClient();
220-
auto session = db.CreateSession().GetValueSync().GetSession();
237+
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableViews(true);
238+
auto db = kikimr.GetQueryClient();
239+
auto session = db.GetSession().GetValueSync().GetSession();
221240

222241
CreateSampleTable(session, true);
223242

224243
const TString actualQuery = GetStatic(queryPath);
225244
TString actualPlan;
226245
{
227-
auto result = session.ExplainDataQuery(actualQuery).ExtractValueSync();
246+
auto result =
247+
session.ExecuteQuery(
248+
actualQuery,
249+
NYdb::NQuery::TTxControl::NoTx(),
250+
NYdb::NQuery::TExecuteQuerySettings().ExecMode(NQuery::EExecMode::Explain)
251+
).ExtractValueSync();
252+
253+
result.GetIssues().PrintTo(Cerr);
228254
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
229-
actualPlan = result.GetPlan();
255+
actualPlan = *result.GetStats()->GetPlan();
230256
PrintPlan(actualPlan);
231-
Cout << result.GetAst() << Endl;
257+
Cout << result.GetStats()->GetAst() << Endl;
232258
}
233259

234260
const TString expectedQuery = R"(PRAGMA kikimr.OptEnableOlapPushdown = "false";)" "\n" + actualQuery;
235261
TString expectedPlan;
236262
{
237-
auto result = session.ExplainDataQuery(expectedQuery).ExtractValueSync();
263+
auto result = session.ExecuteQuery(
264+
expectedQuery,
265+
NYdb::NQuery::TTxControl::NoTx(),
266+
NYdb::NQuery::TExecuteQuerySettings().ExecMode(NQuery::EExecMode::Explain)
267+
).ExtractValueSync();
238268
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
239-
expectedPlan = result.GetPlan();
269+
expectedPlan = *result.GetStats()->GetPlan();
240270
PrintPlan(expectedPlan);
241-
Cout << result.GetAst() << Endl;
271+
Cout << result.GetStats()->GetAst() << Endl;
242272
}
243273

244274
auto expectedDetailedPlan = GetDetailedJoinOrder(actualPlan, {.IncludeFilters = true, .IncludeOptimizerEstimation = true, .IncludeTables = false});
@@ -298,9 +328,9 @@ Y_UNIT_TEST_SUITE(OlapEstimationRowsCorrectness) {
298328
TestOlapEstimationRowsCorrectness("queries/tpcds78.sql", "stats/tpcds1000s.json");
299329
}
300330

301-
Y_UNIT_TEST(TPCDS87) {
302-
TestOlapEstimationRowsCorrectness("queries/tpcds87.sql", "stats/tpcds1000s.json");
303-
}
331+
// Y_UNIT_TEST(TPCDS87) {
332+
// TestOlapEstimationRowsCorrectness("queries/tpcds87.sql", "stats/tpcds1000s.json");
333+
// }
304334

305335
// Y_UNIT_TEST(TPCDS88) { // ???
306336
// TestOlapEstimationRowsCorrectness("queries/tpcds88.sql", "stats/tpcds1000s.json");
@@ -334,39 +364,50 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) {
334364

335365
TString ExecuteJoinOrderTestDataQueryWithStats(const TString& queryPath, const TString& statsPath, bool useStreamLookupJoin, bool useColumnStore, bool useCBO = true) {
336366
auto kikimr = GetKikimrWithJoinSettings(useStreamLookupJoin, GetStatic(statsPath), useCBO);
337-
auto db = kikimr.GetTableClient();
338-
auto session = db.CreateSession().GetValueSync().GetSession();
367+
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableViews(true);
368+
auto db = kikimr.GetQueryClient();
369+
auto session = db.GetSession().GetValueSync().GetSession();
339370

340371
CreateSampleTable(session, useColumnStore);
341372

342373
/* join with parameters */
343374
{
344375
const TString query = GetStatic(queryPath);
345376

346-
auto execRes = db.StreamExecuteScanQuery(query, TStreamExecScanQuerySettings().Explain(true)).ExtractValueSync();
377+
auto execRes = session.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync();
347378
execRes.GetIssues().PrintTo(Cerr);
348379
UNIT_ASSERT_VALUES_EQUAL(execRes.GetStatus(), EStatus::SUCCESS);
349-
auto plan = CollectStreamResult(execRes).PlanJson;
350-
PrintPlan(plan.GetRef());
351-
return plan.GetRef();
380+
381+
auto explainRes = session.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx(), NYdb::NQuery::TExecuteQuerySettings().ExecMode(NQuery::EExecMode::Explain)).ExtractValueSync();
382+
explainRes.GetIssues().PrintTo(Cerr);
383+
UNIT_ASSERT_VALUES_EQUAL(explainRes.GetStatus(), EStatus::SUCCESS);
384+
385+
PrintPlan(*explainRes.GetStats()->GetPlan());
386+
return *explainRes.GetStats()->GetPlan();
352387
}
353388
}
354389

355390
void CheckJoinCardinality(const TString& queryPath, const TString& statsPath, const TString& joinKind, double card, bool useStreamLookupJoin, bool useColumnStore) {
356391
auto kikimr = GetKikimrWithJoinSettings(useStreamLookupJoin, GetStatic(statsPath));
357-
auto db = kikimr.GetTableClient();
358-
auto session = db.CreateSession().GetValueSync().GetSession();
392+
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableViews(true);
393+
auto db = kikimr.GetQueryClient();
394+
auto session = db.GetSession().GetValueSync().GetSession();
359395

360396
CreateSampleTable(session, useColumnStore);
361397

362398
/* join with parameters */
363399
{
364400
const TString query = GetStatic(queryPath);
365401

366-
auto result = session.ExplainDataQuery(query).ExtractValueSync();
367-
PrintPlan(result.GetPlan());
402+
auto result =
403+
session.ExecuteQuery(
404+
query,
405+
NYdb::NQuery::TTxControl::NoTx(),
406+
NYdb::NQuery::TExecuteQuerySettings().ExecMode(NQuery::EExecMode::Explain)
407+
).ExtractValueSync();
408+
PrintPlan(*result.GetStats()->GetPlan());
368409
NJson::TJsonValue plan;
369-
NJson::ReadJsonTree(result.GetPlan(), &plan, true);
410+
NJson::ReadJsonTree(*result.GetStats()->GetPlan(), &plan, true);
370411

371412
if(!useStreamLookupJoin) {
372413
auto joinNode = FindPlanNodeByKv(plan.GetMapSafe().at("SimplifiedPlan"), "Node Type", joinKind);
@@ -438,6 +479,10 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) {
438479
ExecuteJoinOrderTestDataQueryWithStats("queries/datetime_constant_fold.sql", "stats/basic.json", StreamLookupJoin, ColumnStore);
439480
}
440481

482+
Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCHRandomJoinViewJustWorks, StreamLookupJoin, ColumnStore) {
483+
ExecuteJoinOrderTestDataQueryWithStats("queries/tpch_random_join_view_just_works.sql", "stats/tpch1000s.json", StreamLookupJoin, ColumnStore);
484+
}
485+
441486
Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCH3, StreamLookupJoin, ColumnStore) {
442487
ExecuteJoinOrderTestDataQueryWithStats("queries/tpch3.sql", "stats/tpch1000s.json", StreamLookupJoin, ColumnStore);
443488
}
@@ -462,14 +507,6 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) {
462507
ExecuteJoinOrderTestDataQueryWithStats("queries/tpch21.sql", "stats/tpch1000s.json", StreamLookupJoin, ColumnStore);
463508
}
464509

465-
Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCDS9, StreamLookupJoin, ColumnStore) {
466-
ExecuteJoinOrderTestDataQueryWithStats("queries/tpcds9.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore);
467-
}
468-
469-
Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCDS9_SMALL, StreamLookupJoin, ColumnStore) {
470-
ExecuteJoinOrderTestDataQueryWithStats("queries/tpcds9_small.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore);
471-
}
472-
473510
Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCDS16, StreamLookupJoin, ColumnStore) {
474511
ExecuteJoinOrderTestDataQueryWithStats("queries/tpcds16.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore);
475512
}
@@ -609,19 +646,25 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) {
609646
void CanonizedJoinOrderTest(const TString& queryPath, const TString& statsPath, TString correctJoinOrderPath, bool useStreamLookupJoin, bool useColumnStore
610647
) {
611648
auto kikimr = GetKikimrWithJoinSettings(useStreamLookupJoin, GetStatic(statsPath));
612-
auto db = kikimr.GetTableClient();
613-
auto session = db.CreateSession().GetValueSync().GetSession();
649+
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableViews(true);
650+
auto db = kikimr.GetQueryClient();
651+
auto session = db.GetSession().GetValueSync().GetSession();
614652

615653
CreateSampleTable(session, useColumnStore);
616654

617655
/* join with parameters */
618656
{
619657
const TString query = GetStatic(queryPath);
620658

621-
auto result = session.ExplainDataQuery(query).ExtractValueSync();
659+
auto result =
660+
session.ExecuteQuery(
661+
query,
662+
NYdb::NQuery::TTxControl::NoTx(),
663+
NYdb::NQuery::TExecuteQuerySettings().ExecMode(NQuery::EExecMode::Explain)
664+
).ExtractValueSync();
622665

623666
result.GetIssues().PrintTo(Cerr);
624-
PrintPlan(result.GetPlan());
667+
PrintPlan(*result.GetStats()->GetPlan());
625668
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
626669

627670
if (useStreamLookupJoin) {
@@ -632,7 +675,7 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) {
632675
correctJoinOrderPath = correctJoinOrderPath.substr(0, correctJoinOrderPath.find(".json")) + "_column_store.json";
633676
}
634677

635-
auto currentJoinOrder = GetPrettyJSON(GetDetailedJoinOrder(result.GetPlan()));
678+
auto currentJoinOrder = GetPrettyJSON(GetDetailedJoinOrder(*result.GetStats()->GetPlan()));
636679

637680
/* to canonize the tests use --test-param CANONIZE_JOIN_ORDER_TESTS=TRUE */
638681
TString canonize = GetTestParam("CANONIZE_JOIN_ORDER_TESTS"); canonize.to_lower();
@@ -643,9 +686,9 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) {
643686
}
644687

645688
TString ref = GetStatic(correctJoinOrderPath);
646-
Cout << "actual\n" << GetJoinOrder(result.GetPlan()).GetStringRobust() << Endl;
689+
Cout << "actual\n" << GetJoinOrder(*result.GetStats()->GetPlan()).GetStringRobust() << Endl;
647690
Cout << "expected\n" << GetJoinOrderFromDetailedJoinOrder(ref).GetStringRobust() << Endl;
648-
UNIT_ASSERT(JoinOrderAndAlgosMatch(result.GetPlan(), ref));
691+
UNIT_ASSERT(JoinOrderAndAlgosMatch(*result.GetStats()->GetPlan(), ref));
649692
}
650693
}
651694

0 commit comments

Comments
 (0)