Skip to content

Commit f4755b6

Browse files
nepalnepal
authored
allow fieldsubset optimizer with multiusage input (#862)
1 parent f7605ff commit f4755b6

File tree

12 files changed

+280
-134
lines changed

12 files changed

+280
-134
lines changed

‎ydb/library/yql/core/common_opt/yql_co_flow2.cpp

+133-134
Large diffs are not rendered by default.

‎ydb/library/yql/core/yql_type_annotation.h

+1
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,7 @@ struct TTypeAnnotationContext: public TThrRefBase {
269269
// compatibility with v0 or raw s-expression code
270270
bool OrderedColumns = false;
271271
TColumnOrderStorage::TPtr ColumnOrderStorage = new TColumnOrderStorage;
272+
THashSet<TString> OptimizerFlags;
272273

273274
TMaybe<TColumnOrder> LookupColumnOrder(const TExprNode& node) const;
274275
IGraphTransformer::TStatus SetColumnOrder(const TExprNode& node, const TColumnOrder& columnOrder, TExprContext& ctx);

‎ydb/library/yql/providers/config/yql_config_provider.cpp

+9
Original file line numberDiff line numberDiff line change
@@ -900,6 +900,15 @@ namespace {
900900
return false;
901901
}
902902
}
903+
else if (name == "OptimizerFlags") {
904+
for (auto& arg : args) {
905+
if (arg.empty()) {
906+
ctx.AddError(TIssue(pos, "Empty flags are not supported"));
907+
return false;
908+
}
909+
Types.OptimizerFlags.insert(to_lower(ToString(arg)));
910+
}
911+
}
903912
else {
904913
ctx.AddError(TIssue(pos, TStringBuilder() << "Unsupported command: " << name));
905914
return false;

‎ydb/library/yql/tests/sql/dq_file/part14/canondata/result.json

+22
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,28 @@
326326
}
327327
],
328328
"test.test[aggregate-group_by_rollup_rename-default.txt-Results]": [],
329+
"test.test[aggregate-group_by_ru_join_simple_fs_multiusage--Analyze]": [
330+
{
331+
"checksum": "24740252670115e3bbd1772580e48adf",
332+
"size": 9776,
333+
"uri": "https://{canondata_backend}/1781765/75774e90f574004e23fc9aacf32e1f561a8c66ec/resource.tar.gz#test.test_aggregate-group_by_ru_join_simple_fs_multiusage--Analyze_/plan.txt"
334+
}
335+
],
336+
"test.test[aggregate-group_by_ru_join_simple_fs_multiusage--Debug]": [
337+
{
338+
"checksum": "94020f8a1e57024f031cdeecd89e11a9",
339+
"size": 5934,
340+
"uri": "https://{canondata_backend}/1781765/75774e90f574004e23fc9aacf32e1f561a8c66ec/resource.tar.gz#test.test_aggregate-group_by_ru_join_simple_fs_multiusage--Debug_/opt.yql_patched"
341+
}
342+
],
343+
"test.test[aggregate-group_by_ru_join_simple_fs_multiusage--Plan]": [
344+
{
345+
"checksum": "24740252670115e3bbd1772580e48adf",
346+
"size": 9776,
347+
"uri": "https://{canondata_backend}/1781765/75774e90f574004e23fc9aacf32e1f561a8c66ec/resource.tar.gz#test.test_aggregate-group_by_ru_join_simple_fs_multiusage--Plan_/plan.txt"
348+
}
349+
],
350+
"test.test[aggregate-group_by_ru_join_simple_fs_multiusage--Results]": [],
329351
"test.test[ansi_idents-escaped_udf_name-default.txt-Analyze]": [
330352
{
331353
"checksum": "0f0658d4cb6fcef484dd34676f7939e7",

‎ydb/library/yql/tests/sql/dq_file/part3/canondata/result.json

+22
Original file line numberDiff line numberDiff line change
@@ -1500,6 +1500,28 @@
15001500
}
15011501
],
15021502
"test.test[optimizers-direct_row_after_merge--Results]": [],
1503+
"test.test[optimizers-length_over_merge_fs_multiusage--Analyze]": [
1504+
{
1505+
"checksum": "212be881133a20b5b73ef1250dbeda51",
1506+
"size": 960,
1507+
"uri": "https://{canondata_backend}/1599023/892497444bbacbe92ad2c557c09c697b859ad48d/resource.tar.gz#test.test_optimizers-length_over_merge_fs_multiusage--Analyze_/plan.txt"
1508+
}
1509+
],
1510+
"test.test[optimizers-length_over_merge_fs_multiusage--Debug]": [
1511+
{
1512+
"checksum": "880889c8fc79fc17434df304ebe066fb",
1513+
"size": 2791,
1514+
"uri": "https://{canondata_backend}/1599023/892497444bbacbe92ad2c557c09c697b859ad48d/resource.tar.gz#test.test_optimizers-length_over_merge_fs_multiusage--Debug_/opt.yql_patched"
1515+
}
1516+
],
1517+
"test.test[optimizers-length_over_merge_fs_multiusage--Plan]": [
1518+
{
1519+
"checksum": "212be881133a20b5b73ef1250dbeda51",
1520+
"size": 960,
1521+
"uri": "https://{canondata_backend}/1599023/892497444bbacbe92ad2c557c09c697b859ad48d/resource.tar.gz#test.test_optimizers-length_over_merge_fs_multiusage--Plan_/plan.txt"
1522+
}
1523+
],
1524+
"test.test[optimizers-length_over_merge_fs_multiusage--Results]": [],
15031525
"test.test[optimizers-yql-7324_duplicate_arg--Analyze]": [
15041526
{
15051527
"checksum": "b913ead12af51bc046e1f3344ff5134c",

‎ydb/library/yql/tests/sql/sql2yql/canondata/result.json

+28
Original file line numberDiff line numberDiff line change
@@ -2267,6 +2267,13 @@
22672267
"uri": "https://{canondata_backend}/1936997/d01e529b02a008637591c42bdd9c20f7eeda6588/resource.tar.gz#test_sql2yql.test_aggregate-group_by_ru_join_simple_/sql.yql"
22682268
}
22692269
],
2270+
"test_sql2yql.test[aggregate-group_by_ru_join_simple_fs_multiusage]": [
2271+
{
2272+
"checksum": "6e0fe40b211146a9bdab0d2e61d26168",
2273+
"size": 6000,
2274+
"uri": "https://{canondata_backend}/1599023/c4917fc01ae646b5e708e20b50ea7ae2325f5bb7/resource.tar.gz#test_sql2yql.test_aggregate-group_by_ru_join_simple_fs_multiusage_/sql.yql"
2275+
}
2276+
],
22702277
"test_sql2yql.test[aggregate-group_by_ru_join_star]": [
22712278
{
22722279
"checksum": "1d817501fac482312d72ad2e4a7881ba",
@@ -9785,6 +9792,13 @@
97859792
"uri": "https://{canondata_backend}/1937027/973c239492ba32946806ddc66cf0af4b38c06ae8/resource.tar.gz#test_sql2yql.test_optimizers-length_over_merge_/sql.yql"
97869793
}
97879794
],
9795+
"test_sql2yql.test[optimizers-length_over_merge_fs_multiusage]": [
9796+
{
9797+
"checksum": "a923d44d0dc694741762e7272690257f",
9798+
"size": 4487,
9799+
"uri": "https://{canondata_backend}/1599023/c4917fc01ae646b5e708e20b50ea7ae2325f5bb7/resource.tar.gz#test_sql2yql.test_optimizers-length_over_merge_fs_multiusage_/sql.yql"
9800+
}
9801+
],
97889802
"test_sql2yql.test[optimizers-multi_to_empty_constraint]": [
97899803
{
97909804
"checksum": "4288eb5377c92428681954e99403cbf0",
@@ -19557,6 +19571,13 @@
1955719571
"uri": "https://{canondata_backend}/1880306/64654158d6bfb1289c66c626a8162239289559d0/resource.tar.gz#test_sql_format.test_aggregate-group_by_ru_join_simple_/formatted.sql"
1955819572
}
1955919573
],
19574+
"test_sql_format.test[aggregate-group_by_ru_join_simple_fs_multiusage]": [
19575+
{
19576+
"checksum": "0369dc43a2062b648a486f7efb2873d0",
19577+
"size": 334,
19578+
"uri": "https://{canondata_backend}/1599023/c4917fc01ae646b5e708e20b50ea7ae2325f5bb7/resource.tar.gz#test_sql_format.test_aggregate-group_by_ru_join_simple_fs_multiusage_/formatted.sql"
19579+
}
19580+
],
1956019581
"test_sql_format.test[aggregate-group_by_ru_join_star]": [
1956119582
{
1956219583
"checksum": "e4a4a97de8afd6e188163c71da45788d",
@@ -27075,6 +27096,13 @@
2707527096
"uri": "https://{canondata_backend}/1880306/64654158d6bfb1289c66c626a8162239289559d0/resource.tar.gz#test_sql_format.test_optimizers-length_over_merge_/formatted.sql"
2707627097
}
2707727098
],
27099+
"test_sql_format.test[optimizers-length_over_merge_fs_multiusage]": [
27100+
{
27101+
"checksum": "81f76674682498288b92904264c7a0cd",
27102+
"size": 498,
27103+
"uri": "https://{canondata_backend}/1599023/c4917fc01ae646b5e708e20b50ea7ae2325f5bb7/resource.tar.gz#test_sql_format.test_optimizers-length_over_merge_fs_multiusage_/formatted.sql"
27104+
}
27105+
],
2707827106
"test_sql_format.test[optimizers-multi_to_empty_constraint]": [
2707927107
{
2708027108
"checksum": "417fc8b1ee413a204123ed56af1aa32f",

‎ydb/library/yql/tests/sql/suites/aggregate/group_by_ru_join_simple_fs_multiusage.cfg

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
in Input input_intersect.txt

‎ydb/library/yql/tests/sql/suites/aggregate/group_by_ru_join_simple_fs_multiusage.sql

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
/* syntax version 1 */
2+
/* postgres can not */
3+
4+
pragma sampleselect;
5+
pragma config.flags("OptimizerFlags", "FieldSubsetEnableMultiusage");
6+
select kk, sk, max(t2.subkey) as ss FROM plato.Input AS t1 INNER JOIN plato.Input AS t2 USING (key) GROUP BY ROLLUP(t1.key as kk, t1.subkey as sk) ORDER BY kk, sk;

‎ydb/library/yql/tests/sql/suites/optimizers/length_over_merge_fs_multiusage.cfg

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
in Input input3.txt
2+
res result.txt

‎ydb/library/yql/tests/sql/suites/optimizers/length_over_merge_fs_multiusage.sql

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
/* postgres can not */
2+
pragma config.flags("OptimizerFlags", "FieldSubsetEnableMultiusage");
3+
4+
use plato;
5+
6+
$input = (select key, key || subkey as subkey, value from Input);
7+
8+
$total_count = (select count(1) from $input);
9+
10+
$filtered = (select * from $input where key in ("023", "037", "075"));
11+
12+
$filtered_cnt = (select count(1) from $filtered);
13+
14+
select $filtered_cnt / cast($total_count as Double) as cnt;

‎ydb/library/yql/tests/sql/yt_native_file/part14/canondata/result.json

+21
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,27 @@
296296
"uri": "https://{canondata_backend}/1923547/5154c8bd8ef9ead4f609771f831f20c15e795571/resource.tar.gz#test.test_aggregate-group_by_rollup_rename-default.txt-Results_/results.txt"
297297
}
298298
],
299+
"test.test[aggregate-group_by_ru_join_simple_fs_multiusage--Debug]": [
300+
{
301+
"checksum": "ceeb8af83f2b01a10deebe2f805a3d01",
302+
"size": 9322,
303+
"uri": "https://{canondata_backend}/1936997/61b460f95ad76ee0a1465a60da61733fdb91dab4/resource.tar.gz#test.test_aggregate-group_by_ru_join_simple_fs_multiusage--Debug_/opt.yql"
304+
}
305+
],
306+
"test.test[aggregate-group_by_ru_join_simple_fs_multiusage--Plan]": [
307+
{
308+
"checksum": "760c79b2808d478d74cbc5d7682ae91d",
309+
"size": 12501,
310+
"uri": "https://{canondata_backend}/1936997/61b460f95ad76ee0a1465a60da61733fdb91dab4/resource.tar.gz#test.test_aggregate-group_by_ru_join_simple_fs_multiusage--Plan_/plan.txt"
311+
}
312+
],
313+
"test.test[aggregate-group_by_ru_join_simple_fs_multiusage--Results]": [
314+
{
315+
"checksum": "2469d61da89000051d881cc1a1aceb8a",
316+
"size": 7925,
317+
"uri": "https://{canondata_backend}/1936997/61b460f95ad76ee0a1465a60da61733fdb91dab4/resource.tar.gz#test.test_aggregate-group_by_ru_join_simple_fs_multiusage--Results_/results.txt"
318+
}
319+
],
299320
"test.test[ansi_idents-escaped_udf_name-default.txt-Debug]": [
300321
{
301322
"checksum": "4988877c69725bebc3eb77a48625f5cd",

‎ydb/library/yql/tests/sql/yt_native_file/part3/canondata/result.json

+21
Original file line numberDiff line numberDiff line change
@@ -1214,6 +1214,27 @@
12141214
"uri": "https://{canondata_backend}/1881367/66c7cb3390a7d019f5c9ce886d24fe86e37e82d0/resource.tar.gz#test.test_optimizers-direct_row_after_merge--Results_/results.txt"
12151215
}
12161216
],
1217+
"test.test[optimizers-length_over_merge_fs_multiusage--Debug]": [
1218+
{
1219+
"checksum": "e27c0e4ca190c9095c7cdfc6031ab1ea",
1220+
"size": 1740,
1221+
"uri": "https://{canondata_backend}/1889210/f5cbb7beff327fcfa5cbcdb1e04fd02c24f4e33f/resource.tar.gz#test.test_optimizers-length_over_merge_fs_multiusage--Debug_/opt.yql"
1222+
}
1223+
],
1224+
"test.test[optimizers-length_over_merge_fs_multiusage--Plan]": [
1225+
{
1226+
"checksum": "17aa2b0bb49eb66fecaf1b350cccaabc",
1227+
"size": 5190,
1228+
"uri": "https://{canondata_backend}/1889210/f5cbb7beff327fcfa5cbcdb1e04fd02c24f4e33f/resource.tar.gz#test.test_optimizers-length_over_merge_fs_multiusage--Plan_/plan.txt"
1229+
}
1230+
],
1231+
"test.test[optimizers-length_over_merge_fs_multiusage--Results]": [
1232+
{
1233+
"checksum": "5d1098186fdb44867314966f03aded2a",
1234+
"size": 857,
1235+
"uri": "https://{canondata_backend}/1889210/f5cbb7beff327fcfa5cbcdb1e04fd02c24f4e33f/resource.tar.gz#test.test_optimizers-length_over_merge_fs_multiusage--Results_/results.txt"
1236+
}
1237+
],
12171238
"test.test[optimizers-yql-7324_duplicate_arg--Debug]": [
12181239
{
12191240
"checksum": "3bd44f4f74db4b23c13174edccd97e7d",

0 commit comments

Comments
 (0)