Skip to content

Commit 582cd7b

Browse files
Tony-RomanovTony-Romanov
authored
Fuse FlatMap and ShuffleByKeys. (#6727)
1 parent abe4d5b commit 582cd7b

File tree

21 files changed

+269
-269
lines changed

21 files changed

+269
-269
lines changed

‎ydb/library/yql/core/common_opt/yql_co_flow1.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -1409,7 +1409,7 @@ TExprNode::TPtr OptimizeFlatMap(const TExprNode::TPtr& node, TExprContext& ctx,
14091409
return FuseFlatMapOverByKey<false>(*node, ctx);
14101410
}
14111411

1412-
if (node->Head().IsCallable({"PartitionByKey", "PartitionsByKeys"})) {
1412+
if (node->Head().IsCallable({"PartitionByKey", "PartitionsByKeys", "ShuffleByKeys"})) {
14131413
return FuseFlatMapOverByKey<true>(*node, ctx);
14141414
}
14151415
}
@@ -1477,7 +1477,7 @@ TExprNode::TPtr OptimizeFlatMap(const TExprNode::TPtr& node, TExprContext& ctx,
14771477
{
14781478
auto canPush = [&](const auto& child) {
14791479
// we push FlatMap over Extend only if it can later be fused with child
1480-
return child->IsCallable({Ordered ? "OrderedFlatMap" : "FlatMap", "GroupByKey", "CombineByKey", "PartitionByKey", "PartitionsByKeys",
1480+
return child->IsCallable({Ordered ? "OrderedFlatMap" : "FlatMap", "GroupByKey", "CombineByKey", "PartitionByKey", "PartitionsByKeys", "ShuffleByKeys",
14811481
"ListIf", "FlatListIf", "AsList", "ToList"}) && optCtx.IsSingleUsage(*child);
14821482
};
14831483
if (AllOf(node->Head().ChildrenList(), canPush)) {

‎ydb/library/yql/tests/sql/dq_file/part0/canondata/result.json

+6-6
Original file line numberDiff line numberDiff line change
@@ -376,9 +376,9 @@
376376
],
377377
"test.test[aggregate-group_by_ru_join_qualified-default.txt-Debug]": [
378378
{
379-
"checksum": "272ec39ebf5a17f62026a33e2a2934cb",
380-
"size": 6253,
381-
"uri": "https://{canondata_backend}/1937027/de4be6db5a9d9653a2d7cf00ba5ccfe48c1b3a99/resource.tar.gz#test.test_aggregate-group_by_ru_join_qualified-default.txt-Debug_/opt.yql_patched"
379+
"checksum": "3991282d51b7d5dee89f33a9f33722db",
380+
"size": 6308,
381+
"uri": "https://{canondata_backend}/1937367/063b5dea707b1d0cd626a2c0777d3187304767d5/resource.tar.gz#test.test_aggregate-group_by_ru_join_qualified-default.txt-Debug_/opt.yql_patched"
382382
}
383383
],
384384
"test.test[aggregate-group_by_ru_join_qualified-default.txt-Plan]": [
@@ -2622,9 +2622,9 @@
26222622
],
26232623
"test.test[pg-tpch-q05-default.txt-Debug]": [
26242624
{
2625-
"checksum": "1503b12c21cbe005e88e6e147457b5b2",
2626-
"size": 14977,
2627-
"uri": "https://{canondata_backend}/1871182/4d40c08aee85fb231923402132a00b2ae47137cc/resource.tar.gz#test.test_pg-tpch-q05-default.txt-Debug_/opt.yql_patched"
2625+
"checksum": "bb190dc1cc70746fd0f78e38113ccc0d",
2626+
"size": 15012,
2627+
"uri": "https://{canondata_backend}/1920236/572a17259b7fa192c75eeb0cceaf48cca4beb191/resource.tar.gz#test.test_pg-tpch-q05-default.txt-Debug_/opt.yql_patched"
26282628
}
26292629
],
26302630
"test.test[pg-tpch-q05-default.txt-Plan]": [

‎ydb/library/yql/tests/sql/dq_file/part1/canondata/result.json

+18-18
Original file line numberDiff line numberDiff line change
@@ -452,9 +452,9 @@
452452
],
453453
"test.test[aggregate-group_by_gs_duo--Debug]": [
454454
{
455-
"checksum": "25b53d13ad1aaa9594a636741e68d3ea",
456-
"size": 4031,
457-
"uri": "https://{canondata_backend}/1937027/591a1ceca790d81eaf524a7a3e730722b0d7bdb7/resource.tar.gz#test.test_aggregate-group_by_gs_duo--Debug_/opt.yql_patched"
455+
"checksum": "59257d703f2930e08d3b7f69094f53c9",
456+
"size": 4083,
457+
"uri": "https://{canondata_backend}/1130705/223d79eda7e49588c54267c8b7c488154ed801c9/resource.tar.gz#test.test_aggregate-group_by_gs_duo--Debug_/opt.yql_patched"
458458
}
459459
],
460460
"test.test[aggregate-group_by_gs_duo--Plan]": [
@@ -474,9 +474,9 @@
474474
],
475475
"test.test[aggregate-group_by_gs_flatten-default.txt-Debug]": [
476476
{
477-
"checksum": "49cd9fa336a5fec980795d60d46618cc",
478-
"size": 3857,
479-
"uri": "https://{canondata_backend}/1871102/093ef1237a5eb90e2e1f6670f45824dd7aa652e1/resource.tar.gz#test.test_aggregate-group_by_gs_flatten-default.txt-Debug_/opt.yql_patched"
477+
"checksum": "78c0e51be35dd604e8e44b570af2881b",
478+
"size": 3909,
479+
"uri": "https://{canondata_backend}/1937424/0a7fa81182305af7b414a8e11e361266a61bc724/resource.tar.gz#test.test_aggregate-group_by_gs_flatten-default.txt-Debug_/opt.yql_patched"
480480
}
481481
],
482482
"test.test[aggregate-group_by_gs_flatten-default.txt-Plan]": [
@@ -496,9 +496,9 @@
496496
],
497497
"test.test[aggregate-group_by_ru_join_simple--Debug]": [
498498
{
499-
"checksum": "9571d35aa29a9d3ee358bf0f0a41eb15",
500-
"size": 5841,
501-
"uri": "https://{canondata_backend}/1871102/093ef1237a5eb90e2e1f6670f45824dd7aa652e1/resource.tar.gz#test.test_aggregate-group_by_ru_join_simple--Debug_/opt.yql_patched"
499+
"checksum": "162cf76c50d7399318a5e3db3ceebe3c",
500+
"size": 5896,
501+
"uri": "https://{canondata_backend}/1937424/296365bcff5ae3a0411fd98abb5f7c306dac1a7d/resource.tar.gz#test.test_aggregate-group_by_ru_join_simple--Debug_/opt.yql_patched"
502502
}
503503
],
504504
"test.test[aggregate-group_by_ru_join_simple--Plan]": [
@@ -1139,9 +1139,9 @@
11391139
],
11401140
"test.test[hor_join-out_sampling--Debug]": [
11411141
{
1142-
"checksum": "a7a731afc7145a0cee7006d262255061",
1143-
"size": 5549,
1144-
"uri": "https://{canondata_backend}/1917492/1ed6d08398686e90568735860251083949d84e4e/resource.tar.gz#test.test_hor_join-out_sampling--Debug_/opt.yql_patched"
1142+
"checksum": "8606ab607900ab1c285f8ba104e78a82",
1143+
"size": 5612,
1144+
"uri": "https://{canondata_backend}/1880306/b2c00ff823e390f0263acf2dbb68c876e0b31abd/resource.tar.gz#test.test_hor_join-out_sampling--Debug_/opt.yql_patched"
11451145
}
11461146
],
11471147
"test.test[hor_join-out_sampling--Plan]": [
@@ -2447,9 +2447,9 @@
24472447
],
24482448
"test.test[pg-tpch-q22-default.txt-Debug]": [
24492449
{
2450-
"checksum": "6c8a685a6ddac197b5c4729aadaeaddd",
2451-
"size": 12561,
2452-
"uri": "https://{canondata_backend}/1903885/9c646d893756f8f6a633ac2a2027ef874f86cf92/resource.tar.gz#test.test_pg-tpch-q22-default.txt-Debug_/opt.yql_patched"
2450+
"checksum": "fecb4678857263b4b71bb46a61e6f4e7",
2451+
"size": 12565,
2452+
"uri": "https://{canondata_backend}/1942100/e8db2c3a2d1260e3a0b05ff2621b1f53f6bfcf33/resource.tar.gz#test.test_pg-tpch-q22-default.txt-Debug_/opt.yql_patched"
24532453
}
24542454
],
24552455
"test.test[pg-tpch-q22-default.txt-Plan]": [
@@ -2919,9 +2919,9 @@
29192919
],
29202920
"test.test[table_range-range_with_view--Debug]": [
29212921
{
2922-
"checksum": "8c3ab9a7d76fc314c4d5be392e075ad7",
2923-
"size": 3947,
2924-
"uri": "https://{canondata_backend}/1917492/1ed6d08398686e90568735860251083949d84e4e/resource.tar.gz#test.test_table_range-range_with_view--Debug_/opt.yql_patched"
2922+
"checksum": "5b6927fd57b28a292fc0a17afefdf28a",
2923+
"size": 3982,
2924+
"uri": "https://{canondata_backend}/1871102/a14717a3c8c558cf8fefe6d46cd5b04ed47ccc80/resource.tar.gz#test.test_table_range-range_with_view--Debug_/opt.yql_patched"
29252925
}
29262926
],
29272927
"test.test[table_range-range_with_view--Plan]": [

‎ydb/library/yql/tests/sql/dq_file/part10/canondata/result.json

+21-21
Original file line numberDiff line numberDiff line change
@@ -303,9 +303,9 @@
303303
],
304304
"test.test[aggregate-group_by_mul_gs_ru--Debug]": [
305305
{
306-
"checksum": "0638146b59bc226b8388bd56d7b4313f",
307-
"size": 6072,
308-
"uri": "https://{canondata_backend}/1923547/320f607d9e9c19a93a835d3183938f1fba6dd52c/resource.tar.gz#test.test_aggregate-group_by_mul_gs_ru--Debug_/opt.yql_patched"
306+
"checksum": "ea41d7d534054a215601887475a08f36",
307+
"size": 6129,
308+
"uri": "https://{canondata_backend}/1773845/c255bb2163f40b1ca08f81b23e10624ae1969605/resource.tar.gz#test.test_aggregate-group_by_mul_gs_ru--Debug_/opt.yql_patched"
309309
}
310310
],
311311
"test.test[aggregate-group_by_mul_gs_ru--Plan]": [
@@ -340,23 +340,23 @@
340340
"test.test[aggregate-percentiles_ungrouped--Results]": [],
341341
"test.test[aggregate-subquery_aggregation--Analyze]": [
342342
{
343-
"checksum": "9dec3798de4c88e1f79c01259fea4c30",
344-
"size": 6627,
345-
"uri": "https://{canondata_backend}/1600758/32cfdeb8c6377a2e7e62c6c4adbb95f25af7669b/resource.tar.gz#test.test_aggregate-subquery_aggregation--Analyze_/plan.txt"
343+
"checksum": "49fb64ebed43ca080f30a0b8387033c3",
344+
"size": 6149,
345+
"uri": "https://{canondata_backend}/1773845/27425423327af8b10415a6bbb80a5aec7c55b13f/resource.tar.gz#test.test_aggregate-subquery_aggregation--Analyze_/plan.txt"
346346
}
347347
],
348348
"test.test[aggregate-subquery_aggregation--Debug]": [
349349
{
350-
"checksum": "be5464a62cb392e7ba87b2cce10d0a26",
351-
"size": 3147,
352-
"uri": "https://{canondata_backend}/937458/3ce9d3f90d17a09aa182a1ae8e08f2f065219fab/resource.tar.gz#test.test_aggregate-subquery_aggregation--Debug_/opt.yql_patched"
350+
"checksum": "9ebd4251fdeffd3409adc7fd7de958e7",
351+
"size": 3531,
352+
"uri": "https://{canondata_backend}/1923547/3423d2190488ae10821f2c300e70cd1b3ed9fc6d/resource.tar.gz#test.test_aggregate-subquery_aggregation--Debug_/opt.yql_patched"
353353
}
354354
],
355355
"test.test[aggregate-subquery_aggregation--Plan]": [
356356
{
357-
"checksum": "9dec3798de4c88e1f79c01259fea4c30",
358-
"size": 6627,
359-
"uri": "https://{canondata_backend}/1600758/32cfdeb8c6377a2e7e62c6c4adbb95f25af7669b/resource.tar.gz#test.test_aggregate-subquery_aggregation--Plan_/plan.txt"
357+
"checksum": "49fb64ebed43ca080f30a0b8387033c3",
358+
"size": 6149,
359+
"uri": "https://{canondata_backend}/1781765/e399fc9631f4d96fcb5c717860c01dc3564d9b6c/resource.tar.gz#test.test_aggregate-subquery_aggregation--Plan_/plan.txt"
360360
}
361361
],
362362
"test.test[aggregate-subquery_aggregation--Results]": [],
@@ -1097,9 +1097,9 @@
10971097
],
10981098
"test.test[in-in_sorted_by_tuple--Debug]": [
10991099
{
1100-
"checksum": "8ea6ef4b55e6a9ac3a47b1ad51362e85",
1101-
"size": 6179,
1102-
"uri": "https://{canondata_backend}/1031349/6c70521322fc43f752ef6b89f8667fefd006af8b/resource.tar.gz#test.test_in-in_sorted_by_tuple--Debug_/opt.yql_patched"
1100+
"checksum": "0513db9cb6108aa325637f0d9e48a4ad",
1101+
"size": 6163,
1102+
"uri": "https://{canondata_backend}/1781765/73c281723c52f983d35dc6582e63be16c38aa4cc/resource.tar.gz#test.test_in-in_sorted_by_tuple--Debug_/opt.yql_patched"
11031103
}
11041104
],
11051105
"test.test[in-in_sorted_by_tuple--Plan]": [
@@ -1864,9 +1864,9 @@
18641864
],
18651865
"test.test[pg-aggregate_combine--Debug]": [
18661866
{
1867-
"checksum": "9fa29a162f8152ed7a7959ce4ec57f6e",
1868-
"size": 3715,
1869-
"uri": "https://{canondata_backend}/995452/59cb21feb51bcd4aaf002804abbfbb4a05ffe65f/resource.tar.gz#test.test_pg-aggregate_combine--Debug_/opt.yql_patched"
1867+
"checksum": "e50861c99c37ca8ce4bdd20148fa0753",
1868+
"size": 3750,
1869+
"uri": "https://{canondata_backend}/1903280/f275d6f071715007b59c0fcf1ce9a3d4eafb9599/resource.tar.gz#test.test_pg-aggregate_combine--Debug_/opt.yql_patched"
18701870
}
18711871
],
18721872
"test.test[pg-aggregate_combine--Plan]": [
@@ -2765,9 +2765,9 @@
27652765
],
27662766
"test.test[window-win_func_over_group_by_list_names--Debug]": [
27672767
{
2768-
"checksum": "93272f345d0939b83721190ea5597f30",
2769-
"size": 8213,
2770-
"uri": "https://{canondata_backend}/1923547/320f607d9e9c19a93a835d3183938f1fba6dd52c/resource.tar.gz#test.test_window-win_func_over_group_by_list_names--Debug_/opt.yql_patched"
2768+
"checksum": "a18ba714975de5a61f001dc9c971bd68",
2769+
"size": 8248,
2770+
"uri": "https://{canondata_backend}/1871182/0dad491c929525221cf344b9f6f54cd14f2ce4f0/resource.tar.gz#test.test_window-win_func_over_group_by_list_names--Debug_/opt.yql_patched"
27712771
}
27722772
],
27732773
"test.test[window-win_func_over_group_by_list_names--Plan]": [

‎ydb/library/yql/tests/sql/dq_file/part11/canondata/result.json

+15-15
Original file line numberDiff line numberDiff line change
@@ -300,9 +300,9 @@
300300
],
301301
"test.test[aggregate-group_by_expr_only_join--Debug]": [
302302
{
303-
"checksum": "d37afd4b23ea87460590c3e3ec526be0",
304-
"size": 3384,
305-
"uri": "https://{canondata_backend}/1942173/5dda369a5c566435d55e882d65f0212fa3dfb906/resource.tar.gz#test.test_aggregate-group_by_expr_only_join--Debug_/opt.yql_patched"
303+
"checksum": "312134c4c3f844be9cd4bd0723ebe22f",
304+
"size": 3419,
305+
"uri": "https://{canondata_backend}/1784826/24ca0c0f39761c06f1f5ed352136813f7ad8dc1f/resource.tar.gz#test.test_aggregate-group_by_expr_only_join--Debug_/opt.yql_patched"
306306
}
307307
],
308308
"test.test[aggregate-group_by_expr_only_join--Plan]": [
@@ -322,9 +322,9 @@
322322
],
323323
"test.test[aggregate-group_by_gs_few_empty--Debug]": [
324324
{
325-
"checksum": "0ac1e3214471087b86a97fa21e69c3a2",
326-
"size": 6528,
327-
"uri": "https://{canondata_backend}/1936947/a99026e839b7e22714c2a9a81971a3b5e3ed1eb4/resource.tar.gz#test.test_aggregate-group_by_gs_few_empty--Debug_/opt.yql_patched"
325+
"checksum": "3bbb4eb639e235c657a648557ce521ce",
326+
"size": 6580,
327+
"uri": "https://{canondata_backend}/1937367/e5d3b4a217429148a8315cf4e228d45b21f861fe/resource.tar.gz#test.test_aggregate-group_by_gs_few_empty--Debug_/opt.yql_patched"
328328
}
329329
],
330330
"test.test[aggregate-group_by_gs_few_empty--Plan]": [
@@ -344,9 +344,9 @@
344344
],
345345
"test.test[aggregate-group_by_gs_simp--Debug]": [
346346
{
347-
"checksum": "f97f89c53ae93b6a6c7b9912e3950550",
348-
"size": 5421,
349-
"uri": "https://{canondata_backend}/1936947/a99026e839b7e22714c2a9a81971a3b5e3ed1eb4/resource.tar.gz#test.test_aggregate-group_by_gs_simp--Debug_/opt.yql_patched"
347+
"checksum": "873e07e43b5b691bd8457dc9da486ff2",
348+
"size": 5474,
349+
"uri": "https://{canondata_backend}/1937367/e2a772964cf46b8a14a828d48a136378216522b9/resource.tar.gz#test.test_aggregate-group_by_gs_simp--Debug_/opt.yql_patched"
350350
}
351351
],
352352
"test.test[aggregate-group_by_gs_simp--Plan]": [
@@ -366,9 +366,9 @@
366366
],
367367
"test.test[aggregate-group_by_rollup_grouping--Debug]": [
368368
{
369-
"checksum": "1bc4644b215619613368c778f07fa578",
370-
"size": 7107,
371-
"uri": "https://{canondata_backend}/1903280/419b5c18140d44a17c33d80899398c8647846b33/resource.tar.gz#test.test_aggregate-group_by_rollup_grouping--Debug_/opt.yql_patched"
369+
"checksum": "c24d41f35785c78d23a30c634eacb1f8",
370+
"size": 7167,
371+
"uri": "https://{canondata_backend}/1773845/e1901cc86dc30911ad22db641ffb6fe66c04423b/resource.tar.gz#test.test_aggregate-group_by_rollup_grouping--Debug_/opt.yql_patched"
372372
}
373373
],
374374
"test.test[aggregate-group_by_rollup_grouping--Plan]": [
@@ -388,9 +388,9 @@
388388
],
389389
"test.test[aggregate-having_distinct_expr--Debug]": [
390390
{
391-
"checksum": "6a45904e076ad4dbbe5a94c52b69c43f",
392-
"size": 3276,
393-
"uri": "https://{canondata_backend}/1942173/5dda369a5c566435d55e882d65f0212fa3dfb906/resource.tar.gz#test.test_aggregate-having_distinct_expr--Debug_/opt.yql_patched"
391+
"checksum": "4bdc0046de46875ae83dcdda879b5ba7",
392+
"size": 3311,
393+
"uri": "https://{canondata_backend}/1847551/155e040f6efb509114e481612c26cc3259caa89d/resource.tar.gz#test.test_aggregate-having_distinct_expr--Debug_/opt.yql_patched"
394394
}
395395
],
396396
"test.test[aggregate-having_distinct_expr--Plan]": [

‎ydb/library/yql/tests/sql/dq_file/part12/canondata/result.json

+12-12
Original file line numberDiff line numberDiff line change
@@ -272,9 +272,9 @@
272272
],
273273
"test.test[aggregate-group_by_rollup_grouping_hum--Debug]": [
274274
{
275-
"checksum": "6f56ba8703599d31b777058eed994bf0",
276-
"size": 7947,
277-
"uri": "https://{canondata_backend}/1777230/915011f8f5c826c23aaf0fd2e67aa8d2fb1f93cd/resource.tar.gz#test.test_aggregate-group_by_rollup_grouping_hum--Debug_/opt.yql_patched"
275+
"checksum": "0fa3ad8c20368a624649240306680c0e",
276+
"size": 8006,
277+
"uri": "https://{canondata_backend}/1847551/cea98224a0242fa122932bfd335599c5107ce35b/resource.tar.gz#test.test_aggregate-group_by_rollup_grouping_hum--Debug_/opt.yql_patched"
278278
}
279279
],
280280
"test.test[aggregate-group_by_rollup_grouping_hum--Plan]": [
@@ -2838,9 +2838,9 @@
28382838
],
28392839
"test.test[pg-tpch-q12-default.txt-Debug]": [
28402840
{
2841-
"checksum": "6b2a557909360d686cb42cff341ee6cb",
2842-
"size": 8286,
2843-
"uri": "https://{canondata_backend}/1931696/bd85f9162ce16e90dcd72c36d5db0a32dc3b7973/resource.tar.gz#test.test_pg-tpch-q12-default.txt-Debug_/opt.yql_patched"
2841+
"checksum": "470ab72559d1adc4c1158840196e5daf",
2842+
"size": 8321,
2843+
"uri": "https://{canondata_backend}/1903280/e2ac99e9bdb8d69b305b73b814934659ff8e62a2/resource.tar.gz#test.test_pg-tpch-q12-default.txt-Debug_/opt.yql_patched"
28442844
}
28452845
],
28462846
"test.test[pg-tpch-q12-default.txt-Plan]": [
@@ -2860,9 +2860,9 @@
28602860
],
28612861
"test.test[pg-tpch-q16-default.txt-Debug]": [
28622862
{
2863-
"checksum": "0c889dbbe3e6b8c23d01191a889cc2d5",
2864-
"size": 11518,
2865-
"uri": "https://{canondata_backend}/1931696/bd85f9162ce16e90dcd72c36d5db0a32dc3b7973/resource.tar.gz#test.test_pg-tpch-q16-default.txt-Debug_/opt.yql_patched"
2863+
"checksum": "9b7d8707f311b504ad1e28eded169df7",
2864+
"size": 11553,
2865+
"uri": "https://{canondata_backend}/1689644/1070983c3ae32f3ea4fcb7d6e9d37ca583b82a1b/resource.tar.gz#test.test_pg-tpch-q16-default.txt-Debug_/opt.yql_patched"
28662866
}
28672867
],
28682868
"test.test[pg-tpch-q16-default.txt-Plan]": [
@@ -3349,9 +3349,9 @@
33493349
],
33503350
"test.test[select-trivial_having-default.txt-Debug]": [
33513351
{
3352-
"checksum": "1cfeecb75ebfcbc6dd730c75edbb7b8f",
3353-
"size": 2380,
3354-
"uri": "https://{canondata_backend}/1942173/50b4ae48e906d86b27ee0b68ed5a08b5ad6bf50e/resource.tar.gz#test.test_select-trivial_having-default.txt-Debug_/opt.yql_patched"
3352+
"checksum": "36e41a7925a3f6f19fda9f5efced4017",
3353+
"size": 2415,
3354+
"uri": "https://{canondata_backend}/1784117/b9574b141cedb261a8563a97d9a372d9277da2ce/resource.tar.gz#test.test_select-trivial_having-default.txt-Debug_/opt.yql_patched"
33553355
}
33563356
],
33573357
"test.test[select-trivial_having-default.txt-Plan]": [

0 commit comments

Comments
 (0)