Skip to content

Commit a0decc6

Browse files
rvu1024rvu1024
committed
More fixes + tests
1 parent bed5384 commit a0decc6

File tree

10 files changed

+160
-5
lines changed

10 files changed

+160
-5
lines changed

‎ydb/library/yql/providers/yt/provider/yql_yt_logical_optimize.cpp

+9-3
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ class TYtLogicalOptProposalTransformer : public TOptimizeTransformerBase {
5454
AddHandler(0, &TCoUnorderedBase::Match, HNDL(Unordered));
5555
AddHandler(0, &TCoAggregate::Match, HNDL(CountAggregate));
5656
AddHandler(0, &TYtReadTable::Match, HNDL(ZeroSampleToZeroLimit));
57+
AddHandler(0, &TCoMatchRecognize::Match, HNDL(MatchRecognize));
5758

5859
AddHandler(1, &TCoFilterNullMembers::Match, HNDL(FilterNullMemebers<TCoFilterNullMembers>));
5960
AddHandler(1, &TCoSkipNullMembers::Match, HNDL(FilterNullMemebers<TCoSkipNullMembers>));
@@ -75,8 +76,6 @@ class TYtLogicalOptProposalTransformer : public TOptimizeTransformerBase {
7576

7677
AddHandler(2, &TCoEquiJoin::Match, HNDL(ConvertToCommonTypeForForcedMergeJoin));
7778
AddHandler(2, &TCoShuffleByKeys::Match, HNDL(ShuffleByKeys));
78-
79-
AddHandler(0, &TCoMatchRecognize::Match, HNDL(MatchRecognize));
8079
#undef HNDL
8180
}
8281

@@ -139,13 +138,20 @@ class TYtLogicalOptProposalTransformer : public TOptimizeTransformerBase {
139138
effectiveColumns.insert(column.Name);
140139
}
141140

141+
if (NYql::HasSetting(op.Settings().Ref(), EYtSettingType::KeepSorted)) {
142+
for (size_t i = 0; i < rowSpec->SortedBy.size(); ++i) {
143+
const bool inserted = effectiveColumns.insert(rowSpec->SortedBy[i]).second;
144+
keepColumns = keepColumns || inserted;
145+
}
146+
}
147+
142148
if (!path.Ranges().Maybe<TCoVoid>()) {
143149
// add columns which are implicitly used by path.Ranges(), but not included in path.Columns();
144150
const auto ranges = TYtRangesInfo(path.Ranges());
145151
const size_t usedKeyPrefix = ranges.GetUsedKeyPrefixLength();
146152
YQL_ENSURE(usedKeyPrefix <= rowSpec->SortedBy.size());
147153
for (size_t i = 0; i < usedKeyPrefix; ++i) {
148-
bool inserted = effectiveColumns.insert(rowSpec->SortedBy[i]).second;
154+
const bool inserted = effectiveColumns.insert(rowSpec->SortedBy[i]).second;
149155
keepColumns = keepColumns || inserted;
150156
}
151157
}

‎ydb/library/yql/providers/yt/provider/yql_yt_physical_finalizing.cpp

+6
Original file line numberDiff line numberDiff line change
@@ -631,6 +631,12 @@ class TYtPhysicalFinalizingTransformer : public TSyncTransformerBase {
631631

632632
bool good = true;
633633
THashSet<TString> usedColumns;
634+
if (NYql::HasSetting(*writer->Child(TYtTransientOpBase::idx_Settings), EYtSettingType::KeepSorted)) {
635+
for (size_t i = 0; i < rowSpec.SortedBy.size(); ++i) {
636+
usedColumns.insert(rowSpec.SortedBy[i]);
637+
}
638+
}
639+
634640
for (auto& item: x.second) {
635641
if (auto rawSection = std::get<1>(item)) {
636642
if (HasNonEmptyKeyFilter(TYtSection(rawSection))) {

‎ydb/library/yql/providers/yt/provider/yql_yt_physical_optimize.cpp

+20-2
Original file line numberDiff line numberDiff line change
@@ -6903,7 +6903,7 @@ class TYtPhysicalOptProposalTransformer : public TOptimizeTransformerBase {
69036903
continue;
69046904
}
69056905

6906-
if (NYql::HasSettingsExcept(innerMerge.Settings().Ref(), EYtSettingType::Limit)) {
6906+
if (NYql::HasSettingsExcept(innerMerge.Settings().Ref(), EYtSettingType::KeepSorted | EYtSettingType::Limit)) {
69076907
continue;
69086908
}
69096909

@@ -6934,6 +6934,18 @@ class TYtPhysicalOptProposalTransformer : public TOptimizeTransformerBase {
69346934
continue;
69356935
}
69366936

6937+
if (NYql::HasSetting(innerMerge.Settings().Ref(), EYtSettingType::KeepSorted)) {
6938+
if (!AllOf(innerMergeSection.Paths(), [](const auto& path) {
6939+
auto op = path.Table().template Maybe<TYtOutput>().Operation();
6940+
return op && (op.template Maybe<TYtTouch>() || (op.Raw()->HasResult() && op.Raw()->GetResult().IsWorld()));
6941+
})) {
6942+
continue;
6943+
}
6944+
}
6945+
if (hasTakeSkip && AnyOf(innerMergeSection.Paths(), [](const auto& path) { return !path.Ranges().template Maybe<TCoVoid>(); })) {
6946+
continue;
6947+
}
6948+
69376949
const bool unordered = IsUnorderedOutput(path.Table().Cast<TYtOutput>());
69386950
auto mergeOutRowSpec = TYqlRowSpecInfo(innerMerge.Output().Item(0).RowSpec());
69396951
if (innerMergeSection.Paths().Size() > 1) {
@@ -7430,7 +7442,7 @@ class TYtPhysicalOptProposalTransformer : public TOptimizeTransformerBase {
74307442
return node;
74317443
}
74327444

7433-
if (NYql::HasAnySetting(merge.Settings().Ref(), EYtSettingType::ForceTransform | EYtSettingType::CombineChunks | EYtSettingType::KeepSorted)) {
7445+
if (NYql::HasAnySetting(merge.Settings().Ref(), EYtSettingType::ForceTransform | EYtSettingType::CombineChunks)) {
74347446
return node;
74357447
}
74367448

@@ -7461,6 +7473,12 @@ class TYtPhysicalOptProposalTransformer : public TOptimizeTransformerBase {
74617473
if (NYql::HasNonEmptyKeyFilter(section)) {
74627474
return node;
74637475
}
7476+
if (NYql::HasSetting(merge.Settings().Ref(), EYtSettingType::KeepSorted)) {
7477+
auto op = path.Table().Maybe<TYtOutput>().Operation().Cast();
7478+
if (!(op.Ref().HasResult() && op.Ref().GetResult().Type() == TExprNode::World || op.Maybe<TYtTouch>())) {
7479+
return node;
7480+
}
7481+
}
74647482
TYtOutTableInfo outTableInfo(merge.Output().Item(0));
74657483
if (!tableInfo->RowSpec->CompareSortness(*outTableInfo.RowSpec)) {
74667484
return node;

‎ydb/library/yql/tests/sql/dq_file/part19/canondata/result.json

+22
Original file line numberDiff line numberDiff line change
@@ -1713,6 +1713,28 @@
17131713
}
17141714
],
17151715
"test.test[optimizers-unused_columns_group_one_of_multi--Results]": [],
1716+
"test.test[optimizers-yql-17413-topsort--Analyze]": [
1717+
{
1718+
"checksum": "7e96e5445456cbfd2e3622966d474857",
1719+
"size": 8796,
1720+
"uri": "https://{canondata_backend}/1903885/35226bfda385a540a941f0d8ba61073fa4188835/resource.tar.gz#test.test_optimizers-yql-17413-topsort--Analyze_/plan.txt"
1721+
}
1722+
],
1723+
"test.test[optimizers-yql-17413-topsort--Debug]": [
1724+
{
1725+
"checksum": "8958442943f25fd8531144117a7279ba",
1726+
"size": 3948,
1727+
"uri": "https://{canondata_backend}/1903885/35226bfda385a540a941f0d8ba61073fa4188835/resource.tar.gz#test.test_optimizers-yql-17413-topsort--Debug_/opt.yql_patched"
1728+
}
1729+
],
1730+
"test.test[optimizers-yql-17413-topsort--Plan]": [
1731+
{
1732+
"checksum": "7e96e5445456cbfd2e3622966d474857",
1733+
"size": 8796,
1734+
"uri": "https://{canondata_backend}/1903885/35226bfda385a540a941f0d8ba61073fa4188835/resource.tar.gz#test.test_optimizers-yql-17413-topsort--Plan_/plan.txt"
1735+
}
1736+
],
1737+
"test.test[optimizers-yql-17413-topsort--Results]": [],
17161738
"test.test[optimizers-yql-9297_publish_ytcopy--Analyze]": [
17171739
{
17181740
"checksum": "ea9a6185cef1a4220dfde9576ddc292c",

‎ydb/library/yql/tests/sql/sql2yql/canondata/result.json

+14
Original file line numberDiff line numberDiff line change
@@ -9988,6 +9988,13 @@
99889988
"uri": "https://{canondata_backend}/1936997/00f46808be87e2ae2d4ac3ac45675b659c5ace45/resource.tar.gz#test_sql2yql.test_optimizers-yql-16134_/sql.yql"
99899989
}
99909990
],
9991+
"test_sql2yql.test[optimizers-yql-17413-topsort]": [
9992+
{
9993+
"checksum": "aa2443a9da120b817151ed97e57ee43c",
9994+
"size": 3527,
9995+
"uri": "https://{canondata_backend}/1781765/7c17af1b33d2fbf006a6d558af8cb8ec9ef4e4a9/resource.tar.gz#test_sql2yql.test_optimizers-yql-17413-topsort_/sql.yql"
9996+
}
9997+
],
99919998
"test_sql2yql.test[optimizers-yql-2171_aggregate_desc_sort_and_extract]": [
99929999
{
999310000
"checksum": "e46724f353c724da2d05f34ac86ebc12",
@@ -27222,6 +27229,13 @@
2722227229
"uri": "https://{canondata_backend}/1880306/64654158d6bfb1289c66c626a8162239289559d0/resource.tar.gz#test_sql_format.test_optimizers-yql-16134_/formatted.sql"
2722327230
}
2722427231
],
27232+
"test_sql_format.test[optimizers-yql-17413-topsort]": [
27233+
{
27234+
"checksum": "ecd0eaf6cba2da258637dba1c3c1ccec",
27235+
"size": 326,
27236+
"uri": "https://{canondata_backend}/1931696/e0f966d90e1f131295800c00caf45bfec971964e/resource.tar.gz#test_sql_format.test_optimizers-yql-17413-topsort_/formatted.sql"
27237+
}
27238+
],
2722527239
"test_sql_format.test[optimizers-yql-2171_aggregate_desc_sort_and_extract]": [
2722627240
{
2722727241
"checksum": "ea7ad8b12dbf78644e2a7ac94e10159b",

‎ydb/library/yql/tests/sql/suites/optimizers/sorted_desc.txt

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{"_yql_column_0"="\xE0\xC6\xCE\xCE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFC";"key"="911";"subkey"="2";"value"="kkk"};
2+
{"_yql_column_0"="\xE0\xC8\xC9\xCE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFC";"key"="761";"subkey"="6";"value"="ccc"};
3+
{"_yql_column_0"="\xE0\xCA\xCD\xC8\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFC";"key"="527";"subkey"="4";"value"="bbb"};
4+
{"_yql_column_0"="\xE0\xCD\xCF\xCF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFC";"key"="200";"subkey"="7";"value"="qqq"};
5+
{"_yql_column_0"="\xE0\xCE\xCA\xCF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFC";"key"="150";"subkey"="1";"value"="aaa"};
6+
{"_yql_column_0"="\xE0\xCE\xCA\xCF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFC";"key"="150";"subkey"="3";"value"="iii"};
7+
{"_yql_column_0"="\xE0\xCE\xCA\xCF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFC";"key"="150";"subkey"="8";"value"="zzz"};
8+
{"_yql_column_0"="\xE0\xCF\xC8\xCA\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFC";"key"="075";"subkey"="1";"value"="abc"};
9+
{"_yql_column_0"="\xE0\xCF\xCC\xC8\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFC";"key"="037";"subkey"="5";"value"="ddd"};
10+
{"_yql_column_0"="\xE0\xCF\xCD\xCC\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFC";"key"="023";"subkey"="3";"value"="aaa"};

‎ydb/library/yql/tests/sql/suites/optimizers/sorted_desc.txt.attr

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
{
2+
"_yql_row_spec" = {
3+
"SortMembers" = [
4+
"key"
5+
];
6+
"SortDirections" = [
7+
0
8+
];
9+
"UniqueKeys" = %false;
10+
"SortedByTypes" = [
11+
[
12+
"DataType";
13+
"String"
14+
]
15+
];
16+
"StrictSchema" = %true;
17+
"Type" = [
18+
"StructType";
19+
[
20+
[
21+
"key";
22+
[
23+
"DataType";
24+
"String"
25+
]
26+
];
27+
[
28+
"subkey";
29+
[
30+
"DataType";
31+
"String"
32+
]
33+
];
34+
[
35+
"value";
36+
[
37+
"DataType";
38+
"String"
39+
]
40+
]
41+
]
42+
];
43+
"SortedBy" = [
44+
"_yql_column_0"
45+
]
46+
}
47+
}

‎ydb/library/yql/tests/sql/suites/optimizers/yql-17413-topsort.cfg

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
in Input sorted_desc.txt
2+
res result.txt

‎ydb/library/yql/tests/sql/suites/optimizers/yql-17413-topsort.sql

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
-- ignore runonopt plan diff, skip force_blocks
2+
USE plato;
3+
4+
$filtered = select * from Input where value != "xxx";
5+
6+
select distinct(subkey) as subkey
7+
from (select * from $filtered order by key desc limit 3);
8+
9+
select sum(cast(subkey as int32)) as c from $filtered;

‎ydb/library/yql/tests/sql/yt_native_file/part19/canondata/result.json

+21
Original file line numberDiff line numberDiff line change
@@ -1490,6 +1490,27 @@
14901490
"uri": "https://{canondata_backend}/1937027/16b7289b1b8f5fdff728155d836fa2b238949b2d/resource.tar.gz#test.test_optimizers-unused_columns_group_one_of_multi--Results_/results.txt"
14911491
}
14921492
],
1493+
"test.test[optimizers-yql-17413-topsort--Debug]": [
1494+
{
1495+
"checksum": "a6b253a26321a218f3e46aa86dbe4c50",
1496+
"size": 5712,
1497+
"uri": "https://{canondata_backend}/1784117/b3e9aeb7a41a8fdbf9c6bfb4a2025eafe94d4a19/resource.tar.gz#test.test_optimizers-yql-17413-topsort--Debug_/opt.yql"
1498+
}
1499+
],
1500+
"test.test[optimizers-yql-17413-topsort--Plan]": [
1501+
{
1502+
"checksum": "0372943ac566f8ee02229da75bd4c1d2",
1503+
"size": 10305,
1504+
"uri": "https://{canondata_backend}/1784117/b3e9aeb7a41a8fdbf9c6bfb4a2025eafe94d4a19/resource.tar.gz#test.test_optimizers-yql-17413-topsort--Plan_/plan.txt"
1505+
}
1506+
],
1507+
"test.test[optimizers-yql-17413-topsort--Results]": [
1508+
{
1509+
"checksum": "d800fdb51897877b5908555f9f9499db",
1510+
"size": 1661,
1511+
"uri": "https://{canondata_backend}/1784117/b3e9aeb7a41a8fdbf9c6bfb4a2025eafe94d4a19/resource.tar.gz#test.test_optimizers-yql-17413-topsort--Results_/results.txt"
1512+
}
1513+
],
14931514
"test.test[optimizers-yql-9297_publish_ytcopy--Debug]": [
14941515
{
14951516
"checksum": "4dff550757ceeaceb30f02aae54b5d63",

0 commit comments

Comments
 (0)