Skip to content

Commit 0972511

Browse files
nepalnepal
authored
[YQL-17625] Fix ORDER BY by column missing in projections (#1617)
1 parent 1c12b54 commit 0972511

File tree

6 files changed

+120
-7
lines changed

6 files changed

+120
-7
lines changed

‎ydb/library/yql/sql/v1/select.cpp

+47-1
Original file line numberDiff line numberDiff line change
@@ -1736,7 +1736,16 @@ class TSelectCore: public IRealSource {
17361736
return nullptr;
17371737
}
17381738

1739-
return Y("let", label, BuildSortSpec(OrderBy, label, false, AssumeSorted));
1739+
auto sorted = BuildSortSpec(OrderBy, label, false, AssumeSorted);
1740+
if (ExtraSortColumns.empty()) {
1741+
return Y("let", label, sorted);
1742+
}
1743+
auto body = Y();
1744+
for (const auto& [column, _] : ExtraSortColumns) {
1745+
body = L(body, Y("let", "row", Y("RemoveMember", "row", Q(column))));
1746+
}
1747+
body = L(body, Y("let", "res", "row"));
1748+
return Y("let", label, Y("OrderedMap", sorted, BuildLambda(Pos, Y("row"), body, "res")));
17401749
}
17411750

17421751
TNodePtr BuildCleanupColumns(TContext& ctx, const TString& label) override {
@@ -1801,6 +1810,31 @@ class TSelectCore: public IRealSource {
18011810
}
18021811
return Source->AddColumn(ctx, column);
18031812
}
1813+
1814+
if (OrderByInit && !Distinct && !GroupBy) {
1815+
bool reliable = column.IsReliable();
1816+
column.SetAsNotReliable();
1817+
auto maybeExist = IRealSource::AddColumn(ctx, column);
1818+
if (reliable) {
1819+
column.ResetAsReliable();
1820+
}
1821+
if (maybeExist && maybeExist.GetRef()) {
1822+
return true;
1823+
}
1824+
1825+
auto maybeSourceExist = Source->AddColumn(ctx, column);
1826+
if (!maybeSourceExist.Defined()) {
1827+
return maybeSourceExist;
1828+
}
1829+
1830+
// order by references column which is missing in projection, but may exists in source
1831+
const auto columnName = column.GetColumnName();
1832+
if (columnName) {
1833+
ExtraSortColumns.emplace(*columnName, TNodePtr(&column));
1834+
}
1835+
return true;
1836+
}
1837+
18041838
return IRealSource::AddColumn(ctx, column);
18051839
}
18061840

@@ -2139,6 +2173,17 @@ class TSelectCore: public IRealSource {
21392173
++column;
21402174
++isNamedColumn;
21412175
}
2176+
2177+
for (const auto& [columnName, column]: ExtraSortColumns) {
2178+
auto body = Y();
2179+
if (haveCompositeTerms) {
2180+
body = L(body, Y("let", "row", Y("Apply", "addCompositTerms", "row")));
2181+
}
2182+
body = L(body, Y("let", "res", column));
2183+
TPosition pos = column->GetPos();
2184+
auto projectItem = Y("SqlProjectItem", "projectCoreType", BuildQuotedAtom(pos, columnName), BuildLambda(pos, Y("row"), body, "res"));
2185+
sqlProjectArgs = L(sqlProjectArgs, projectItem);
2186+
}
21422187
}
21432188

21442189
auto block(Y(Y("let", "projectCoreType", Y("TypeOf", "core"))));
@@ -2199,6 +2244,7 @@ class TSelectCore: public IRealSource {
21992244
const bool SelectStream;
22002245
const TWriteSettings Settings;
22012246
const TColumnsSets UniqueSets, DistinctSets;
2247+
TMap<TString, TNodePtr> ExtraSortColumns;
22022248
};
22032249

22042250
class TProcessSource: public IRealSource {

‎ydb/library/yql/sql/v1/sql_ut.cpp

+5-6
Original file line numberDiff line numberDiff line change
@@ -2950,6 +2950,11 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) {
29502950
UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:45: Error: You should use in ORDER BY column name, qualified field, callable function or expression\n");
29512951
}
29522952

2953+
Y_UNIT_TEST(ErrorsInOrderByWhenColumnIsMissingInProjection) {
2954+
ExpectFailWithError("select subkey from (select 1 as subkey) order by key", "<main>:1:50: Error: Column key is not in source column set\n");
2955+
ExpectFailWithError("select subkey from plato.Input as a order by x.key", "<main>:1:46: Error: Unknown correlation name: x\n");
2956+
}
2957+
29532958
Y_UNIT_TEST(SelectAggregatedWhere) {
29542959
NYql::TAstParseResult res = SqlToYql("select * from plato.Input where count(key)");
29552960
UNIT_ASSERT(!res.Root);
@@ -3202,12 +3207,6 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) {
32023207
UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Duplicate column: key\n");
32033208
}
32043209

3205-
Y_UNIT_TEST(SelectOrderByUnknownLabel) {
3206-
NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by b");
3207-
UNIT_ASSERT(!res.Root);
3208-
UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:36: Error: Column b is not in source column set. Did you mean a?\n");
3209-
}
3210-
32113210
Y_UNIT_TEST(SelectFlattenBySameColumns) {
32123211
NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key, key as kk)");
32133212
UNIT_ASSERT(!res.Root);

‎ydb/library/yql/tests/sql/dq_file/part6/canondata/result.json

+22
Original file line numberDiff line numberDiff line change
@@ -1821,6 +1821,28 @@
18211821
}
18221822
],
18231823
"test.test[order_by-literal_single_item_sort--Results]": [],
1824+
"test.test[order_by-order_by_missing_project_column-default.txt-Analyze]": [
1825+
{
1826+
"checksum": "b5c2e8e2b9513028037ab15707571f52",
1827+
"size": 14060,
1828+
"uri": "https://{canondata_backend}/1871002/2fcd813e80c98be1c3c62b9a854d09ccca8851f8/resource.tar.gz#test.test_order_by-order_by_missing_project_column-default.txt-Analyze_/plan.txt"
1829+
}
1830+
],
1831+
"test.test[order_by-order_by_missing_project_column-default.txt-Debug]": [
1832+
{
1833+
"checksum": "749e10f58df2a7d1b43d444d02d08b4c",
1834+
"size": 4022,
1835+
"uri": "https://{canondata_backend}/1871002/2fcd813e80c98be1c3c62b9a854d09ccca8851f8/resource.tar.gz#test.test_order_by-order_by_missing_project_column-default.txt-Debug_/opt.yql_patched"
1836+
}
1837+
],
1838+
"test.test[order_by-order_by_missing_project_column-default.txt-Plan]": [
1839+
{
1840+
"checksum": "b5c2e8e2b9513028037ab15707571f52",
1841+
"size": 14060,
1842+
"uri": "https://{canondata_backend}/1871002/2fcd813e80c98be1c3c62b9a854d09ccca8851f8/resource.tar.gz#test.test_order_by-order_by_missing_project_column-default.txt-Plan_/plan.txt"
1843+
}
1844+
],
1845+
"test.test[order_by-order_by_missing_project_column-default.txt-Results]": [],
18241846
"test.test[order_by-sort_with_take--Analyze]": [
18251847
{
18261848
"checksum": "c78b6f43c8394d398e0c63aa70d6458c",

‎ydb/library/yql/tests/sql/sql2yql/canondata/result.json

+14
Original file line numberDiff line numberDiff line change
@@ -10464,6 +10464,13 @@
1046410464
"uri": "https://{canondata_backend}/1936947/659b615f15086142a8960946dabd06b519d43335/resource.tar.gz#test_sql2yql.test_order_by-order_by_list_of_strings_/sql.yql"
1046510465
}
1046610466
],
10467+
"test_sql2yql.test[order_by-order_by_missing_project_column]": [
10468+
{
10469+
"checksum": "5279857b1fc3341d168708325cb3f123",
10470+
"size": 10491,
10471+
"uri": "https://{canondata_backend}/1937429/114961b26476797b95763679349a6ea2e9f8cea8/resource.tar.gz#test_sql2yql.test_order_by-order_by_missing_project_column_/sql.yql"
10472+
}
10473+
],
1046710474
"test_sql2yql.test[order_by-order_by_mul_columns]": [
1046810475
{
1046910476
"checksum": "3e69e4405b6dff02d1127352fe46de57",
@@ -27950,6 +27957,13 @@
2795027957
"uri": "https://{canondata_backend}/1880306/64654158d6bfb1289c66c626a8162239289559d0/resource.tar.gz#test_sql_format.test_order_by-order_by_list_of_strings_/formatted.sql"
2795127958
}
2795227959
],
27960+
"test_sql_format.test[order_by-order_by_missing_project_column]": [
27961+
{
27962+
"checksum": "4cc3522c6b6eb8309231e398a855394f",
27963+
"size": 506,
27964+
"uri": "https://{canondata_backend}/1937429/114961b26476797b95763679349a6ea2e9f8cea8/resource.tar.gz#test_sql_format.test_order_by-order_by_missing_project_column_/formatted.sql"
27965+
}
27966+
],
2795327967
"test_sql_format.test[order_by-order_by_mul_columns]": [
2795427968
{
2795527969
"checksum": "ecd62b80c49799026e8275d8c3daeaef",

‎ydb/library/yql/tests/sql/suites/order_by/order_by_missing_project_column.sql

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
use plato;
2+
3+
select subkey from Input order by key, value;
4+
select subkey from Input order by "x" || key, value;
5+
select subkey from Input order by key || "x" limit 3;
6+
7+
select subkey from Input as a order by "x" || key, a.value limit 3;
8+
select subkey from Input as a order by a.key, value limit 1;
9+
select subkey from Input as a order by key, value limit 2;
10+
11+
select subkey, key from Input order by key, value;

‎ydb/library/yql/tests/sql/yt_native_file/part6/canondata/result.json

+21
Original file line numberDiff line numberDiff line change
@@ -1565,6 +1565,27 @@
15651565
"uri": "https://{canondata_backend}/1936997/a879c594903fb53f2cbc586abd0c0e9fac5dd32a/resource.tar.gz#test.test_order_by-literal_single_item_sort--Results_/Output.txt"
15661566
}
15671567
],
1568+
"test.test[order_by-order_by_missing_project_column-default.txt-Debug]": [
1569+
{
1570+
"checksum": "ad11528b26030156b137c66b82500102",
1571+
"size": 6459,
1572+
"uri": "https://{canondata_backend}/1871002/3978b39df1ca2a819856a9709f7dca5f5a74e2ac/resource.tar.gz#test.test_order_by-order_by_missing_project_column-default.txt-Debug_/opt.yql"
1573+
}
1574+
],
1575+
"test.test[order_by-order_by_missing_project_column-default.txt-Plan]": [
1576+
{
1577+
"checksum": "1926e1707b041bf34e0b061ad7ae3ec0",
1578+
"size": 19658,
1579+
"uri": "https://{canondata_backend}/1871002/3978b39df1ca2a819856a9709f7dca5f5a74e2ac/resource.tar.gz#test.test_order_by-order_by_missing_project_column-default.txt-Plan_/plan.txt"
1580+
}
1581+
],
1582+
"test.test[order_by-order_by_missing_project_column-default.txt-Results]": [
1583+
{
1584+
"checksum": "0bd033a7c39d748d49027400811d2547",
1585+
"size": 6050,
1586+
"uri": "https://{canondata_backend}/1871002/3978b39df1ca2a819856a9709f7dca5f5a74e2ac/resource.tar.gz#test.test_order_by-order_by_missing_project_column-default.txt-Results_/results.txt"
1587+
}
1588+
],
15681589
"test.test[order_by-sort_with_take--Debug]": [
15691590
{
15701591
"checksum": "ea8b90a7be2c019a9ee9ace75d9abd90",

0 commit comments

Comments
 (0)