[YQL-17625] Fix ORDER BY by column missing in projections

ydb/library/yql/sql/v1/select.cpp

@@ -1736,7 +1736,16 @@ class TSelectCore: public IRealSource {
             return nullptr;
         }
 
-        return Y("let", label, BuildSortSpec(OrderBy, label, false, AssumeSorted));
+        auto sorted = BuildSortSpec(OrderBy, label, false, AssumeSorted);
+        if (ExtraSortColumns.empty()) {
+            return Y("let", label, sorted);
+        }
+        auto body = Y();
+        for (const auto& [column, _] : ExtraSortColumns) {
+            body = L(body, Y("let", "row", Y("RemoveMember", "row", Q(column))));
+        }
+        body = L(body, Y("let", "res", "row"));
+        return Y("let", label, Y("OrderedMap", sorted, BuildLambda(Pos, Y("row"), body, "res")));
     }
 
     TNodePtr BuildCleanupColumns(TContext& ctx, const TString& label) override {
@@ -1801,6 +1810,31 @@ class TSelectCore: public IRealSource {
             }
             return Source->AddColumn(ctx, column);
         }
+
+        if (OrderByInit && !Distinct && !GroupBy) {
+            bool reliable = column.IsReliable();
+            column.SetAsNotReliable();
+            auto maybeExist = IRealSource::AddColumn(ctx, column);
+            if (reliable) {
+                column.ResetAsReliable();
+            }
+            if (maybeExist && maybeExist.GetRef()) {
+                return true;
+            }
+
+            auto maybeSourceExist = Source->AddColumn(ctx, column);
+            if (!maybeSourceExist.Defined()) {
+                return maybeSourceExist;
+            }
+
+            // order by references column which is missing in projection, but may exists in source
+            const auto columnName = column.GetColumnName();
+            if (columnName) {
+                ExtraSortColumns.emplace(*columnName, TNodePtr(&column));
+            }
+            return true;
+        }
+
         return IRealSource::AddColumn(ctx, column);
     }
 
@@ -2139,6 +2173,17 @@ class TSelectCore: public IRealSource {
                 ++column;
                 ++isNamedColumn;
             }
+
+            for (const auto& [columnName, column]: ExtraSortColumns) {
+                auto body = Y();
+                if (haveCompositeTerms) {
+                    body = L(body, Y("let", "row", Y("Apply", "addCompositTerms", "row")));
+                }
+                body = L(body, Y("let", "res", column));
+                TPosition pos = column->GetPos();
+                auto projectItem = Y("SqlProjectItem", "projectCoreType", BuildQuotedAtom(pos, columnName), BuildLambda(pos, Y("row"), body, "res"));
+                sqlProjectArgs = L(sqlProjectArgs, projectItem);
+            }
         }
 
         auto block(Y(Y("let", "projectCoreType", Y("TypeOf", "core"))));
@@ -2199,6 +2244,7 @@ class TSelectCore: public IRealSource {
     const bool SelectStream;
     const TWriteSettings Settings;
     const TColumnsSets UniqueSets, DistinctSets;
+    TMap<TString, TNodePtr> ExtraSortColumns;
 };
 
 class TProcessSource: public IRealSource {

ydb/library/yql/sql/v1/sql_ut.cpp

@@ -2950,6 +2950,11 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) {
         UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:45: Error: You should use in ORDER BY column name, qualified field, callable function or expression\n");
     }
 
+    Y_UNIT_TEST(ErrorsInOrderByWhenColumnIsMissingInProjection) {
+        ExpectFailWithError("select subkey from (select 1 as subkey) order by key", "<main>:1:50: Error: Column key is not in source column set\n");
+        ExpectFailWithError("select subkey from plato.Input as a order by x.key", "<main>:1:46: Error: Unknown correlation name: x\n");
+    }
+
     Y_UNIT_TEST(SelectAggregatedWhere) {
         NYql::TAstParseResult res = SqlToYql("select * from plato.Input where count(key)");
         UNIT_ASSERT(!res.Root);
@@ -3202,12 +3207,6 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) {
         UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:1: Error: Duplicate column: key\n");
     }
 
-    Y_UNIT_TEST(SelectOrderByUnknownLabel) {
-        NYql::TAstParseResult res = SqlToYql("select a from plato.Input order by b");
-        UNIT_ASSERT(!res.Root);
-        UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:36: Error: Column b is not in source column set. Did you mean a?\n");
-    }
-
     Y_UNIT_TEST(SelectFlattenBySameColumns) {
         NYql::TAstParseResult res = SqlToYql("select key from plato.Input flatten by (key, key as kk)");
         UNIT_ASSERT(!res.Root);

ydb/library/yql/tests/sql/dq_file/part6/canondata/result.json

@@ -1821,6 +1821,28 @@
         }
     ],
     "test.test[order_by-literal_single_item_sort--Results]": [],
+    "test.test[order_by-order_by_missing_project_column-default.txt-Analyze]": [
+        {
+            "checksum": "b5c2e8e2b9513028037ab15707571f52",
+            "size": 14060,
+            "uri": "https://{canondata_backend}/1871002/2fcd813e80c98be1c3c62b9a854d09ccca8851f8/resource.tar.gz#test.test_order_by-order_by_missing_project_column-default.txt-Analyze_/plan.txt"
+        }
+    ],
+    "test.test[order_by-order_by_missing_project_column-default.txt-Debug]": [
+        {
+            "checksum": "749e10f58df2a7d1b43d444d02d08b4c",
+            "size": 4022,
+            "uri": "https://{canondata_backend}/1871002/2fcd813e80c98be1c3c62b9a854d09ccca8851f8/resource.tar.gz#test.test_order_by-order_by_missing_project_column-default.txt-Debug_/opt.yql_patched"
+        }
+    ],
+    "test.test[order_by-order_by_missing_project_column-default.txt-Plan]": [
+        {
+            "checksum": "b5c2e8e2b9513028037ab15707571f52",
+            "size": 14060,
+            "uri": "https://{canondata_backend}/1871002/2fcd813e80c98be1c3c62b9a854d09ccca8851f8/resource.tar.gz#test.test_order_by-order_by_missing_project_column-default.txt-Plan_/plan.txt"
+        }
+    ],
+    "test.test[order_by-order_by_missing_project_column-default.txt-Results]": [],
     "test.test[order_by-sort_with_take--Analyze]": [
         {
             "checksum": "c78b6f43c8394d398e0c63aa70d6458c",

ydb/library/yql/tests/sql/sql2yql/canondata/result.json

@@ -10457,6 +10457,13 @@
             "uri": "https://{canondata_backend}/1936947/659b615f15086142a8960946dabd06b519d43335/resource.tar.gz#test_sql2yql.test_order_by-order_by_list_of_strings_/sql.yql"
         }
     ],
+    "test_sql2yql.test[order_by-order_by_missing_project_column]": [
+        {
+            "checksum": "5279857b1fc3341d168708325cb3f123",
+            "size": 10491,
+            "uri": "https://{canondata_backend}/1937429/114961b26476797b95763679349a6ea2e9f8cea8/resource.tar.gz#test_sql2yql.test_order_by-order_by_missing_project_column_/sql.yql"
+        }
+    ],
     "test_sql2yql.test[order_by-order_by_mul_columns]": [
         {
             "checksum": "3e69e4405b6dff02d1127352fe46de57",
@@ -27936,6 +27943,13 @@
             "uri": "https://{canondata_backend}/1880306/64654158d6bfb1289c66c626a8162239289559d0/resource.tar.gz#test_sql_format.test_order_by-order_by_list_of_strings_/formatted.sql"
         }
     ],
+    "test_sql_format.test[order_by-order_by_missing_project_column]": [
+        {
+            "checksum": "4cc3522c6b6eb8309231e398a855394f",
+            "size": 506,
+            "uri": "https://{canondata_backend}/1937429/114961b26476797b95763679349a6ea2e9f8cea8/resource.tar.gz#test_sql_format.test_order_by-order_by_missing_project_column_/formatted.sql"
+        }
+    ],
     "test_sql_format.test[order_by-order_by_mul_columns]": [
         {
             "checksum": "ecd62b80c49799026e8275d8c3daeaef",

ydb/library/yql/tests/sql/suites/order_by/order_by_missing_project_column.sql

@@ -0,0 +1,11 @@
+use plato;
+
+select subkey from Input order by key, value;
+select subkey from Input order by "x" || key, value;
+select subkey from Input order by key || "x" limit 3;
+
+select subkey from Input as a order by "x" || key, a.value limit 3;
+select subkey from Input as a order by a.key, value limit 1;
+select subkey from Input as a order by key, value limit 2;
+
+select subkey, key from Input order by key, value;

ydb/library/yql/tests/sql/yt_native_file/part6/canondata/result.json

@@ -1565,6 +1565,27 @@
             "uri": "https://{canondata_backend}/1936997/a879c594903fb53f2cbc586abd0c0e9fac5dd32a/resource.tar.gz#test.test_order_by-literal_single_item_sort--Results_/Output.txt"
         }
     ],
+    "test.test[order_by-order_by_missing_project_column-default.txt-Debug]": [
+        {
+            "checksum": "ad11528b26030156b137c66b82500102",
+            "size": 6459,
+            "uri": "https://{canondata_backend}/1871002/3978b39df1ca2a819856a9709f7dca5f5a74e2ac/resource.tar.gz#test.test_order_by-order_by_missing_project_column-default.txt-Debug_/opt.yql"
+        }
+    ],
+    "test.test[order_by-order_by_missing_project_column-default.txt-Plan]": [
+        {
+            "checksum": "1926e1707b041bf34e0b061ad7ae3ec0",
+            "size": 19658,
+            "uri": "https://{canondata_backend}/1871002/3978b39df1ca2a819856a9709f7dca5f5a74e2ac/resource.tar.gz#test.test_order_by-order_by_missing_project_column-default.txt-Plan_/plan.txt"
+        }
+    ],
+    "test.test[order_by-order_by_missing_project_column-default.txt-Results]": [
+        {
+            "checksum": "0bd033a7c39d748d49027400811d2547",
+            "size": 6050,
+            "uri": "https://{canondata_backend}/1871002/3978b39df1ca2a819856a9709f7dca5f5a74e2ac/resource.tar.gz#test.test_order_by-order_by_missing_project_column-default.txt-Results_/results.txt"
+        }
+    ],
     "test.test[order_by-sort_with_take--Debug]": [
         {
             "checksum": "ea8b90a7be2c019a9ee9ace75d9abd90",