[YQL-17474] Do FieldSubset optimizer on multi usage input (with feature flag)

ydb/library/yql/core/yql_type_annotation.h

@@ -269,6 +269,7 @@ struct TTypeAnnotationContext: public TThrRefBase {
     // compatibility with v0 or raw s-expression code
     bool OrderedColumns = false;
     TColumnOrderStorage::TPtr ColumnOrderStorage = new TColumnOrderStorage;
+    THashSet<TString> OptimizerFlags;
 
     TMaybe<TColumnOrder> LookupColumnOrder(const TExprNode& node) const;
     IGraphTransformer::TStatus SetColumnOrder(const TExprNode& node, const TColumnOrder& columnOrder, TExprContext& ctx);

ydb/library/yql/providers/config/yql_config_provider.cpp

@@ -900,6 +900,15 @@ namespace {
                     return false;
                 }
             }
+            else if (name == "OptimizerFlags") {
+                for (auto& arg : args) {
+                    if (arg.empty()) {
+                        ctx.AddError(TIssue(pos, "Empty flags are not supported"));
+                        return false;
+                    }
+                    Types.OptimizerFlags.insert(to_lower(ToString(arg)));
+                }
+            }
             else {
                 ctx.AddError(TIssue(pos, TStringBuilder() << "Unsupported command: " << name));
                 return false;

ydb/library/yql/tests/sql/dq_file/part14/canondata/result.json

@@ -326,6 +326,28 @@
         }
     ],
     "test.test[aggregate-group_by_rollup_rename-default.txt-Results]": [],
+    "test.test[aggregate-group_by_ru_join_simple_fs_multiusage--Analyze]": [
+        {
+            "checksum": "24740252670115e3bbd1772580e48adf",
+            "size": 9776,
+            "uri": "https://{canondata_backend}/1781765/75774e90f574004e23fc9aacf32e1f561a8c66ec/resource.tar.gz#test.test_aggregate-group_by_ru_join_simple_fs_multiusage--Analyze_/plan.txt"
+        }
+    ],
+    "test.test[aggregate-group_by_ru_join_simple_fs_multiusage--Debug]": [
+        {
+            "checksum": "94020f8a1e57024f031cdeecd89e11a9",
+            "size": 5934,
+            "uri": "https://{canondata_backend}/1781765/75774e90f574004e23fc9aacf32e1f561a8c66ec/resource.tar.gz#test.test_aggregate-group_by_ru_join_simple_fs_multiusage--Debug_/opt.yql_patched"
+        }
+    ],
+    "test.test[aggregate-group_by_ru_join_simple_fs_multiusage--Plan]": [
+        {
+            "checksum": "24740252670115e3bbd1772580e48adf",
+            "size": 9776,
+            "uri": "https://{canondata_backend}/1781765/75774e90f574004e23fc9aacf32e1f561a8c66ec/resource.tar.gz#test.test_aggregate-group_by_ru_join_simple_fs_multiusage--Plan_/plan.txt"
+        }
+    ],
+    "test.test[aggregate-group_by_ru_join_simple_fs_multiusage--Results]": [],
     "test.test[ansi_idents-escaped_udf_name-default.txt-Analyze]": [
         {
             "checksum": "0f0658d4cb6fcef484dd34676f7939e7",

ydb/library/yql/tests/sql/dq_file/part3/canondata/result.json

@@ -1500,6 +1500,28 @@
         }
     ],
     "test.test[optimizers-direct_row_after_merge--Results]": [],
+    "test.test[optimizers-length_over_merge_fs_multiusage--Analyze]": [
+        {
+            "checksum": "212be881133a20b5b73ef1250dbeda51",
+            "size": 960,
+            "uri": "https://{canondata_backend}/1599023/892497444bbacbe92ad2c557c09c697b859ad48d/resource.tar.gz#test.test_optimizers-length_over_merge_fs_multiusage--Analyze_/plan.txt"
+        }
+    ],
+    "test.test[optimizers-length_over_merge_fs_multiusage--Debug]": [
+        {
+            "checksum": "880889c8fc79fc17434df304ebe066fb",
+            "size": 2791,
+            "uri": "https://{canondata_backend}/1599023/892497444bbacbe92ad2c557c09c697b859ad48d/resource.tar.gz#test.test_optimizers-length_over_merge_fs_multiusage--Debug_/opt.yql_patched"
+        }
+    ],
+    "test.test[optimizers-length_over_merge_fs_multiusage--Plan]": [
+        {
+            "checksum": "212be881133a20b5b73ef1250dbeda51",
+            "size": 960,
+            "uri": "https://{canondata_backend}/1599023/892497444bbacbe92ad2c557c09c697b859ad48d/resource.tar.gz#test.test_optimizers-length_over_merge_fs_multiusage--Plan_/plan.txt"
+        }
+    ],
+    "test.test[optimizers-length_over_merge_fs_multiusage--Results]": [],
     "test.test[optimizers-yql-7324_duplicate_arg--Analyze]": [
         {
             "checksum": "b913ead12af51bc046e1f3344ff5134c",

ydb/library/yql/tests/sql/sql2yql/canondata/result.json

@@ -2267,6 +2267,13 @@
             "uri": "https://{canondata_backend}/1936997/d01e529b02a008637591c42bdd9c20f7eeda6588/resource.tar.gz#test_sql2yql.test_aggregate-group_by_ru_join_simple_/sql.yql"
         }
     ],
+    "test_sql2yql.test[aggregate-group_by_ru_join_simple_fs_multiusage]": [
+        {
+            "checksum": "6e0fe40b211146a9bdab0d2e61d26168",
+            "size": 6000,
+            "uri": "https://{canondata_backend}/1599023/c4917fc01ae646b5e708e20b50ea7ae2325f5bb7/resource.tar.gz#test_sql2yql.test_aggregate-group_by_ru_join_simple_fs_multiusage_/sql.yql"
+        }
+    ],
     "test_sql2yql.test[aggregate-group_by_ru_join_star]": [
         {
             "checksum": "1d817501fac482312d72ad2e4a7881ba",
@@ -9785,6 +9792,13 @@
             "uri": "https://{canondata_backend}/1937027/973c239492ba32946806ddc66cf0af4b38c06ae8/resource.tar.gz#test_sql2yql.test_optimizers-length_over_merge_/sql.yql"
         }
     ],
+    "test_sql2yql.test[optimizers-length_over_merge_fs_multiusage]": [
+        {
+            "checksum": "a923d44d0dc694741762e7272690257f",
+            "size": 4487,
+            "uri": "https://{canondata_backend}/1599023/c4917fc01ae646b5e708e20b50ea7ae2325f5bb7/resource.tar.gz#test_sql2yql.test_optimizers-length_over_merge_fs_multiusage_/sql.yql"
+        }
+    ],
     "test_sql2yql.test[optimizers-multi_to_empty_constraint]": [
         {
             "checksum": "4288eb5377c92428681954e99403cbf0",
@@ -19557,6 +19571,13 @@
             "uri": "https://{canondata_backend}/1880306/64654158d6bfb1289c66c626a8162239289559d0/resource.tar.gz#test_sql_format.test_aggregate-group_by_ru_join_simple_/formatted.sql"
         }
     ],
+    "test_sql_format.test[aggregate-group_by_ru_join_simple_fs_multiusage]": [
+        {
+            "checksum": "0369dc43a2062b648a486f7efb2873d0",
+            "size": 334,
+            "uri": "https://{canondata_backend}/1599023/c4917fc01ae646b5e708e20b50ea7ae2325f5bb7/resource.tar.gz#test_sql_format.test_aggregate-group_by_ru_join_simple_fs_multiusage_/formatted.sql"
+        }
+    ],
     "test_sql_format.test[aggregate-group_by_ru_join_star]": [
         {
             "checksum": "e4a4a97de8afd6e188163c71da45788d",
@@ -27075,6 +27096,13 @@
             "uri": "https://{canondata_backend}/1880306/64654158d6bfb1289c66c626a8162239289559d0/resource.tar.gz#test_sql_format.test_optimizers-length_over_merge_/formatted.sql"
         }
     ],
+    "test_sql_format.test[optimizers-length_over_merge_fs_multiusage]": [
+        {
+            "checksum": "81f76674682498288b92904264c7a0cd",
+            "size": 498,
+            "uri": "https://{canondata_backend}/1599023/c4917fc01ae646b5e708e20b50ea7ae2325f5bb7/resource.tar.gz#test_sql_format.test_optimizers-length_over_merge_fs_multiusage_/formatted.sql"
+        }
+    ],
     "test_sql_format.test[optimizers-multi_to_empty_constraint]": [
         {
             "checksum": "417fc8b1ee413a204123ed56af1aa32f",

ydb/library/yql/tests/sql/suites/aggregate/group_by_ru_join_simple_fs_multiusage.cfg

@@ -0,0 +1 @@
+in Input input_intersect.txt

ydb/library/yql/tests/sql/suites/aggregate/group_by_ru_join_simple_fs_multiusage.sql

@@ -0,0 +1,6 @@
+/* syntax version 1 */
+/* postgres can not */
+
+pragma sampleselect;
+pragma config.flags("OptimizerFlags", "FieldSubsetEnableMultiusage");
+select kk, sk, max(t2.subkey) as ss FROM plato.Input AS t1 INNER JOIN plato.Input AS t2 USING (key) GROUP BY ROLLUP(t1.key as kk, t1.subkey as sk) ORDER BY kk, sk;

ydb/library/yql/tests/sql/suites/optimizers/length_over_merge_fs_multiusage.cfg

@@ -0,0 +1,2 @@
+in Input input3.txt
+res result.txt

ydb/library/yql/tests/sql/suites/optimizers/length_over_merge_fs_multiusage.sql

@@ -0,0 +1,14 @@
+/* postgres can not */
+pragma config.flags("OptimizerFlags", "FieldSubsetEnableMultiusage");
+
+use plato;
+
+$input = (select key, key || subkey as subkey, value from Input);
+
+$total_count = (select count(1) from $input);
+
+$filtered = (select * from $input where key in ("023", "037", "075"));
+
+$filtered_cnt = (select count(1) from $filtered);
+
+select $filtered_cnt / cast($total_count as Double) as cnt;

ydb/library/yql/tests/sql/yt_native_file/part14/canondata/result.json

@@ -296,6 +296,27 @@
             "uri": "https://{canondata_backend}/1923547/5154c8bd8ef9ead4f609771f831f20c15e795571/resource.tar.gz#test.test_aggregate-group_by_rollup_rename-default.txt-Results_/results.txt"
         }
     ],
+    "test.test[aggregate-group_by_ru_join_simple_fs_multiusage--Debug]": [
+        {
+            "checksum": "ceeb8af83f2b01a10deebe2f805a3d01",
+            "size": 9322,
+            "uri": "https://{canondata_backend}/1936997/61b460f95ad76ee0a1465a60da61733fdb91dab4/resource.tar.gz#test.test_aggregate-group_by_ru_join_simple_fs_multiusage--Debug_/opt.yql"
+        }
+    ],
+    "test.test[aggregate-group_by_ru_join_simple_fs_multiusage--Plan]": [
+        {
+            "checksum": "760c79b2808d478d74cbc5d7682ae91d",
+            "size": 12501,
+            "uri": "https://{canondata_backend}/1936997/61b460f95ad76ee0a1465a60da61733fdb91dab4/resource.tar.gz#test.test_aggregate-group_by_ru_join_simple_fs_multiusage--Plan_/plan.txt"
+        }
+    ],
+    "test.test[aggregate-group_by_ru_join_simple_fs_multiusage--Results]": [
+        {
+            "checksum": "2469d61da89000051d881cc1a1aceb8a",
+            "size": 7925,
+            "uri": "https://{canondata_backend}/1936997/61b460f95ad76ee0a1465a60da61733fdb91dab4/resource.tar.gz#test.test_aggregate-group_by_ru_join_simple_fs_multiusage--Results_/results.txt"
+        }
+    ],
     "test.test[ansi_idents-escaped_udf_name-default.txt-Debug]": [
         {
             "checksum": "4988877c69725bebc3eb77a48625f5cd",

ydb/library/yql/tests/sql/yt_native_file/part3/canondata/result.json

@@ -1214,6 +1214,27 @@
             "uri": "https://{canondata_backend}/1881367/66c7cb3390a7d019f5c9ce886d24fe86e37e82d0/resource.tar.gz#test.test_optimizers-direct_row_after_merge--Results_/results.txt"
         }
     ],
+    "test.test[optimizers-length_over_merge_fs_multiusage--Debug]": [
+        {
+            "checksum": "e27c0e4ca190c9095c7cdfc6031ab1ea",
+            "size": 1740,
+            "uri": "https://{canondata_backend}/1889210/f5cbb7beff327fcfa5cbcdb1e04fd02c24f4e33f/resource.tar.gz#test.test_optimizers-length_over_merge_fs_multiusage--Debug_/opt.yql"
+        }
+    ],
+    "test.test[optimizers-length_over_merge_fs_multiusage--Plan]": [
+        {
+            "checksum": "17aa2b0bb49eb66fecaf1b350cccaabc",
+            "size": 5190,
+            "uri": "https://{canondata_backend}/1889210/f5cbb7beff327fcfa5cbcdb1e04fd02c24f4e33f/resource.tar.gz#test.test_optimizers-length_over_merge_fs_multiusage--Plan_/plan.txt"
+        }
+    ],
+    "test.test[optimizers-length_over_merge_fs_multiusage--Results]": [
+        {
+            "checksum": "5d1098186fdb44867314966f03aded2a",
+            "size": 857,
+            "uri": "https://{canondata_backend}/1889210/f5cbb7beff327fcfa5cbcdb1e04fd02c24f4e33f/resource.tar.gz#test.test_optimizers-length_over_merge_fs_multiusage--Results_/results.txt"
+        }
+    ],
     "test.test[optimizers-yql-7324_duplicate_arg--Debug]": [
         {
             "checksum": "3bd44f4f74db4b23c13174edccd97e7d",